Skip to content

Add sandbox failures to SARIF report (#1276) #1364

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Nov 16, 2022
Merged

Add sandbox failures to SARIF report (#1276) #1364

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1c9571e
Add sandbox failures to SARIF report (#1276)
mmvpm Nov 14, 2022
74d6ac2
Add unit test
mmvpm Nov 14, 2022
88a87fd
Merge branch 'main' into ideaseeker/sandbox_failure_in_sarif
mmvpm Nov 16, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 14 additions & 6 deletions utbot-framework-test/src/test/kotlin/org/utbot/sarif/SarifReportTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,7 @@ package org.utbot.sarif

import org.junit.Test
import org.mockito.Mockito
import org.utbot.framework.plugin.api.ExecutableId
import org.utbot.framework.plugin.api.UtExecution
import org.utbot.framework.plugin.api.UtImplicitlyThrownException
import org.utbot.framework.plugin.api.UtPrimitiveModel
import org.utbot.framework.plugin.api.UtMethodTestSet
import org.utbot.framework.plugin.api.UtSymbolicExecution
import org.utbot.framework.plugin.api.*

class SarifReportTest {

Expand Down Expand Up @@ -137,6 +132,19 @@ class SarifReportTest {
}
}

@Test
fun testProcessSandboxFailure() {
mockUtMethodNames()

val uncheckedException = Mockito.mock(java.security.AccessControlException::class.java)
Mockito.`when`(uncheckedException.stackTrace).thenReturn(arrayOf())
Mockito.`when`(mockUtExecution.result).thenReturn(UtSandboxFailure(uncheckedException))

val report = sarifReportMain.createReport()
val result = report.runs.first().results.first()
assert(result.message.text.contains("AccessControlException"))
}

@Test
fun testCodeFlowsStartsWithMethodCall() {
mockUtMethodNames()
Expand Down
13 changes: 10 additions & 3 deletions utbot-framework/src/main/kotlin/org/utbot/sarif/SarifReport.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ class SarifReport(

// prepending stack trace by `method` call in generated tests
val methodCallLocation: SarifPhysicalLocation? =
findMethodCallInTestBody(utExecution.testMethodName, method.name)
findMethodCallInTestBody(utExecution.testMethodName, method.name, utExecution)
if (methodCallLocation != null) {
val testFileName = sourceFinding.testsRelativePath.toPath().fileName
val testClassName = testFileName.nameWithoutExtension
Expand Down Expand Up @@ -255,9 +255,15 @@ class SarifReport(
generatedTestsCode.split('\n')
}

private fun findMethodCallInTestBody(testMethodName: String?, methodName: String): SarifPhysicalLocation? {
private fun findMethodCallInTestBody(
testMethodName: String?,
methodName: String,
utExecution: UtExecution,
): SarifPhysicalLocation? {
if (testMethodName == null)
return null
if (utExecution.result is UtSandboxFailure) // if there is no method call in test
return getRelatedLocations(utExecution).firstOrNull()?.physicalLocation

// searching needed test
val testMethodStartsAt = testsBodyLines.indexOfFirst { line ->
Expand Down Expand Up @@ -343,6 +349,7 @@ class SarifReport(
val implicitlyThrown = result is UtImplicitlyThrownException
val overflowFailure = result is UtOverflowFailure && UtSettings.treatOverflowAsError
val assertionError = result is UtExplicitlyThrownException && result.exception is AssertionError
return implicitlyThrown || overflowFailure || assertionError
val sandboxFailure = result is UtSandboxFailure
return implicitlyThrown || overflowFailure || assertionError || sandboxFailure
}
}