Skip to content

Commit 1042fb7

Browse files
authored
Add timeout tests to SARIF report (#1496)
* Add timeout tests to SARIF report * Add unit tests
1 parent 7ba93d6 commit 1042fb7

File tree

2 files changed

+92
-4
lines changed

2 files changed

+92
-4
lines changed

utbot-framework-test/src/test/kotlin/org/utbot/sarif/SarifReportTest.kt

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,49 @@ class SarifReportTest {
155155
assert(result.message.text.contains("AccessControlException"))
156156
}
157157

158+
@Test
159+
fun testProcessTimeoutException() {
160+
mockUtMethodNames()
161+
162+
val timeoutMessage = "Timeout 1000 elapsed"
163+
val timeoutException = TimeoutException(timeoutMessage)
164+
Mockito.`when`(mockUtExecution.result).thenReturn(UtTimeoutException(timeoutException))
165+
166+
defaultMockCoverage(mockUtExecution)
167+
168+
val report = sarifReportMain.createReport()
169+
val result = report.runs.first().results.first()
170+
assert(result.message.text.contains(timeoutMessage))
171+
}
172+
173+
@Test
174+
fun testConvertCoverageToStackTrace() {
175+
mockUtMethodNames()
176+
177+
val timeoutException = TimeoutException("Timeout 1000 elapsed")
178+
Mockito.`when`(mockUtExecution.result).thenReturn(UtTimeoutException(timeoutException))
179+
180+
val classMainPath = "com/abc/Main"
181+
val classUtilPath = "com/cba/Util"
182+
Mockito.`when`(mockUtExecution.coverage?.coveredInstructions).thenReturn(listOf(
183+
Instruction(classMainPath, "main(l)l", 3, 1),
184+
Instruction(classMainPath, "main(l)l", 4, 2),
185+
Instruction(classMainPath, "main(l)l", 4, 3), // last for main
186+
Instruction(classUtilPath, "util(ll)l", 6, 4),
187+
Instruction(classUtilPath, "util(ll)l", 7, 5), // last for util
188+
))
189+
190+
val report = sarifReportMain.createReport()
191+
val result = report.runs.first().results.first()
192+
val stackTrace = result.codeFlows.first().threadFlows.first().locations
193+
194+
assert(stackTrace[0].location.physicalLocation.artifactLocation.uri == "$classMainPath.java")
195+
assert(stackTrace[0].location.physicalLocation.region.startLine == 4)
196+
197+
assert(stackTrace[1].location.physicalLocation.artifactLocation.uri == "$classUtilPath.java")
198+
assert(stackTrace[1].location.physicalLocation.region.startLine == 7)
199+
}
200+
158201
@Test
159202
fun testCodeFlowsStartsWithMethodCall() {
160203
mockUtMethodNames()

utbot-framework/src/main/kotlin/org/utbot/sarif/SarifReport.kt

Lines changed: 49 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -153,12 +153,17 @@ class SarifReport(
153153
val methodArguments = utExecution.stateBefore.parameters
154154
.joinToString(prefix = "", separator = ", ", postfix = "") { it.preview() }
155155

156+
val errorMessage = if (executionFailure is UtTimeoutException)
157+
"Unexpected behavior: ${executionFailure.exception.message}"
158+
else
159+
"Unexpected exception: ${executionFailure.exception}"
160+
156161
val sarifResult = SarifResult(
157162
ruleId,
158163
Level.Error,
159164
Message(
160165
text = """
161-
Unexpected exception: ${executionFailure.exception}.
166+
$errorMessage.
162167
Test case: `$methodName($methodArguments)`
163168
[Generated test for this case]($relatedLocationId)
164169
""".trimIndent()
@@ -229,8 +234,10 @@ class SarifReport(
229234
val stackTraceResolved = filterStackTrace(method, utExecution, executionFailure)
230235
.mapNotNull { findStackTraceElementLocation(it) }
231236
.toMutableList()
232-
if (stackTraceResolved.isEmpty())
233-
return listOf() // empty stack trace is not shown
237+
238+
if (stackTraceResolved.isEmpty()) {
239+
stackTraceResolved += stackTraceFromCoverage(utExecution) // fallback logic
240+
}
234241

235242
// prepending stack trace by `method` call in generated tests
236243
val methodCallLocation: SarifPhysicalLocation? =
@@ -279,6 +286,10 @@ class SarifReport(
279286
if (lastMethodCallIndex != -1) {
280287
// taking all elements before the last `method` call
281288
stackTrace = stackTrace.take(lastMethodCallIndex + 1)
289+
} else { // no `method` call in the stack trace
290+
if (executionFailure.exception !is StackOverflowError) {
291+
stackTrace = listOf() // (likely) the stack trace contains only our internal calls
292+
}
282293
}
283294

284295
if (executionFailure.exception is StackOverflowError) {
@@ -292,6 +303,39 @@ class SarifReport(
292303
return stackTraceFiltered
293304
}
294305

306+
/**
307+
* Constructs the stack trace from the list of covered instructions.
308+
*/
309+
private fun stackTraceFromCoverage(utExecution: UtExecution): List<SarifFlowLocationWrapper> {
310+
val coveredInstructions = utExecution.coverage?.coveredInstructions
311+
?: return listOf()
312+
313+
val executionTrace = coveredInstructions.groupBy { instruction ->
314+
instruction.className to instruction.methodSignature // group by method
315+
}.map { (_, instructionsForOneMethod) ->
316+
instructionsForOneMethod.last() // we need only last to construct the stack trace
317+
}
318+
319+
val sarifExecutionTrace = executionTrace.map { instruction ->
320+
val classFqn = instruction.className.replace('/', '.')
321+
val methodName = instruction.methodSignature.substringBefore('(')
322+
val lineNumber = instruction.lineNumber
323+
324+
val sourceFilePath = sourceFinding.getSourceRelativePath(classFqn)
325+
val sourceFileName = sourceFilePath.substringAfterLast('/')
326+
327+
SarifFlowLocationWrapper(SarifFlowLocation(
328+
message = Message("$classFqn.$methodName($sourceFileName:$lineNumber)"),
329+
physicalLocation = SarifPhysicalLocation(
330+
SarifArtifact(uri = sourceFilePath),
331+
SarifRegion(startLine = lineNumber)
332+
)
333+
))
334+
}
335+
336+
return sarifExecutionTrace.reversed() // to stack trace
337+
}
338+
295339
private fun findStackTraceElementLocation(stackTraceElement: StackTraceElement): SarifFlowLocationWrapper? {
296340
val lineNumber = stackTraceElement.lineNumber
297341
if (lineNumber < 1)
@@ -429,6 +473,7 @@ class SarifReport(
429473
val overflowFailure = result is UtOverflowFailure && UtSettings.treatOverflowAsError
430474
val assertionError = result is UtExplicitlyThrownException && result.exception is AssertionError
431475
val sandboxFailure = result is UtSandboxFailure
432-
return implicitlyThrown || overflowFailure || assertionError || sandboxFailure
476+
val timeoutException = result is UtTimeoutException
477+
return implicitlyThrown || overflowFailure || assertionError || sandboxFailure || timeoutException
433478
}
434479
}

0 commit comments

Comments
 (0)