From 613e0f2ca4d7a505f378dc62fc8dd6a4685d9a03 Mon Sep 17 00:00:00 2001 From: Nikita Stroganov Date: Thu, 8 Dec 2022 15:25:31 +0300 Subject: [PATCH 1/2] Add timeout tests to SARIF report --- .../kotlin/org/utbot/sarif/SarifReport.kt | 53 +++++++++++++++++-- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/utbot-framework/src/main/kotlin/org/utbot/sarif/SarifReport.kt b/utbot-framework/src/main/kotlin/org/utbot/sarif/SarifReport.kt index 8f6e4438a5..94de6244cc 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/sarif/SarifReport.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/sarif/SarifReport.kt @@ -153,12 +153,17 @@ class SarifReport( val methodArguments = utExecution.stateBefore.parameters .joinToString(prefix = "", separator = ", ", postfix = "") { it.preview() } + val errorMessage = if (executionFailure is UtTimeoutException) + "Unexpected behavior: ${executionFailure.exception.message}" + else + "Unexpected exception: ${executionFailure.exception}" + val sarifResult = SarifResult( ruleId, Level.Error, Message( text = """ - Unexpected exception: ${executionFailure.exception}. + $errorMessage. Test case: `$methodName($methodArguments)` [Generated test for this case]($relatedLocationId) """.trimIndent() @@ -229,8 +234,10 @@ class SarifReport( val stackTraceResolved = filterStackTrace(method, utExecution, executionFailure) .mapNotNull { findStackTraceElementLocation(it) } .toMutableList() - if (stackTraceResolved.isEmpty()) - return listOf() // empty stack trace is not shown + + if (stackTraceResolved.isEmpty()) { + stackTraceResolved += stackTraceFromCoverage(utExecution) // fallback logic + } // prepending stack trace by `method` call in generated tests val methodCallLocation: SarifPhysicalLocation? = @@ -279,6 +286,10 @@ class SarifReport( if (lastMethodCallIndex != -1) { // taking all elements before the last `method` call stackTrace = stackTrace.take(lastMethodCallIndex + 1) + } else { // no `method` call in the stack trace + if (executionFailure.exception !is StackOverflowError) { + stackTrace = listOf() // (likely) the stack trace contains only our internal calls + } } if (executionFailure.exception is StackOverflowError) { @@ -292,6 +303,39 @@ class SarifReport( return stackTraceFiltered } + /** + * Constructs the stack trace from the list of covered instructions. + */ + private fun stackTraceFromCoverage(utExecution: UtExecution): List { + val coveredInstructions = utExecution.coverage?.coveredInstructions + ?: return listOf() + + val executionTrace = coveredInstructions.groupBy { instruction -> + instruction.className to instruction.methodSignature // group by method + }.map { (_, instructionsForOneMethod) -> + instructionsForOneMethod.last() // we need only last to construct the stack trace + } + + val sarifExecutionTrace = executionTrace.map { instruction -> + val classFqn = instruction.className.replace('/', '.') + val methodName = instruction.methodSignature.substringBefore('(') + val lineNumber = instruction.lineNumber + + val sourceFilePath = sourceFinding.getSourceRelativePath(classFqn) + val sourceFileName = sourceFilePath.substringAfterLast('/') + + SarifFlowLocationWrapper(SarifFlowLocation( + message = Message("$classFqn.$methodName($sourceFileName:$lineNumber)"), + physicalLocation = SarifPhysicalLocation( + SarifArtifact(uri = sourceFilePath), + SarifRegion(startLine = lineNumber) + ) + )) + } + + return sarifExecutionTrace.reversed() // to stack trace + } + private fun findStackTraceElementLocation(stackTraceElement: StackTraceElement): SarifFlowLocationWrapper? { val lineNumber = stackTraceElement.lineNumber if (lineNumber < 1) @@ -429,6 +473,7 @@ class SarifReport( val overflowFailure = result is UtOverflowFailure && UtSettings.treatOverflowAsError val assertionError = result is UtExplicitlyThrownException && result.exception is AssertionError val sandboxFailure = result is UtSandboxFailure - return implicitlyThrown || overflowFailure || assertionError || sandboxFailure + val timeoutException = result is UtTimeoutException + return implicitlyThrown || overflowFailure || assertionError || sandboxFailure || timeoutException } } \ No newline at end of file From c86278e523e1238143e72298e3e65fd3c5d04565 Mon Sep 17 00:00:00 2001 From: Nikita Stroganov Date: Thu, 8 Dec 2022 19:08:59 +0300 Subject: [PATCH 2/2] Add unit tests --- .../kotlin/org/utbot/sarif/SarifReportTest.kt | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/utbot-framework-test/src/test/kotlin/org/utbot/sarif/SarifReportTest.kt b/utbot-framework-test/src/test/kotlin/org/utbot/sarif/SarifReportTest.kt index 7473cdff94..ed9f415b92 100644 --- a/utbot-framework-test/src/test/kotlin/org/utbot/sarif/SarifReportTest.kt +++ b/utbot-framework-test/src/test/kotlin/org/utbot/sarif/SarifReportTest.kt @@ -155,6 +155,49 @@ class SarifReportTest { assert(result.message.text.contains("AccessControlException")) } + @Test + fun testProcessTimeoutException() { + mockUtMethodNames() + + val timeoutMessage = "Timeout 1000 elapsed" + val timeoutException = TimeoutException(timeoutMessage) + Mockito.`when`(mockUtExecution.result).thenReturn(UtTimeoutException(timeoutException)) + + defaultMockCoverage(mockUtExecution) + + val report = sarifReportMain.createReport() + val result = report.runs.first().results.first() + assert(result.message.text.contains(timeoutMessage)) + } + + @Test + fun testConvertCoverageToStackTrace() { + mockUtMethodNames() + + val timeoutException = TimeoutException("Timeout 1000 elapsed") + Mockito.`when`(mockUtExecution.result).thenReturn(UtTimeoutException(timeoutException)) + + val classMainPath = "com/abc/Main" + val classUtilPath = "com/cba/Util" + Mockito.`when`(mockUtExecution.coverage?.coveredInstructions).thenReturn(listOf( + Instruction(classMainPath, "main(l)l", 3, 1), + Instruction(classMainPath, "main(l)l", 4, 2), + Instruction(classMainPath, "main(l)l", 4, 3), // last for main + Instruction(classUtilPath, "util(ll)l", 6, 4), + Instruction(classUtilPath, "util(ll)l", 7, 5), // last for util + )) + + val report = sarifReportMain.createReport() + val result = report.runs.first().results.first() + val stackTrace = result.codeFlows.first().threadFlows.first().locations + + assert(stackTrace[0].location.physicalLocation.artifactLocation.uri == "$classMainPath.java") + assert(stackTrace[0].location.physicalLocation.region.startLine == 4) + + assert(stackTrace[1].location.physicalLocation.artifactLocation.uri == "$classUtilPath.java") + assert(stackTrace[1].location.physicalLocation.region.startLine == 7) + } + @Test fun testCodeFlowsStartsWithMethodCall() { mockUtMethodNames()