From 8e396f0db3226f7dafe08352c2140dd43995935d Mon Sep 17 00:00:00 2001 From: Nikita Stroganov Date: Mon, 11 Jul 2022 18:28:31 +0300 Subject: [PATCH 1/5] Add sarif results minimization (#490) --- .../main/kotlin/org/utbot/sarif/DataClasses.kt | 12 +++++++++++- .../main/kotlin/org/utbot/sarif/SarifReport.kt | 16 +++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/utbot-framework/src/main/kotlin/org/utbot/sarif/DataClasses.kt b/utbot-framework/src/main/kotlin/org/utbot/sarif/DataClasses.kt index 1a6aae7e8e..fec0268b55 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/sarif/DataClasses.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/sarif/DataClasses.kt @@ -99,7 +99,17 @@ data class SarifResult( val locations: List = listOf(), val relatedLocations: List = listOf(), val codeFlows: List = listOf() -) +) { + /** + * Returns the total number of locations in all [codeFlows]. + */ + fun totalCodeFlowLocations() = + codeFlows.sumBy { codeFlow -> + codeFlow.threadFlows.sumBy { threadFlow -> + threadFlow.locations.size + } + } +} /** * The severity of the result. "Error" for detected unchecked exceptions. diff --git a/utbot-framework/src/main/kotlin/org/utbot/sarif/SarifReport.kt b/utbot-framework/src/main/kotlin/org/utbot/sarif/SarifReport.kt index de414528a4..75d742ff15 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/sarif/SarifReport.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/sarif/SarifReport.kt @@ -90,11 +90,25 @@ class SarifReport( return Sarif.fromRun( SarifRun( SarifTool.fromRules(sarifRules.toList()), - sarifResults + minimizeResults(sarifResults) ) ) } + /** + * Minimizes detected errors and removes duplicates. + * Between two [SarifResult]s with the same `ruleId` and `locations` + * it chooses the one with the shorter length of the execution trace. + */ + private fun minimizeResults(sarifResults: List): List = + sarifResults.groupBy { sarifResult -> + Pair(sarifResult.ruleId, sarifResult.locations) + }.map { (_, sarifResultsGroup) -> + sarifResultsGroup.minByOrNull { sarifResult -> + sarifResult.totalCodeFlowLocations() + }!! + } + private fun processUncheckedException( method: UtMethod<*>, utExecution: UtExecution, From d8f34a644569f48d7ee210380c48e3e05b1fcf4b Mon Sep 17 00:00:00 2001 From: Nikita Stroganov Date: Tue, 12 Jul 2022 11:27:22 +0300 Subject: [PATCH 2/5] Add unit tests --- .../kotlin/org/utbot/sarif/SarifReportTest.kt | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/utbot-framework/src/test/kotlin/org/utbot/sarif/SarifReportTest.kt b/utbot-framework/src/test/kotlin/org/utbot/sarif/SarifReportTest.kt index d3430d5218..95333b349e 100644 --- a/utbot-framework/src/test/kotlin/org/utbot/sarif/SarifReportTest.kt +++ b/utbot-framework/src/test/kotlin/org/utbot/sarif/SarifReportTest.kt @@ -186,6 +186,59 @@ class SarifReportTest { assert(codeFlowPhysicalLocations[0].region.startColumn == 5) } + @Test + fun testMinimizationRemovesDuplicates() { + mockUtMethodNames() + + val mockUtExecution = Mockito.mock(UtExecution::class.java, Mockito.RETURNS_DEEP_STUBS) + Mockito.`when`(mockUtExecution.result).thenReturn(UtImplicitlyThrownException(NullPointerException(), false)) + + val testCases = listOf( + UtTestCase(mockUtMethod, listOf(mockUtExecution)), + UtTestCase(mockUtMethod, listOf(mockUtExecution)) // duplicate + ) + + val report = SarifReport( + testCases = testCases, + generatedTestsCode = "", + sourceFindingEmpty + ).createReport().toSarif() + + assert(report.runs.first().results.size == 1) // no duplicates + } + + @Test + fun testMinimizationChoosesShortestCodeFlow() { + mockUtMethodNames() + + val mockNPE1 = Mockito.mock(NullPointerException::class.java) + val mockNPE2 = Mockito.mock(NullPointerException::class.java) + + val mockUtExecution1 = Mockito.mock(UtExecution::class.java, Mockito.RETURNS_DEEP_STUBS) + val mockUtExecution2 = Mockito.mock(UtExecution::class.java, Mockito.RETURNS_DEEP_STUBS) + Mockito.`when`(mockUtExecution1.result).thenReturn(UtImplicitlyThrownException(mockNPE1, false)) + Mockito.`when`(mockUtExecution2.result).thenReturn(UtImplicitlyThrownException(mockNPE2, false)) + + val stackTraceElement1 = StackTraceElement("Main", "main", "Main.java", 3) + val stackTraceElement2 = StackTraceElement("Main", "main", "Main.java", 7) + Mockito.`when`(mockNPE1.stackTrace).thenReturn(arrayOf(stackTraceElement1)) + Mockito.`when`(mockNPE2.stackTrace).thenReturn(arrayOf(stackTraceElement1, stackTraceElement2)) + + val testCases = listOf( + UtTestCase(mockUtMethod, listOf(mockUtExecution1)), + UtTestCase(mockUtMethod, listOf(mockUtExecution2)) // duplicate with longer stack trace + ) + + val report = SarifReport( + testCases = testCases, + generatedTestsCode = "", + sourceFindingEmpty + ).createReport().toSarif() + + assert(report.runs.first().results.size == 1) // no duplicates + assert(report.runs.first().results.first().totalCodeFlowLocations() == 1) // with shorter stack trace + } + // internal private val mockUtMethod = Mockito.mock(UtMethod::class.java, Mockito.RETURNS_DEEP_STUBS) From 1a08be97276e4b5a81b7aff3b4483f2ca07b8706 Mon Sep 17 00:00:00 2001 From: Nikita Stroganov Date: Tue, 12 Jul 2022 11:29:22 +0300 Subject: [PATCH 3/5] Fix typos in comments --- .../src/test/kotlin/org/utbot/sarif/SarifReportTest.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utbot-framework/src/test/kotlin/org/utbot/sarif/SarifReportTest.kt b/utbot-framework/src/test/kotlin/org/utbot/sarif/SarifReportTest.kt index 95333b349e..4aef2ddff2 100644 --- a/utbot-framework/src/test/kotlin/org/utbot/sarif/SarifReportTest.kt +++ b/utbot-framework/src/test/kotlin/org/utbot/sarif/SarifReportTest.kt @@ -226,7 +226,7 @@ class SarifReportTest { val testCases = listOf( UtTestCase(mockUtMethod, listOf(mockUtExecution1)), - UtTestCase(mockUtMethod, listOf(mockUtExecution2)) // duplicate with longer stack trace + UtTestCase(mockUtMethod, listOf(mockUtExecution2)) // duplicate with a longer stack trace ) val report = SarifReport( @@ -236,7 +236,7 @@ class SarifReportTest { ).createReport().toSarif() assert(report.runs.first().results.size == 1) // no duplicates - assert(report.runs.first().results.first().totalCodeFlowLocations() == 1) // with shorter stack trace + assert(report.runs.first().results.first().totalCodeFlowLocations() == 1) // with a shorter stack trace } // internal From 587cf116ecff55bc4a91769468132cf1e63a4bce Mon Sep 17 00:00:00 2001 From: Nikita Stroganov Date: Tue, 12 Jul 2022 17:11:27 +0300 Subject: [PATCH 4/5] Fix after review --- .../main/kotlin/org/utbot/sarif/SarifReport.kt | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/utbot-framework/src/main/kotlin/org/utbot/sarif/SarifReport.kt b/utbot-framework/src/main/kotlin/org/utbot/sarif/SarifReport.kt index 75d742ff15..aef6b98f0a 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/sarif/SarifReport.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/sarif/SarifReport.kt @@ -100,14 +100,17 @@ class SarifReport( * Between two [SarifResult]s with the same `ruleId` and `locations` * it chooses the one with the shorter length of the execution trace. */ - private fun minimizeResults(sarifResults: List): List = - sarifResults.groupBy { sarifResult -> + private fun minimizeResults(sarifResults: List): List { + val groupedResults = sarifResults.groupBy { sarifResult -> Pair(sarifResult.ruleId, sarifResult.locations) - }.map { (_, sarifResultsGroup) -> - sarifResultsGroup.minByOrNull { sarifResult -> - sarifResult.totalCodeFlowLocations() - }!! } + val minimizedResults = groupedResults.map { (_, sarifResultsGroup) -> + sarifResultsGroup.minByOrNull { sarifResult -> + sarifResult.totalCodeFlowLocations() + }!! + } + return minimizedResults + } private fun processUncheckedException( method: UtMethod<*>, From c3c2b03674609b09a413f664bbdd5b34694e710a Mon Sep 17 00:00:00 2001 From: Nikita Stroganov Date: Wed, 13 Jul 2022 11:00:48 +0300 Subject: [PATCH 5/5] Fix after review --- .../kotlin/org/utbot/sarif/SarifReport.kt | 17 ++++++ .../kotlin/org/utbot/sarif/SarifReportTest.kt | 61 ++++++++++++++++++- 2 files changed, 76 insertions(+), 2 deletions(-) diff --git a/utbot-framework/src/main/kotlin/org/utbot/sarif/SarifReport.kt b/utbot-framework/src/main/kotlin/org/utbot/sarif/SarifReport.kt index aef6b98f0a..1e35722ba5 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/sarif/SarifReport.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/sarif/SarifReport.kt @@ -97,8 +97,25 @@ class SarifReport( /** * Minimizes detected errors and removes duplicates. + * * Between two [SarifResult]s with the same `ruleId` and `locations` * it chooses the one with the shorter length of the execution trace. + * + * __Example:__ + * + * The SARIF report for the code below contains only one unchecked exception in `methodB`. + * But without minimization, the report will contain two results: for `methodA` and for `methodB`. + * + * ``` + * class Example { + * int methodA(int a) { + * return methodB(a); + * } + * int methodB(int b) { + * return 1 / b; + * } + * } + * ``` */ private fun minimizeResults(sarifResults: List): List { val groupedResults = sarifResults.groupBy { sarifResult -> diff --git a/utbot-framework/src/test/kotlin/org/utbot/sarif/SarifReportTest.kt b/utbot-framework/src/test/kotlin/org/utbot/sarif/SarifReportTest.kt index 4aef2ddff2..7c200508d5 100644 --- a/utbot-framework/src/test/kotlin/org/utbot/sarif/SarifReportTest.kt +++ b/utbot-framework/src/test/kotlin/org/utbot/sarif/SarifReportTest.kt @@ -201,12 +201,66 @@ class SarifReportTest { val report = SarifReport( testCases = testCases, generatedTestsCode = "", - sourceFindingEmpty + sourceFindingMain ).createReport().toSarif() assert(report.runs.first().results.size == 1) // no duplicates } + @Test + fun testMinimizationDoesNotRemoveResultsWithDifferentRuleId() { + mockUtMethodNames() + + val mockUtExecution1 = Mockito.mock(UtExecution::class.java, Mockito.RETURNS_DEEP_STUBS) + val mockUtExecution2 = Mockito.mock(UtExecution::class.java, Mockito.RETURNS_DEEP_STUBS) + + // different ruleId's + Mockito.`when`(mockUtExecution1.result).thenReturn(UtImplicitlyThrownException(NullPointerException(), false)) + Mockito.`when`(mockUtExecution2.result).thenReturn(UtImplicitlyThrownException(ArithmeticException(), false)) + + val testCases = listOf( + UtTestCase(mockUtMethod, listOf(mockUtExecution1)), + UtTestCase(mockUtMethod, listOf(mockUtExecution2)) // not a duplicate + ) + + val report = SarifReport( + testCases = testCases, + generatedTestsCode = "", + sourceFindingMain + ).createReport().toSarif() + + assert(report.runs.first().results.size == 2) // no results have been removed + } + + @Test + fun testMinimizationDoesNotRemoveResultsWithDifferentLocations() { + mockUtMethodNames() + + val mockUtExecution1 = Mockito.mock(UtExecution::class.java, Mockito.RETURNS_DEEP_STUBS) + val mockUtExecution2 = Mockito.mock(UtExecution::class.java, Mockito.RETURNS_DEEP_STUBS) + + // the same ruleId's + Mockito.`when`(mockUtExecution1.result).thenReturn(UtImplicitlyThrownException(NullPointerException(), false)) + Mockito.`when`(mockUtExecution2.result).thenReturn(UtImplicitlyThrownException(NullPointerException(), false)) + + // different locations + Mockito.`when`(mockUtExecution1.path.lastOrNull()?.stmt?.javaSourceStartLineNumber).thenReturn(11) + Mockito.`when`(mockUtExecution2.path.lastOrNull()?.stmt?.javaSourceStartLineNumber).thenReturn(22) + + val testCases = listOf( + UtTestCase(mockUtMethod, listOf(mockUtExecution1)), + UtTestCase(mockUtMethod, listOf(mockUtExecution2)) // not a duplicate + ) + + val report = SarifReport( + testCases = testCases, + generatedTestsCode = "", + sourceFindingMain + ).createReport().toSarif() + + assert(report.runs.first().results.size == 2) // no results have been removed + } + @Test fun testMinimizationChoosesShortestCodeFlow() { mockUtMethodNames() @@ -216,9 +270,12 @@ class SarifReportTest { val mockUtExecution1 = Mockito.mock(UtExecution::class.java, Mockito.RETURNS_DEEP_STUBS) val mockUtExecution2 = Mockito.mock(UtExecution::class.java, Mockito.RETURNS_DEEP_STUBS) + + // the same ruleId's Mockito.`when`(mockUtExecution1.result).thenReturn(UtImplicitlyThrownException(mockNPE1, false)) Mockito.`when`(mockUtExecution2.result).thenReturn(UtImplicitlyThrownException(mockNPE2, false)) + // but different stack traces val stackTraceElement1 = StackTraceElement("Main", "main", "Main.java", 3) val stackTraceElement2 = StackTraceElement("Main", "main", "Main.java", 7) Mockito.`when`(mockNPE1.stackTrace).thenReturn(arrayOf(stackTraceElement1)) @@ -232,7 +289,7 @@ class SarifReportTest { val report = SarifReport( testCases = testCases, generatedTestsCode = "", - sourceFindingEmpty + sourceFindingMain ).createReport().toSarif() assert(report.runs.first().results.size == 1) // no duplicates