diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt index 93797852e5..7635273e0a 100644 --- a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt +++ b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt @@ -378,6 +378,11 @@ object UtSettings : AbstractSettings( */ var disableSandbox by getBooleanProperty(false) + /** + * Exclude fuzzer-produced executions during minimization if there is an equivalent symbolic execution + */ + var preferSymbolicExecutionsDuringMinimization by getBooleanProperty(true) + } /** diff --git a/utbot-framework-test/src/test/kotlin/org/utbot/framework/minimization/MinimizationGreedyEssentialTest.kt b/utbot-framework-test/src/test/kotlin/org/utbot/framework/minimization/MinimizationGreedyEssentialTest.kt index 993f23059a..94104a034c 100644 --- a/utbot-framework-test/src/test/kotlin/org/utbot/framework/minimization/MinimizationGreedyEssentialTest.kt +++ b/utbot-framework-test/src/test/kotlin/org/utbot/framework/minimization/MinimizationGreedyEssentialTest.kt @@ -68,4 +68,46 @@ class MinimizationGreedyEssentialTest { val minimizedExecutions = GreedyEssential.minimize(executions) assertEquals((1..size).toList(), minimizedExecutions.sorted()) } + + @Test + fun testWithSourcePriority() { + val executions = mapOf( + 10 to listOf(1, 2, 3, 4, 5), + 20 to listOf(2, 3, 4, 5, 6, 7), + 21 to listOf(2, 3, 4, 5, 6, 7), + 25 to listOf(1, 7, 2, 3, 5), + 30 to listOf(8, 9), + 50 to listOf(1, 8, 9), + 60 to listOf(1, 9) + ) + + val sourcePriorities = mapOf( + 10 to 0, + 20 to 1, + 21 to 0, + 25 to 0, + 30 to 1, + 50 to 1, + 60 to 0 + ) + + val minimizedExecutions = GreedyEssential.minimize(executions, sourcePriorities) + assertEquals(listOf(21, 50), minimizedExecutions) + } + + @Test + fun testWithoutSourcePriority() { + val executions = mapOf( + 10 to listOf(1, 2, 3, 4, 5), + 20 to listOf(2, 3, 4, 5, 6, 7), + 21 to listOf(2, 3, 4, 5, 6, 7), + 25 to listOf(1, 7, 2, 3, 5), + 30 to listOf(8, 9), + 50 to listOf(1, 8, 9), + 60 to listOf(1, 9) + ) + + val minimizedExecutions = GreedyEssential.minimize(executions) + assertEquals(listOf(20, 50), minimizedExecutions) + } } \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/minimization/GreedyEssential.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/minimization/GreedyEssential.kt index bbbcafd17f..2ce917b536 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/framework/minimization/GreedyEssential.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/framework/minimization/GreedyEssential.kt @@ -2,33 +2,58 @@ package org.utbot.framework.minimization import java.util.PriorityQueue -private inline class ExecutionNumber(val number: Int) +@JvmInline +private value class ExecutionNumber(val number: Int) -private inline class LineNumber(val number: Int) +@JvmInline +private value class LineNumber(val number: Int) + +/** + * Execution source priority (symbolic executions are considered more important than fuzzed executions). + * @see [UtSettings.preferSymbolicExecutionsDuringMinimization] + */ +@JvmInline +private value class SourcePriority(val number: Int) + +/** + * A wrapper that combines covered lines with the execution source priority + */ +private data class ExecutionCoverageInfo( + val sourcePriority: SourcePriority, + val coveredLines: List +) /** * [Greedy essential algorithm](CONFLUENCE:Test+Minimization) */ class GreedyEssential private constructor( - executionToCoveredLines: Map> + executionToCoveredLines: Map ) { + private val executionToSourcePriority: Map = + executionToCoveredLines + .mapValues { it.value.sourcePriority } + private val executionToUsefulLines: Map> = executionToCoveredLines - .mapValues { it.value.toMutableSet() } + .mapValues { it.value.coveredLines.toMutableSet() } private val lineToUnusedCoveringExecutions: Map> = executionToCoveredLines - .flatMap { (execution, lines) -> lines.map { it to execution } } + .flatMap { (execution, lines) -> lines.coveredLines.map { it to execution } } .groupBy({ it.first }) { it.second } .mapValues { it.value.toMutableSet() } private val executionByPriority = - PriorityQueue(compareByDescending> { it.second }.thenBy { it.first.number }) + PriorityQueue( + compareByDescending> { it.second } + .thenBy { it.third.number } + .thenBy { it.first.number } + ) .apply { addAll( executionToCoveredLines .keys - .map { it to executionToUsefulLines[it]!!.size } + .map { Triple(it, executionToUsefulLines[it]!!.size, executionToSourcePriority[it]!!) } ) } @@ -69,7 +94,7 @@ class GreedyEssential private constructor( } private fun executionToPriority(execution: ExecutionNumber) = - execution to executionToUsefulLines[execution]!!.size + Triple(execution, executionToUsefulLines[execution]!!.size, executionToSourcePriority[execution]!!) private fun removeLineFromExecution(execution: ExecutionNumber, line: LineNumber) { executionByPriority.remove(executionToPriority(execution)) @@ -87,12 +112,21 @@ class GreedyEssential private constructor( * Minimizes the given [executions] assuming the map represents mapping from execution id to covered * instruction ids. * + * @param executions the mapping of execution ids to lists of lines covered by the execution. + * @param sourcePriorities execution priorities: lower values correspond to more important executions + * that should be kept everything else being equal. + * * @return retained execution ids. */ - fun minimize(executions: Map>): List { + fun minimize(executions: Map>, sourcePriorities: Map = emptyMap()): List { val convertedExecutions = executions .entries - .associate { (execution, lines) -> ExecutionNumber(execution) to lines.map { LineNumber(it) } } + .associate { (execution, lines) -> + ExecutionNumber(execution) to ExecutionCoverageInfo( + SourcePriority(sourcePriorities.getOrDefault(execution, 0)), + lines.map { LineNumber(it) } + ) + } val prioritizer = GreedyEssential(convertedExecutions) val list = mutableListOf() diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/minimization/Minimization.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/minimization/Minimization.kt index 380ab74d2c..0bb63a74b5 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/framework/minimization/Minimization.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/framework/minimization/Minimization.kt @@ -13,11 +13,14 @@ import org.utbot.framework.plugin.api.UtExecutableCallModel import org.utbot.framework.plugin.api.UtExecution import org.utbot.framework.plugin.api.UtExecutionFailure import org.utbot.framework.plugin.api.UtExecutionResult +import org.utbot.framework.plugin.api.UtFailedExecution import org.utbot.framework.plugin.api.UtModel import org.utbot.framework.plugin.api.UtNullModel import org.utbot.framework.plugin.api.UtPrimitiveModel import org.utbot.framework.plugin.api.UtStatementModel +import org.utbot.framework.plugin.api.UtSymbolicExecution import org.utbot.framework.plugin.api.UtVoidModel +import org.utbot.fuzzer.UtFuzzedExecution /** @@ -52,8 +55,17 @@ fun minimizeExecutions(executions: List): List { executions.indices.filter { executions[it].coverage?.coveredInstructions?.isEmpty() ?: true }.toSet() // ^^^ here we add executions with empty or null coverage, because it happens only if a concrete execution failed, // so we don't know the actual coverage for such executions + + // If the minimizer is configured to prefer symbolic executions, compute the priority for each execution, + // otherwise use an empty priority map to mark that all executions have the same priority + val sourcePriorities = if (UtSettings.preferSymbolicExecutionsDuringMinimization) { + executions.withIndex().associate { it.index to it.value.getSourcePriorityForMinimization() } + } else { + emptyMap() + } + val mapping = buildMapping(executions.filterIndexed { idx, _ -> idx !in unknownCoverageExecutions }) - val usedExecutionIndexes = (GreedyEssential.minimize(mapping) + unknownCoverageExecutions).toSet() + val usedExecutionIndexes = (GreedyEssential.minimize(mapping, sourcePriorities) + unknownCoverageExecutions).toSet() val usedMinimizedExecutions = executions.filterIndexed { idx, _ -> idx in usedExecutionIndexes } @@ -64,6 +76,18 @@ fun minimizeExecutions(executions: List): List { } } +/** + * Compute the priority of the given execution during minimization. Lower values correspond to higher priority. + * + * If [UtSettings.preferSymbolicExecutionsDuringMinimization] is true, minimizer will use these priorities + * to select symbolic executions instead of fuzzer-generated executions with the same coverage. + */ +private fun UtExecution.getSourcePriorityForMinimization(): Int = when (this) { + is UtSymbolicExecution -> 0 + is UtFuzzedExecution -> 1 + else -> 2 +} + /** * Groups the [executions] by their `paths` on `first` [branchInstructionsNumber] `branch` instructions. * @@ -150,7 +174,6 @@ private fun buildMapping(executions: List): Map> { val thrownExceptions = mutableMapOf() val mapping = mutableMapOf>() - executions.forEachIndexed { idx, execution -> execution.coverage?.let { coverage -> val instructionsWithoutExtra = coverage.coveredInstructions.map { it.id }