Prefer symbolic executions to fuzzed execution in minimization

Author: dtim

utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt

@@ -378,6 +378,11 @@ object UtSettings : AbstractSettings(
      */
     var disableSandbox by getBooleanProperty(false)
 
+    /**
+     * Exclude fuzzer-produced executions during minimization if there is an equivalent symbolic execution
+     */
+    var preferSymbolicExecutionsDuringMinimization by getBooleanProperty(true)
+
 }
 
 /**

utbot-framework-test/src/test/kotlin/org/utbot/framework/minimization/MinimizationGreedyEssentialTest.kt

@@ -68,4 +68,46 @@ class MinimizationGreedyEssentialTest {
         val minimizedExecutions = GreedyEssential.minimize(executions)
         assertEquals((1..size).toList(), minimizedExecutions.sorted())
     }
+
+    @Test
+    fun testWithSourcePriority() {
+        val executions = mapOf(
+            10 to listOf(1, 2, 3, 4, 5),
+            20 to listOf(2, 3, 4, 5, 6, 7),
+            21 to listOf(2, 3, 4, 5, 6, 7),
+            25 to listOf(1, 7, 2, 3, 5),
+            30 to listOf(8, 9),
+            50 to listOf(1, 8, 9),
+            60 to listOf(1, 9)
+        )
+
+        val sourcePriorities = mapOf(
+            10 to 0,
+            20 to 1,
+            21 to 0,
+            25 to 0,
+            30 to 1,
+            50 to 1,
+            60 to 0
+        )
+
+        val minimizedExecutions = GreedyEssential.minimize(executions, sourcePriorities)
+        assertEquals(listOf(21, 50), minimizedExecutions)
+    }
+
+    @Test
+    fun testWithoutSourcePriority() {
+        val executions = mapOf(
+            10 to listOf(1, 2, 3, 4, 5),
+            20 to listOf(2, 3, 4, 5, 6, 7),
+            21 to listOf(2, 3, 4, 5, 6, 7),
+            25 to listOf(1, 7, 2, 3, 5),
+            30 to listOf(8, 9),
+            50 to listOf(1, 8, 9),
+            60 to listOf(1, 9)
+        )
+
+        val minimizedExecutions = GreedyEssential.minimize(executions)
+        assertEquals(listOf(20, 50), minimizedExecutions)
+    }
 }

utbot-framework/src/main/kotlin/org/utbot/framework/minimization/GreedyEssential.kt

@@ -2,33 +2,51 @@ package org.utbot.framework.minimization
 
 import java.util.PriorityQueue
 
-private inline class ExecutionNumber(val number: Int)
+@JvmInline
+private value class ExecutionNumber(val number: Int)
 
-private inline class LineNumber(val number: Int)
+@JvmInline
+private value class LineNumber(val number: Int)
+
+@JvmInline
+private value class SourcePriority(val number: Int)
+
+private data class CoveredLinesWithSourcePriority(
+    val sourcePriority: SourcePriority,
+    val coveredLines: List<LineNumber>
+)
 
 /**
  * [Greedy essential algorithm](CONFLUENCE:Test+Minimization)
  */
 class GreedyEssential private constructor(
-    executionToCoveredLines: Map<ExecutionNumber, List<LineNumber>>
+    executionToCoveredLines: Map<ExecutionNumber, CoveredLinesWithSourcePriority>
 ) {
+    private val executionToSourcePriority: Map<ExecutionNumber, SourcePriority> =
+        executionToCoveredLines
+            .mapValues { it.value.sourcePriority }
+
     private val executionToUsefulLines: Map<ExecutionNumber, MutableSet<LineNumber>> =
         executionToCoveredLines
-            .mapValues { it.value.toMutableSet() }
+            .mapValues { it.value.coveredLines.toMutableSet() }
 
     private val lineToUnusedCoveringExecutions: Map<LineNumber, MutableSet<ExecutionNumber>> =
         executionToCoveredLines
-            .flatMap { (execution, lines) -> lines.map { it to execution } }
+            .flatMap { (execution, lines) -> lines.coveredLines.map { it to execution } }
             .groupBy({ it.first }) { it.second }
             .mapValues { it.value.toMutableSet() }
 
     private val executionByPriority =
-        PriorityQueue(compareByDescending<Pair<ExecutionNumber, Int>> { it.second }.thenBy { it.first.number })
+        PriorityQueue(
+            compareByDescending<Triple<ExecutionNumber, Int, SourcePriority>> { it.second }
+                    .thenBy { it.third.number }
+                    .thenBy { it.first.number }
+        )
             .apply {
                 addAll(
                     executionToCoveredLines
                         .keys
-                        .map { it to executionToUsefulLines[it]!!.size }
+                        .map { Triple(it, executionToUsefulLines[it]!!.size, executionToSourcePriority[it]!!) }
                 )
             }
 
@@ -69,7 +87,7 @@ class GreedyEssential private constructor(
     }
 
     private fun executionToPriority(execution: ExecutionNumber) =
-        execution to executionToUsefulLines[execution]!!.size
+        Triple(execution, executionToUsefulLines[execution]!!.size, executionToSourcePriority[execution]!!)
 
     private fun removeLineFromExecution(execution: ExecutionNumber, line: LineNumber) {
         executionByPriority.remove(executionToPriority(execution))
@@ -89,10 +107,15 @@ class GreedyEssential private constructor(
          *
          * @return retained execution ids.
          */
-        fun minimize(executions: Map<Int, List<Int>>): List<Int> {
+        fun minimize(executions: Map<Int, List<Int>>, sourcePriorities: Map<Int, Int> = emptyMap()): List<Int> {
             val convertedExecutions = executions
                 .entries
-                .associate { (execution, lines) -> ExecutionNumber(execution) to lines.map { LineNumber(it) } }
+                .associate { (execution, lines) ->
+                    ExecutionNumber(execution) to CoveredLinesWithSourcePriority(
+                        SourcePriority(sourcePriorities.getOrDefault(execution, 0)),
+                        lines.map { LineNumber(it) }
+                    )
+                }
 
             val prioritizer = GreedyEssential(convertedExecutions)
             val list = mutableListOf<ExecutionNumber>()

utbot-framework/src/main/kotlin/org/utbot/framework/minimization/Minimization.kt

@@ -13,11 +13,14 @@ import org.utbot.framework.plugin.api.UtExecutableCallModel
 import org.utbot.framework.plugin.api.UtExecution
 import org.utbot.framework.plugin.api.UtExecutionFailure
 import org.utbot.framework.plugin.api.UtExecutionResult
+import org.utbot.framework.plugin.api.UtFailedExecution
 import org.utbot.framework.plugin.api.UtModel
 import org.utbot.framework.plugin.api.UtNullModel
 import org.utbot.framework.plugin.api.UtPrimitiveModel
 import org.utbot.framework.plugin.api.UtStatementModel
+import org.utbot.framework.plugin.api.UtSymbolicExecution
 import org.utbot.framework.plugin.api.UtVoidModel
+import org.utbot.fuzzer.UtFuzzedExecution
 
 
 /**
@@ -52,8 +55,15 @@ fun minimizeExecutions(executions: List<UtExecution>): List<UtExecution> {
         executions.indices.filter { executions[it].coverage?.coveredInstructions?.isEmpty() ?: true }.toSet()
     // ^^^ here we add executions with empty or null coverage, because it happens only if a concrete execution failed,
     //     so we don't know the actual coverage for such executions
+
+    val sourcePriorities = if (UtSettings.preferSymbolicExecutionsDuringMinimization) {
+        executions.withIndex().associate { it.index to it.value.getSourcePriorityForMinimization() }
+    } else {
+        emptyMap()
+    }
+
     val mapping = buildMapping(executions.filterIndexed { idx, _ -> idx !in unknownCoverageExecutions })
-    val usedExecutionIndexes = (GreedyEssential.minimize(mapping) + unknownCoverageExecutions).toSet()
+    val usedExecutionIndexes = (GreedyEssential.minimize(mapping, sourcePriorities) + unknownCoverageExecutions).toSet()
 
     val usedMinimizedExecutions = executions.filterIndexed { idx, _ -> idx in usedExecutionIndexes }
 
@@ -64,6 +74,19 @@ fun minimizeExecutions(executions: List<UtExecution>): List<UtExecution> {
     }
 }
 
+/**
+ * Compute the priority of the given execution during minimization. Lower values correspond to higher priority.
+ *
+ * If [UtSettings.preferSymbolicExecutionsDuringMinimization] is true, minimizer will use these priorities
+ * to select symbolic executions instead of fuzzer-generated executions with the same coverage.
+ */
+private fun UtExecution.getSourcePriorityForMinimization(): Int = when (this) {
+    is UtSymbolicExecution -> 0
+    is UtFuzzedExecution -> 1
+    is UtFailedExecution -> 2
+    else -> 3
+}
+
 /**
  * Groups the [executions] by their `paths` on `first` [branchInstructionsNumber] `branch` instructions.
  *
@@ -150,7 +173,6 @@ private fun buildMapping(executions: List<UtExecution>): Map<Int, List<Int>> {
     val thrownExceptions = mutableMapOf<String, Long>()
     val mapping = mutableMapOf<Int, List<Int>>()
 
-
     executions.forEachIndexed { idx, execution ->
         execution.coverage?.let { coverage ->
             val instructionsWithoutExtra = coverage.coveredInstructions.map { it.id }