Fuzzer fails to generate an empty list for type list[Any]

Author: Markoutte

utbot-fuzzers/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt

@@ -9,8 +9,8 @@ import org.utbot.fuzzer.*
 import org.utbot.fuzzing.providers.*
 import org.utbot.fuzzing.utils.Trie
 import java.lang.reflect.*
+import java.util.concurrent.CancellationException
 import java.util.concurrent.TimeUnit
-import kotlin.system.measureNanoTime
 import kotlin.random.Random
 
 private val logger = KotlinLogging.logger {}
@@ -105,23 +105,31 @@ suspend fun runJavaFuzzing(
     val tracer = Trie(Instruction::id)
     val descriptionWithOptionalThisInstance = FuzzedDescription(createFuzzedMethodDescription(thisInstance), tracer, typeCache, random)
     val descriptionWithOnlyParameters = FuzzedDescription(createFuzzedMethodDescription(null), tracer, typeCache, random)
+    val start = System.nanoTime()
     try {
         logger.info { "Starting fuzzing for method: $methodUnderTest" }
         logger.info { "\tuse thisInstance = ${thisInstance != null}" }
         logger.info { "\tparameters = $parameters" }
         var totalExecutionCalled = 0
-        val totalFuzzingTime = measureNanoTime {
-            runFuzzing(ValueProvider.of(providers), descriptionWithOptionalThisInstance, random) { _, t ->
-                totalExecutionCalled++
-                if (thisInstance == null) {
-                    exec(null, descriptionWithOnlyParameters, t)
-                } else {
-                    exec(t.first(), descriptionWithOnlyParameters, t.drop(1))
-                }
+        runFuzzing(
+            provider = ValueProvider.of(providers),
+            description = descriptionWithOptionalThisInstance, random,
+            configuration = Configuration()
+        ) { _, t ->
+            totalExecutionCalled++
+            if (thisInstance == null) {
+                exec(null, descriptionWithOnlyParameters, t)
+            } else {
+                exec(t.first(), descriptionWithOnlyParameters, t.drop(1))
             }
         }
+        val totalFuzzingTime = System.nanoTime() - start
         logger.info { "Finishing fuzzing for method: $methodUnderTest in ${TimeUnit.NANOSECONDS.toMillis(totalFuzzingTime)} ms" }
         logger.info { "\tTotal execution called: $totalExecutionCalled" }
+    } catch (ce: CancellationException) {
+        val totalFuzzingTime = System.nanoTime() - start
+        logger.info { "Fuzzing is stopped because of timeout. Total execution time: ${TimeUnit.NANOSECONDS.toMillis(totalFuzzingTime)} ms" }
+        logger.debug(ce) { "\tStacktrace:" }
     } catch (t: Throwable) {
         logger.info(t) { "Fuzzing is stopped because of an error" }
     }

utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt

@@ -4,6 +4,7 @@ package org.utbot.fuzzing
 import kotlinx.coroutines.yield
 import mu.KotlinLogging
 import org.utbot.fuzzing.seeds.KnownValue
+import org.utbot.fuzzing.utils.Multiset
 import org.utbot.fuzzing.utils.chooseOne
 import org.utbot.fuzzing.utils.flipCoin
 import org.utbot.fuzzing.utils.transformIfNotEmpty
@@ -36,18 +37,34 @@ interface Fuzzing<TYPE, RESULT, DESCRIPTION : Description<TYPE>, FEEDBACK : Feed
      *
      * Fuzzing combines, randomize and mutates values using the seeds.
      * Then it generates values and runs them with this method. This method should provide some feedback,
-     * which is the most important part for good fuzzing result. [emptyFeedback] can be provided only for test
-     * or infinite loops. Consider to implement own implementation of [Feedback] to provide more correct data or
+     * which is the most important part for a good fuzzing result. [emptyFeedback] can be provided only for test
+     * or infinite loops. Consider implementing own implementation of [Feedback] to provide more correct data or
      * use [BaseFeedback] to generate key based feedback. In this case, the key is used to analyse what value should be next.
      *
      * @param description contains user-defined information about current run. Can be used as a state of the run.
      * @param values current values to process.
      */
     suspend fun handle(description: DESCRIPTION, values: List<RESULT>): FEEDBACK
+
+    /**
+     * This method is called before every fuzzing attempt.
+     *
+     * Usually, is used to update configuration or manipulate some data in the statistics
+     * (for example, clear some [Statistic.missedTypes] types).
+     *
+     * @param description contains user-defined information about current run. Can be used as a state of the run.
+     * @param statistic statistic about fuzzing generation like elapsed time or number of the runs.
+     * @param configuration current used configuration; it can be changes for tuning fuzzing.
+     */
+    fun update(description: DESCRIPTION, statistic: Statistic<TYPE>, configuration: Configuration) {
+        if (!configuration.generateEmptyListForMissedTypes && statistic.missedTypes.isNotEmpty()) {
+            error("No seed candidates generated for types:\n\t${statistic.missedTypes.joinToString("\n\t")}")
+        }
+    }
 }
 
 /**
- * Some description of current fuzzing run. Usually, contains name of the target method and its parameter list.
+ * Some description of current fuzzing run. Usually, it contains the name of the target method and its parameter list.
  */
 open class Description<TYPE>(
     val parameters: List<TYPE>
@@ -235,6 +252,11 @@ private object EmptyFeedback : Feedback<Nothing, Nothing> {
     }
 }
 
+private class NoSeedValueException(
+    // this type cannot be generalized because Java forbids types for [Throwable].
+    val type: Any?
+) : Exception("No seed candidates generated for type: $type")
+
 /**
  * Starts fuzzing for this [Fuzzing] object.
  *
@@ -245,6 +267,15 @@ suspend fun <T, R, D : Description<T>, F : Feedback<T, R>> Fuzzing<T, R, D, F>.f
     random: Random = Random(0),
     configuration: Configuration = Configuration()
 ) {
+    class StatImpl(
+        override var totalRuns: Long = 0,
+        val startTime: Long = System.nanoTime(),
+        override var missedTypes: Multiset<T> = Multiset(),
+    ) : Statistic<T> {
+        override val elapsedTime: Long
+            get() = System.nanoTime() - startTime
+    }
+    val userStatistic = StatImpl()
     val fuzzing = this
     val typeCache = hashMapOf<T, List<Seed<T, R>>>()
     fun fuzzOne(): Node<T, R> = fuzz(
@@ -254,17 +285,24 @@ suspend fun <T, R, D : Description<T>, F : Feedback<T, R>> Fuzzing<T, R, D, F>.f
         random = random,
         configuration = configuration,
         builder = PassRoutine("Main Routine"),
-        state = State(1, typeCache),
+        state = State(1, typeCache, userStatistic.missedTypes),
     )
     val dynamicallyGenerated = mutableListOf<Node<T, R>>()
     val seeds = Statistics<T, R, F>()
     run breaking@ {
         sequence {
             while (description.parameters.isNotEmpty()) {
+                fuzzing.update(description, userStatistic, configuration)
                 if (dynamicallyGenerated.isNotEmpty()) {
                     yield(dynamicallyGenerated.removeFirst())
                 } else {
-                    val fuzzOne = fuzzOne()
+                    val fuzzOne = try {
+                        fuzzOne()
+                    } catch (nsv: NoSeedValueException) {
+                        @Suppress("UNCHECKED_CAST")
+                        userStatistic.missedTypes.add(nsv.type as T)
+                        continue
+                    }
                     // fuzz one value, seems to be bad, when have only a few and simple values
                     yield(fuzzOne)
 
@@ -275,7 +313,7 @@ suspend fun <T, R, D : Description<T>, F : Feedback<T, R>> Fuzzing<T, R, D, F>.f
                             fuzzing,
                             random,
                             configuration,
-                            State(1, typeCache)
+                            State(1, typeCache, userStatistic.missedTypes)
                         )
                     }
                 }
@@ -318,6 +356,7 @@ private fun <TYPE, RESULT, DESCRIPTION : Description<TYPE>, FEEDBACK : Feedback<
     val result = parameters.map { type ->
         val results = typeCache.computeIfAbsent(type) { mutableListOf() }
         if (results.isNotEmpty() && random.flipCoin(configuration.probReuseGeneratedValueForSameType)) {
+            // we need to check cases when one value is passed for different arguments
             results.random(random)
         } else {
             produce(type, fuzzing, description, random, configuration, state).also {
@@ -347,7 +386,7 @@ private fun <TYPE, RESULT, DESCRIPTION : Description<TYPE>, FEEDBACK : Feedback<
         }
     }
     if (candidates.isEmpty()) {
-        error("No seed candidates generated for type: $type")
+        throw NoSeedValueException(type)
     }
     return candidates.random(random)
 }
@@ -367,10 +406,10 @@ private fun <TYPE, RESULT, DESCRIPTION : Description<TYPE>, FEEDBACK : Feedback<
     return if (state.recursionTreeDepth > configuration.recursionTreeDepth) {
         Result.Empty { task.construct.builder(0) }
     } else {
-        val iterations = if (state.iterations >= 0 && random.flipCoin(configuration.probCreateRectangleCollectionInsteadSawLike)) {
-            state.iterations
-        } else {
-            random.nextInt(0, configuration.collectionIterations + 1)
+        val iterations = when {
+            configuration.generateEmptyListForMissedTypes && task.modify.types.any(state.missedTypes::contains) -> 0
+            state.iterations >= 0 && random.flipCoin(configuration.probCreateRectangleCollectionInsteadSawLike) -> state.iterations
+            else -> random.nextInt(0, configuration.collectionIterations + 1)
         }
         Result.Collection(
             construct = fuzz(
@@ -380,18 +419,18 @@ private fun <TYPE, RESULT, DESCRIPTION : Description<TYPE>, FEEDBACK : Feedback<
                 random,
                 configuration,
                 task.construct,
-                State(state.recursionTreeDepth + 1, state.cache, iterations)
+                State(state.recursionTreeDepth + 1, state.cache, state.missedTypes, iterations)
             ),
             modify = if (random.flipCoin(configuration.probCollectionMutationInsteadCreateNew)) {
-                val result = fuzz(task.modify.types, fuzzing, description, random, configuration, task.modify, State(state.recursionTreeDepth + 1, state.cache, iterations))
+                val result = fuzz(task.modify.types, fuzzing, description, random, configuration, task.modify, State(state.recursionTreeDepth + 1, state.cache, state.missedTypes, iterations))
                 arrayListOf(result).apply {
                     (1 until iterations).forEach { _ ->
                         add(mutate(result, fuzzing, random, configuration, state))
                     }
                 }
             } else {
                 (0 until iterations).map {
-                    fuzz(task.modify.types, fuzzing, description, random, configuration, task.modify, State(state.recursionTreeDepth + 1, state.cache, iterations))
+                    fuzz(task.modify.types, fuzzing, description, random, configuration, task.modify, State(state.recursionTreeDepth + 1, state.cache, state.missedTypes, iterations))
                 }
             },
             iterations = iterations
@@ -422,7 +461,7 @@ private fun <TYPE, RESULT, DESCRIPTION : Description<TYPE>, FEEDBACK : Feedback<
                 random,
                 configuration,
                 task.construct,
-                State(state.recursionTreeDepth + 1, state.cache)
+                State(state.recursionTreeDepth + 1, state.cache, state.missedTypes)
             ),
             modify = task.modify
                 .shuffled(random)
@@ -434,7 +473,7 @@ private fun <TYPE, RESULT, DESCRIPTION : Description<TYPE>, FEEDBACK : Feedback<
                         random,
                         configuration,
                         routine,
-                        State(state.recursionTreeDepth + 1, state.cache)
+                        State(state.recursionTreeDepth + 1, state.cache, state.missedTypes)
                     )
                 }.transformIfNotEmpty {
                     take(random.nextInt(size + 1).coerceAtLeast(1))
@@ -472,7 +511,7 @@ private fun <TYPE, RESULT, DESCRIPTION : Description<TYPE>, FEEDBACK : Feedback<
         is Result.Recursive<TYPE, RESULT> -> {
             if (resultToMutate.modify.isEmpty() || random.flipCoin(configuration.probConstructorMutationInsteadModificationMutation)) {
                 Result.Recursive(
-                    construct = mutate(resultToMutate.construct, fuzzing, random, configuration, State(state.recursionTreeDepth + 1, state.cache)),
+                    construct = mutate(resultToMutate.construct, fuzzing, random, configuration, State(state.recursionTreeDepth + 1, state.cache, state.missedTypes)),
                     modify = resultToMutate.modify
                 )
             } else if (random.flipCoin(configuration.probShuffleAndCutRecursiveObjectModificationMutation)) {
@@ -485,7 +524,7 @@ private fun <TYPE, RESULT, DESCRIPTION : Description<TYPE>, FEEDBACK : Feedback<
                     construct = resultToMutate.construct,
                     modify = resultToMutate.modify.toMutableList().apply {
                         val i = random.nextInt(0, resultToMutate.modify.size)
-                        set(i, mutate(resultToMutate.modify[i], fuzzing, random, configuration, State(state.recursionTreeDepth + 1, state.cache)))
+                        set(i, mutate(resultToMutate.modify[i], fuzzing, random, configuration, State(state.recursionTreeDepth + 1, state.cache, state.missedTypes)))
                     }
                 )
             }
@@ -496,7 +535,7 @@ private fun <TYPE, RESULT, DESCRIPTION : Description<TYPE>, FEEDBACK : Feedback<
                 if (isNotEmpty()) {
                     if (random.flipCoin(100 - configuration.probCollectionShuffleInsteadResultMutation)) {
                         val i = random.nextInt(0, resultToMutate.modify.size)
-                        set(i, mutate(resultToMutate.modify[i], fuzzing, random, configuration, State(state.recursionTreeDepth + 1, state.cache)))
+                        set(i, mutate(resultToMutate.modify[i], fuzzing, random, configuration, State(state.recursionTreeDepth + 1, state.cache, state.missedTypes)))
                     } else {
                         shuffle(random)
                     }
@@ -578,7 +617,8 @@ private data class PassRoutine<T, R>(val description: String) : Routine<T, R>(em
 private class State<TYPE, RESULT>(
     val recursionTreeDepth: Int = 1,
     val cache: MutableMap<TYPE, List<Seed<TYPE, RESULT>>>,
-    val iterations: Int = -1
+    val missedTypes: Multiset<TYPE>,
+    val iterations: Int = -1,
 )
 
 /**

utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Configuration.kt

@@ -71,5 +71,11 @@ class Configuration(
     /**
      * Probability of reusing same generated value when 2 or more parameters have the same type.
      */
-    var probReuseGeneratedValueForSameType: Int = 1
+    var probReuseGeneratedValueForSameType: Int = 1,
+
+    /**
+     * When true any [Seed.Collection] will not try
+     * to generate modification if a current type is already known to fail to generate values.
+     */
+    var generateEmptyListForMissedTypes: Boolean = true,
 )

utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Providers.kt

@@ -29,6 +29,8 @@ class BaseFuzzing<T, R, D : Description<T>, F : Feedback<T, R>>(
     val exec: suspend (description: D, values: List<R>) -> F
 ) : Fuzzing<T, R, D, F> {
 
+    var update: (D, Statistic<T>, Configuration) -> Unit = { d, s, c -> super.update(d, s, c) }
+
     constructor(vararg providers: ValueProvider<T, R, D>, exec: suspend (description: D, values: List<R>) -> F) : this(providers.toList(), exec)
 
     override fun generate(description: D, type: T): Sequence<Seed<T, R>> {
@@ -49,6 +51,10 @@ class BaseFuzzing<T, R, D : Description<T>, F : Feedback<T, R>>(
     override suspend fun handle(description: D, values: List<R>): F {
         return exec(description, values)
     }
+
+    override fun update(description: D, statistic: Statistic<T>, configuration: Configuration) {
+        update.invoke(description, statistic, configuration)
+    }
 }
 
 /**

utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Statistic.kt

@@ -0,0 +1,14 @@
+package org.utbot.fuzzing
+
+import org.utbot.fuzzing.utils.Multiset
+
+/**
+ * User class that holds data about current fuzzing running.
+ *
+ * Concrete implementation is passed to the [Fuzzing.update].
+ */
+interface Statistic<TYPE> {
+    val totalRuns: Long
+    val elapsedTime: Long
+    val missedTypes: Multiset<TYPE>
+}

utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/utils/Multiset.kt

@@ -0,0 +1,34 @@
+package org.utbot.fuzzing.utils
+
+/**
+ * Simple implementation of multiset.
+ */
+class Multiset<T> : Iterable<T> {
+
+    private val values = hashMapOf<T, Long>()
+
+    fun add(value: T): Long {
+        val result = values.getOrDefault(value, 0L) + 1
+        values[value] = result
+        return result
+    }
+
+    operator fun get(value: T): Long {
+        return values.getOrDefault(value, 0L)
+    }
+
+    fun isEmpty(): Boolean {
+        return values.size == 0
+    }
+
+    fun isNotEmpty(): Boolean = !isEmpty()
+
+    fun removeAll(value: T) {
+        values.remove(value)
+    }
+
+    override fun iterator(): Iterator<T> {
+        return values.keys.iterator()
+    }
+
+}