Introduce fork for fuzzing tasks

Author: Markoutte

utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt

@@ -1,7 +1,7 @@
 @file:JvmName("FuzzingApi")
 package org.utbot.fuzzing
 
-import kotlinx.coroutines.yield
+import kotlinx.coroutines.*
 import mu.KotlinLogging
 import org.utbot.fuzzing.seeds.KnownValue
 import org.utbot.fuzzing.utils.MissedSeed
@@ -47,24 +47,31 @@ interface Fuzzing<TYPE, RESULT, DESCRIPTION : Description<TYPE>, FEEDBACK : Feed
     suspend fun handle(description: DESCRIPTION, values: List<RESULT>): FEEDBACK
 
     /**
-     * This method is called before every fuzzing attempt.
-     *
-     * Usually, is used to update configuration or manipulate some data in the statistics
-     * (for example, clear some [Statistic.missedTypes] types).
-     *
-     * @param description contains user-defined information about current run. Can be used as a state of the run.
-     * @param statistic statistic about fuzzing generation like elapsed time or number of the runs.
-     * @param configuration current used configuration; it can be changes for tuning fuzzing.
+     * Starts fuzzing with new description but with copy of [Statistic].
+     */
+    suspend fun fork(description: DESCRIPTION, statistics: Statistic<TYPE, RESULT>) {
+        fuzz(description, StatisticImpl(statistics))
+    }
+
+    /**
+     * Checks whether the fuzzer should stop.
      */
-    suspend fun update(description: DESCRIPTION, statistic: Statistic<TYPE, RESULT>, configuration: Configuration) {}
+    suspend fun isCancelled(description: DESCRIPTION, stats: Statistic<TYPE, RESULT>): Boolean {
+        return description.parameters.isEmpty()
+    }
+
+    suspend fun beforeIteration(description: DESCRIPTION, statistics: Statistic<TYPE, RESULT>) { }
+    suspend fun afterIteration(description: DESCRIPTION, statistics: Statistic<TYPE, RESULT>) { }
 }
 
 /**
  * Some description of current fuzzing run. Usually, it contains the name of the target method and its parameter list.
  */
 open class Description<TYPE>(
-    val parameters: List<TYPE>
-)
+    parameters: List<TYPE>
+) {
+    val parameters: List<TYPE> = parameters.toList()
+}
 
 /**
  * Input value that fuzzing knows how to build and use them.
@@ -115,7 +122,7 @@ sealed interface Seed<TYPE, RESULT> {
  * Routine is a task that is used to build a value.
  *
  * There are several types of a routine, which all are generally only functions.
- * These function accepts some data and generates target value.
+ * These functions accept some data and generate target value.
  */
 sealed class Routine<T, R>(val types: List<T>) : Iterable<T> by types {
 
@@ -211,13 +218,6 @@ enum class Control {
      */
     CONTINUE,
 
-    /**
-     * Reset type cache and continue.
-     *
-     * Current seed and result will be analysed and cached.
-     */
-    RESET_TYPE_CACHE_AND_CONTINUE,
-
     /**
      * Do not process this feedback and just start next value generation.
      */
@@ -253,81 +253,71 @@ class NoSeedValueException internal constructor(
     val type: Any?
 ) : Exception("No seed candidates generated for type: $type")
 
+suspend fun <T, R, D : Description<T>, F : Feedback<T, R>> Fuzzing<T, R, D, F>.fuzz(
+    description: D,
+    random: Random = Random(0),
+    configuration: Configuration = Configuration()
+) {
+    fuzz(description, StatisticImpl(random = random, configuration = configuration))
+}
+
 /**
  * Starts fuzzing for this [Fuzzing] object.
  *
  * This is an entry point for every fuzzing.
  */
-suspend fun <T, R, D : Description<T>, F : Feedback<T, R>> Fuzzing<T, R, D, F>.fuzz(
+private suspend fun <T, R, D : Description<T>, F : Feedback<T, R>> Fuzzing<T, R, D, F>.fuzz(
     description: D,
-    random: Random = Random(0),
-    configuration: Configuration = Configuration()
+    statistic: StatisticImpl<T, R, F>,
 ) {
-    class StatImpl(
-        override var totalRuns: Long = 0,
-        val startTime: Long = System.nanoTime(),
-        override var missedTypes: MissedSeed<T, R> = MissedSeed(),
-    ) : Statistic<T, R> {
-        override val elapsedTime: Long
-            get() = System.nanoTime() - startTime
-    }
-    val userStatistic = StatImpl()
+    val random = statistic.random
+    val configuration = statistic.configuration
     val fuzzing = this
     val typeCache = hashMapOf<T, List<Seed<T, R>>>()
-    fun fuzzOne(): Node<T, R> = fuzz(
-        parameters = description.parameters,
+    fun fuzzOne(parameters: List<T>): Node<T, R> = fuzz(
+        parameters = parameters,
         fuzzing = fuzzing,
         description = description,
         random = random,
         configuration = configuration,
         builder = PassRoutine("Main Routine"),
-        state = State(1, typeCache, userStatistic.missedTypes),
+        state = State(1, typeCache, statistic.missedTypes),
     )
-    val dynamicallyGenerated = mutableListOf<Node<T, R>>()
-    val seeds = Statistics<T, R, F>()
-    run breaking@ {
-        sequence {
-            while (description.parameters.isNotEmpty()) {
-                if (dynamicallyGenerated.isNotEmpty()) {
-                    yield(dynamicallyGenerated.removeFirst())
-                } else {
-                    val fuzzOne = fuzzOne()
-                    // fuzz one value, seems to be bad, when have only a few and simple values
-                    yield(fuzzOne)
-
-                    val randomSeed = seeds.getRandomSeed(random, configuration)
-                    if (randomSeed != null) {
-                        dynamicallyGenerated += mutate(
-                            randomSeed,
-                            fuzzing,
-                            random,
-                            configuration,
-                            State(1, typeCache, userStatistic.missedTypes)
-                        )
-                    }
-                }
+
+    while (!fuzzing.isCancelled(description, statistic)) {
+        beforeIteration(description, statistic)
+        val seed = if (statistic.isNotEmpty() && random.flipCoin(configuration.probSeedRetrievingInsteadGenerating)) {
+            statistic.getRandomSeed(random, configuration)
+        } else {
+            val actualParameters = description.parameters
+            // fuzz one value, seems to be bad, when have only a few and simple values
+            fuzzOne(actualParameters)
+        }
+        val values = mutate(
+            seed,
+            fuzzing,
+            random,
+            configuration,
+            State(1, typeCache, statistic.missedTypes)
+        )
+        afterIteration(description, statistic)
+
+        yield()
+        statistic.apply {
+            totalRuns++
+        }
+        check(values.parameters.size == values.result.size) { "Cannot create value for ${values.parameters}" }
+        val valuesCache = mutableMapOf<Result<T, R>, R>()
+        val result = values.result.map { valuesCache.computeIfAbsent(it) { r -> create(r) } }
+        val feedback = fuzzing.handle(description, result)
+        when (feedback.control) {
+            Control.CONTINUE -> {
+                statistic.put(random, configuration, feedback, values)
             }
-        }.forEach execution@ { values ->
-            yield()
-            fuzzing.update(description, userStatistic.apply {
-                totalRuns++
-            }, configuration)
-            check(values.parameters.size == values.result.size) { "Cannot create value for ${values.parameters}" }
-            val valuesCache = mutableMapOf<Result<T, R>, R>()
-            val result = values.result.map { valuesCache.computeIfAbsent(it) { r -> create(r) } }
-            val feedback = fuzzing.handle(description, result)
-            when (feedback.control) {
-                Control.CONTINUE -> {
-                    seeds.put(random, configuration, feedback, values)
-                }
-                Control.RESET_TYPE_CACHE_AND_CONTINUE -> {
-                    dynamicallyGenerated.clear()
-                    typeCache.clear()
-                    seeds.put(random, configuration, feedback, values)
-                }
-                Control.STOP -> { return@breaking }
-                Control.PASS -> {}
+            Control.STOP -> {
+                break
             }
+            Control.PASS -> {}
         }
     }
 }
@@ -678,8 +668,24 @@ private class Node<TYPE, RESULT>(
     val builder: Routine<TYPE, RESULT>,
 )
 
+private class StatisticImpl<TYPE, RESULT, FEEDBACK : Feedback<TYPE, RESULT>>(
+    override var totalRuns: Long = 0,
+    override val startTime: Long = System.nanoTime(),
+    override var missedTypes: MissedSeed<TYPE, RESULT> = MissedSeed(),
+    override val random: Random,
+    override val configuration: Configuration,
+) : Statistic<TYPE, RESULT> {
+
+    constructor(source: Statistic<TYPE, RESULT>) : this(
+        totalRuns = source.totalRuns,
+        startTime = source.startTime,
+        missedTypes = source.missedTypes,
+        random = source.random,
+        configuration = source.configuration.copy(),
+    )
 
-private class Statistics<TYPE, RESULT, FEEDBACK : Feedback<TYPE, RESULT>> {
+    override val elapsedTime: Long
+        get() = System.nanoTime() - startTime
     private val seeds = linkedMapOf<FEEDBACK, Node<TYPE, RESULT>>()
     private val count = linkedMapOf<FEEDBACK, Long>()
 
@@ -692,8 +698,8 @@ private class Statistics<TYPE, RESULT, FEEDBACK : Feedback<TYPE, RESULT>> {
         count[feedback] = count.getOrDefault(feedback, 0L) + 1L
     }
 
-    fun getRandomSeed(random: Random, configuration: Configuration): Node<TYPE, RESULT>? {
-        if (seeds.isEmpty()) return null
+    fun getRandomSeed(random: Random, configuration: Configuration): Node<TYPE, RESULT> {
+        if (seeds.isEmpty()) error("Call `isNotEmpty` before getting the seed")
         val entries = seeds.entries.toList()
         val frequencies = DoubleArray(seeds.size).also { f ->
             entries.forEachIndexed { index, (key, _) ->
@@ -703,6 +709,7 @@ private class Statistics<TYPE, RESULT, FEEDBACK : Feedback<TYPE, RESULT>> {
         val index = random.chooseOne(frequencies)
         return entries[index].value
     }
-}
 
-///endregion
+    fun isNotEmpty() = seeds.isNotEmpty()
+}
+///endregion

utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Configuration.kt

@@ -5,7 +5,13 @@ import kotlin.math.pow
 /**
  * Configures fuzzing behaviour. Usually, it is not required to tune anything.
  */
-class Configuration(
+data class Configuration(
+
+    /**
+     * Choose between already generated values and new generation of values.
+     */
+    var probSeedRetrievingInsteadGenerating: Int = 70,
+
     /**
      * Fuzzer creates a tree of object for generating values. At some point this recursion should be stopped.
      *

utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Providers.kt

@@ -29,8 +29,6 @@ class BaseFuzzing<T, R, D : Description<T>, F : Feedback<T, R>>(
     val exec: suspend (description: D, values: List<R>) -> F
 ) : Fuzzing<T, R, D, F> {
 
-    var update: suspend (D, Statistic<T, R>, Configuration) -> Unit = { d, s, c -> super.update(d, s, c) }
-
     constructor(vararg providers: ValueProvider<T, R, D>, exec: suspend (description: D, values: List<R>) -> F) : this(providers.toList(), exec)
 
     override fun generate(description: D, type: T): Sequence<Seed<T, R>> {
@@ -51,10 +49,6 @@ class BaseFuzzing<T, R, D : Description<T>, F : Feedback<T, R>>(
     override suspend fun handle(description: D, values: List<R>): F {
         return exec(description, values)
     }
-
-    override suspend fun update(description: D, statistic: Statistic<T, R>, configuration: Configuration) {
-        update.invoke(description, statistic, configuration)
-    }
 }
 
 /**

utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Statistic.kt

@@ -1,14 +1,16 @@
 package org.utbot.fuzzing
 
 import org.utbot.fuzzing.utils.MissedSeed
+import kotlin.random.Random
 
 /**
  * User class that holds data about current fuzzing running.
- *
- * Concrete implementation is passed to the [Fuzzing.update].
  */
 interface Statistic<TYPE, RESULT> {
+    val startTime: Long
     val totalRuns: Long
     val elapsedTime: Long
     val missedTypes: MissedSeed<TYPE, RESULT>
+    val random: Random
+    val configuration: Configuration
 }

utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/demo/ForkFuzzing.kt

@@ -0,0 +1,57 @@
+package org.utbot.fuzzing.demo
+
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.runBlocking
+import org.utbot.fuzzing.*
+import java.util.concurrent.atomic.AtomicLong
+
+private enum class Type {
+    ANY, CONCRETE, MORE_CONCRETE
+}
+
+fun main(): Unit = runBlocking {
+    launch {
+        object : Fuzzing<Type, String, Description<Type>, Feedback<Type, String>> {
+
+            private val runs = mutableMapOf<Type, AtomicLong>()
+
+            override fun generate(description: Description<Type>, type: Type): Sequence<Seed<Type, String>> {
+                return sequenceOf(Seed.Simple(type.name))
+            }
+
+            override suspend fun handle(description: Description<Type>, values: List<String>): Feedback<Type, String> {
+                description.parameters.forEach {
+                    runs[it]!!.incrementAndGet()
+                }
+                println(values)
+                return emptyFeedback()
+            }
+
+            override suspend fun afterIteration(
+                description: Description<Type>,
+                stats: Statistic<Type, String>,
+            ) {
+                if (stats.totalRuns % 10 == 0L && description.parameters.size == 1) {
+                    val newTypes = when (description.parameters[0]) {
+                        Type.ANY -> listOf(Type.CONCRETE)
+                        Type.CONCRETE -> listOf(Type.MORE_CONCRETE)
+                        Type.MORE_CONCRETE -> listOf()
+                    }
+                    if (newTypes.isNotEmpty()) {
+                        val d = Description(newTypes)
+                        fork(d, stats)
+                        // Description can be used as a transfer object,
+                        // that collects information about the current running.
+                        println("Fork ended: ${d.parameters}")
+                    }
+                }
+            }
+
+            override suspend fun isCancelled(description: Description<Type>, stats: Statistic<Type, String>): Boolean {
+                println("info: ${description.parameters} runs ${stats.totalRuns}")
+                return description.parameters.all { runs.computeIfAbsent(it) { AtomicLong(0) }.get() >= 10 }
+            }
+        }.fuzz(Description(listOf(Type.ANY)))
+    }
+//        .cancel()
+}

utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/demo/JsonFuzzing.kt

@@ -4,6 +4,7 @@ import org.utbot.fuzzing.*
 import org.utbot.fuzzing.seeds.BitVectorValue
 import org.utbot.fuzzing.seeds.Signed
 import org.utbot.fuzzing.seeds.StringValue
+import java.util.concurrent.atomic.AtomicLong
 import kotlin.random.Random
 
 private enum class CustomType {
@@ -32,6 +33,7 @@ private class JsonBuilder(
 @Suppress("RemoveExplicitTypeArguments")
 suspend fun main() {
     var count = 0
+    val set = mutableMapOf<String, AtomicLong>()
     BaseFuzzing<CustomType, JsonBuilder, Description<CustomType>, Feedback<CustomType, JsonBuilder>>(
         TypeProvider(CustomType.INT) { _, _ ->
             for (b in Signed.values()) {
@@ -79,8 +81,13 @@ suspend fun main() {
             }
         },
     ) { _, values ->
-        println(values)
-        if (++count < 1000) emptyFeedback() else error("")
+        val result = values.toString()
+        println(result)
+        set.computeIfAbsent(result) { AtomicLong() }.incrementAndGet()
+        if (++count < 10000) emptyFeedback() else {
+            println("Duplication ratio:" + set.size / count.toDouble())
+            error("Forced from the example")
+        }
     }.fuzz(
         Description(listOf(CustomType.LST, CustomType.OBJ)),
         Random(0),