Add tests for fuzzing platform (#1516)

Author: Markoutte

utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt

@@ -344,7 +344,6 @@ class UtBotSymbolicEngine(
             methodUnderTest,
             collectConstantsForFuzzer(graph),
             names,
-            { mockStrategy.eligibleToMock(it, classUnderTest) },
             listOf(transform(ValueProvider.of(defaultValueProviders(defaultIdGenerator))))
         ) { thisInstance, descr, values ->
             if (controller.job?.isActive == false || System.currentTimeMillis() >= until) {

utbot-fuzzers/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt

@@ -44,7 +44,6 @@ suspend fun runJavaFuzzing(
     methodUnderTest: ExecutableId,
     constants: Collection<FuzzedConcreteValue>,
     names: List<String>,
-    mock: (ClassId) -> Boolean = { false },
     providers: List<ValueProvider<FuzzedType, FuzzedValue, FuzzedDescription>> = defaultValueProviders(idGenerator),
     exec: suspend (thisInstance: FuzzedValue?, description: FuzzedDescription, values: List<FuzzedValue>) -> BaseFeedback<Trie.Node<Instruction>, FuzzedType, FuzzedValue>
 ) {
@@ -83,7 +82,7 @@ suspend fun runJavaFuzzing(
                 null
             }
         }
-        shouldMock = mock
+        shouldMock = { false }
     }
 
     val thisInstance = with(methodUnderTest) {

utbot-fuzzers/src/test/java/org/utbot/fuzzing/samples/Stubs.java

@@ -0,0 +1,8 @@
+package org.utbot.fuzzing.samples;
+
+@SuppressWarnings("unused")
+public class Stubs {
+
+    public static void name(String value) {}
+
+}

utbot-fuzzers/src/test/kotlin/org/utbot/fuzzing/JavaFuzzingTest.kt

@@ -0,0 +1,53 @@
+package org.utbot.fuzzing
+
+import kotlinx.coroutines.runBlocking
+import org.junit.jupiter.api.Assertions
+import org.junit.jupiter.api.Test
+import org.utbot.framework.plugin.api.MethodId
+import org.utbot.framework.plugin.api.TestIdentityPreservingIdGenerator
+import org.utbot.framework.plugin.api.UtPrimitiveModel
+import org.utbot.framework.plugin.api.util.*
+import org.utbot.fuzzer.FuzzedConcreteValue
+import org.utbot.fuzzing.samples.Stubs
+import org.utbot.fuzzing.utils.Trie
+
+class JavaFuzzingTest {
+
+    @Test
+    fun `string generates same values`() {
+        fun collect(): List<String> {
+            return runBlockingWithContext {
+                val results = mutableListOf<String>()
+                var count = 0
+                val probes = 10000
+                runJavaFuzzing(
+                    TestIdentityPreservingIdGenerator,
+                    methodUnderTest = MethodId(Stubs::class.id, "name", voidClassId, listOf(stringClassId)),
+                    constants = listOf(
+                        FuzzedConcreteValue(stringClassId, "Hello"),
+                        FuzzedConcreteValue(stringClassId, "World"),
+                        FuzzedConcreteValue(stringClassId, "!"),
+                    ),
+                    names = emptyList()
+                ) { _, _, values ->
+                    results += (values.first().model as UtPrimitiveModel).value as String
+                    BaseFeedback(Trie.emptyNode(), if (++count < probes) Control.CONTINUE else Control.STOP)
+                }
+                Assertions.assertEquals(count, results.size)
+                results
+            }
+        }
+
+        val probe1 = collect()
+        val probe2 = collect()
+        Assertions.assertEquals(probe1, probe2)
+    }
+
+    private fun <T> runBlockingWithContext(block: suspend () -> T) : T {
+        return withUtContext(UtContext(this::class.java.classLoader)) {
+            runBlocking {
+                block()
+            }
+        }
+    }
+}

utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt

@@ -1,6 +1,7 @@
 @file:JvmName("FuzzingApi")
 package org.utbot.fuzzing
 
+import kotlinx.coroutines.yield
 import mu.KotlinLogging
 import org.utbot.fuzzing.seeds.KnownValue
 import org.utbot.fuzzing.utils.chooseOne
@@ -259,7 +260,7 @@ suspend fun <T, R, D : Description<T>, F : Feedback<T, R>> Fuzzing<T, R, D, F>.f
     val seeds = Statistics<T, R, F>()
     run breaking@ {
         sequence {
-            while (true) {
+            while (description.parameters.isNotEmpty()) {
                 if (dynamicallyGenerated.isNotEmpty()) {
                     yield(dynamicallyGenerated.removeFirst())
                 } else {
@@ -280,15 +281,11 @@ suspend fun <T, R, D : Description<T>, F : Feedback<T, R>> Fuzzing<T, R, D, F>.f
                 }
             }
         }.forEach execution@ { values ->
+            yield()
             check(values.parameters.size == values.result.size) { "Cannot create value for ${values.parameters}" }
             val valuesCache = mutableMapOf<Result<T, R>, R>()
             val result = values.result.map { valuesCache.computeIfAbsent(it) { r -> create(r) } }
-            val feedback = try {
-                fuzzing.handle(description, result)
-            } catch (t: Throwable) {
-                logger.error(t) { "Error when running fuzzing with $values" }
-                return@execution
-            }
+            val feedback = fuzzing.handle(description, result)
             when (feedback.control) {
                 Control.CONTINUE -> {
                     seeds.put(random, configuration, feedback, values)
@@ -634,8 +631,8 @@ private class Node<TYPE, RESULT>(
 
 
 private class Statistics<TYPE, RESULT, FEEDBACK : Feedback<TYPE, RESULT>> {
-    private val seeds = hashMapOf<FEEDBACK, Node<TYPE, RESULT>>()
-    private val count = hashMapOf<FEEDBACK, Long>()
+    private val seeds = linkedMapOf<FEEDBACK, Node<TYPE, RESULT>>()
+    private val count = linkedMapOf<FEEDBACK, Long>()
 
     fun put(random: Random, configuration: Configuration, feedback: FEEDBACK, seed: Node<TYPE, RESULT>) {
         if (random.flipCoin(configuration.probUpdateSeedInsteadOfKeepOld)) {

utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Providers.kt

@@ -70,7 +70,7 @@ fun interface ValueProvider<T, R, D : Description<T>> {
      *
      * This model provider is called before `anotherValueProviders`.
      */
-    fun with(anotherValueProvider: ValueProvider<T, R, D>): ValueProvider<T, R, D> {
+    infix fun with(anotherValueProvider: ValueProvider<T, R, D>): ValueProvider<T, R, D> {
         fun toList(m: ValueProvider<T, R, D>) = if (m is Combined<T, R, D>) m.providers else listOf(m)
         return Combined(toList(this) + toList(anotherValueProvider))
     }
@@ -122,15 +122,8 @@ fun interface ValueProvider<T, R, D : Description<T>> {
      * Creates new value provider that creates default value if no values are generated by this provider.
      */
     fun withFallback(fallbackSupplier: (T) -> Seed<T, R>) : ValueProvider<T, R, D> {
-        val thisProvider = this
-        return ValueProvider { description, type ->
-            if (accept(type)) {
-                thisProvider.generate(description, type)
-                    .takeIf { it.iterator().hasNext() }
-                    ?: sequenceOf(fallbackSupplier(type))
-            } else {
-                emptySequence()
-            }
+        return withFallback { _, type ->
+            sequenceOf(fallbackSupplier(type))
         }
     }
 
@@ -170,10 +163,6 @@ fun interface ValueProvider<T, R, D : Description<T>> {
     }
 }
 
-fun<T, R, D : Description<T>> List<ValueProvider<T, R, D>>.withFallback(fallbackSupplier: (T) -> Seed<T, R>) : List<ValueProvider<T, R, D>> {
-    return listOf(ValueProvider.of(this).withFallback(fallbackSupplier))
-}
-
 /**
  * Simple value provider for a concrete type.
  *
@@ -186,6 +175,8 @@ class TypeProvider<T, R, D : Description<T>>(
 ) : ValueProvider<T, R, D> {
     override fun accept(type: T) = this.type == type
     override fun generate(description: D, type: T) = sequence {
-        this.generate(description, type)
+        if (accept(type)) {
+            this.generate(description, type)
+        }
     }
 }

utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/demo/JsonFuzzing.kt

@@ -31,6 +31,7 @@ private class JsonBuilder(
  */
 @Suppress("RemoveExplicitTypeArguments")
 suspend fun main() {
+    var count = 0
     BaseFuzzing<CustomType, JsonBuilder, Description<CustomType>, Feedback<CustomType, JsonBuilder>>(
         TypeProvider(CustomType.INT) { _, _ ->
             for (b in Signed.values()) {
@@ -79,7 +80,7 @@ suspend fun main() {
         },
     ) { _, values ->
         println(values)
-        emptyFeedback()
+        if (++count < 1000) emptyFeedback() else error("")
     }.fuzz(
         Description(listOf(CustomType.LST, CustomType.OBJ)),
         Random(0),

utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/utils/Trie.kt

@@ -175,12 +175,6 @@ open class Trie<T, K>(
             get() = error("empty node has no data")
         override val count: Int
             get() = 0
-        override fun equals(other: Any?): Boolean {
-            return false
-        }
-        override fun hashCode(): Int {
-            return 0
-        }
     }
 
     companion object {