diff --git a/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt index 6eca308d5f..4e06efe1ea 100644 --- a/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt +++ b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt @@ -9,8 +9,8 @@ import org.utbot.fuzzer.* import org.utbot.fuzzing.providers.* import org.utbot.fuzzing.utils.Trie import java.lang.reflect.* +import java.util.concurrent.CancellationException import java.util.concurrent.TimeUnit -import kotlin.system.measureNanoTime import kotlin.random.Random private val logger = KotlinLogging.logger {} @@ -105,23 +105,31 @@ suspend fun runJavaFuzzing( val tracer = Trie(Instruction::id) val descriptionWithOptionalThisInstance = FuzzedDescription(createFuzzedMethodDescription(thisInstance), tracer, typeCache, random) val descriptionWithOnlyParameters = FuzzedDescription(createFuzzedMethodDescription(null), tracer, typeCache, random) + val start = System.nanoTime() try { logger.info { "Starting fuzzing for method: $methodUnderTest" } logger.info { "\tuse thisInstance = ${thisInstance != null}" } logger.info { "\tparameters = $parameters" } var totalExecutionCalled = 0 - val totalFuzzingTime = measureNanoTime { - runFuzzing(ValueProvider.of(providers), descriptionWithOptionalThisInstance, random) { _, t -> - totalExecutionCalled++ - if (thisInstance == null) { - exec(null, descriptionWithOnlyParameters, t) - } else { - exec(t.first(), descriptionWithOnlyParameters, t.drop(1)) - } + runFuzzing( + provider = ValueProvider.of(providers), + description = descriptionWithOptionalThisInstance, random, + configuration = Configuration() + ) { _, t -> + totalExecutionCalled++ + if (thisInstance == null) { + exec(null, descriptionWithOnlyParameters, t) + } else { + exec(t.first(), descriptionWithOnlyParameters, t.drop(1)) } } + val totalFuzzingTime = System.nanoTime() - start logger.info { "Finishing fuzzing for method: $methodUnderTest in ${TimeUnit.NANOSECONDS.toMillis(totalFuzzingTime)} ms" } logger.info { "\tTotal execution called: $totalExecutionCalled" } + } catch (ce: CancellationException) { + val totalFuzzingTime = System.nanoTime() - start + logger.info { "Fuzzing is stopped because of timeout. Total execution time: ${TimeUnit.NANOSECONDS.toMillis(totalFuzzingTime)} ms" } + logger.debug(ce) { "\tStacktrace:" } } catch (t: Throwable) { logger.info(t) { "Fuzzing is stopped because of an error" } } diff --git a/utbot-fuzzers/src/test/java/org/utbot/fuzzing/samples/FailToGenerateListGeneric.java b/utbot-fuzzers/src/test/java/org/utbot/fuzzing/samples/FailToGenerateListGeneric.java new file mode 100644 index 0000000000..0e8a3d9aaf --- /dev/null +++ b/utbot-fuzzers/src/test/java/org/utbot/fuzzing/samples/FailToGenerateListGeneric.java @@ -0,0 +1,14 @@ +package org.utbot.fuzzing.samples; + +import java.util.List; + +@SuppressWarnings("unused") +public class FailToGenerateListGeneric { + + interface Something {} + + int func(List x) { + return x.size(); + } + +} diff --git a/utbot-fuzzers/src/test/kotlin/org/utbot/fuzzing/JavaFuzzingTest.kt b/utbot-fuzzers/src/test/kotlin/org/utbot/fuzzing/JavaFuzzingTest.kt index 09dc76233e..0c9993e38d 100644 --- a/utbot-fuzzers/src/test/kotlin/org/utbot/fuzzing/JavaFuzzingTest.kt +++ b/utbot-fuzzers/src/test/kotlin/org/utbot/fuzzing/JavaFuzzingTest.kt @@ -1,17 +1,19 @@ package org.utbot.fuzzing import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeout import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.Test -import org.junit.jupiter.api.assertDoesNotThrow import org.utbot.framework.plugin.api.MethodId import org.utbot.framework.plugin.api.TestIdentityPreservingIdGenerator +import org.utbot.framework.plugin.api.UtAssembleModel import org.utbot.framework.plugin.api.UtPrimitiveModel import org.utbot.framework.plugin.api.util.* import org.utbot.fuzzer.FuzzedConcreteValue import org.utbot.fuzzing.samples.DeepNested import org.utbot.fuzzer.FuzzedType import org.utbot.fuzzing.samples.AccessibleObjects +import org.utbot.fuzzing.samples.FailToGenerateListGeneric import org.utbot.fuzzing.samples.Stubs import org.utbot.fuzzing.utils.Trie import java.lang.reflect.GenericArrayType @@ -214,10 +216,36 @@ class JavaFuzzingTest { assertEquals(0, exec) { "Fuzzer should not create any values of private classes" } } + @Test + fun `fuzzing generate single test in case of collection with fail-to-generate generic type`() { + val size = 100 + var exec = size + val collections = ArrayList(exec) + runBlockingWithContext { + runJavaFuzzing( + TestIdentityPreservingIdGenerator, + methodUnderTest = FailToGenerateListGeneric::class.java.declaredMethods.first { it.name == "func" }.executableId, + constants = emptyList(), + names = emptyList() + ) { _, _, v -> + collections.add(v.first().model as? UtAssembleModel) + BaseFeedback(Trie.emptyNode(), if (--exec > 0) Control.CONTINUE else Control.STOP) + } + } + assertEquals(0, exec) { "Total fuzzer run number must be 0" } + assertEquals(size, collections.size) { "Total generated values number must be $size" } + assertEquals(size, collections.count { it is UtAssembleModel }) { "Total assemble models size must be $size" } + collections.filterIsInstance().forEach { + assertEquals(0, it.modificationsChain.size) + } + } + private fun runBlockingWithContext(block: suspend () -> T) : T { return withUtContext(UtContext(this::class.java.classLoader)) { runBlocking { - block() + withTimeout(10000) { + block() + } } } } diff --git a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt index cfa84ed253..fc1ace3ea4 100644 --- a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt +++ b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt @@ -4,6 +4,7 @@ package org.utbot.fuzzing import kotlinx.coroutines.yield import mu.KotlinLogging import org.utbot.fuzzing.seeds.KnownValue +import org.utbot.fuzzing.utils.MissedSeed import org.utbot.fuzzing.utils.chooseOne import org.utbot.fuzzing.utils.flipCoin import org.utbot.fuzzing.utils.transformIfNotEmpty @@ -36,18 +37,30 @@ interface Fuzzing, FEEDBACK : Feed * * Fuzzing combines, randomize and mutates values using the seeds. * Then it generates values and runs them with this method. This method should provide some feedback, - * which is the most important part for good fuzzing result. [emptyFeedback] can be provided only for test - * or infinite loops. Consider to implement own implementation of [Feedback] to provide more correct data or + * which is the most important part for a good fuzzing result. [emptyFeedback] can be provided only for test + * or infinite loops. Consider implementing own implementation of [Feedback] to provide more correct data or * use [BaseFeedback] to generate key based feedback. In this case, the key is used to analyse what value should be next. * * @param description contains user-defined information about current run. Can be used as a state of the run. * @param values current values to process. */ suspend fun handle(description: DESCRIPTION, values: List): FEEDBACK + + /** + * This method is called before every fuzzing attempt. + * + * Usually, is used to update configuration or manipulate some data in the statistics + * (for example, clear some [Statistic.missedTypes] types). + * + * @param description contains user-defined information about current run. Can be used as a state of the run. + * @param statistic statistic about fuzzing generation like elapsed time or number of the runs. + * @param configuration current used configuration; it can be changes for tuning fuzzing. + */ + suspend fun update(description: DESCRIPTION, statistic: Statistic, configuration: Configuration) {} } /** - * Some description of current fuzzing run. Usually, contains name of the target method and its parameter list. + * Some description of current fuzzing run. Usually, it contains the name of the target method and its parameter list. */ open class Description( val parameters: List @@ -235,6 +248,11 @@ private object EmptyFeedback : Feedback { } } +private class NoSeedValueException( + // this type cannot be generalized because Java forbids types for [Throwable]. + val type: Any? +) : Exception("No seed candidates generated for type: $type") + /** * Starts fuzzing for this [Fuzzing] object. * @@ -245,6 +263,15 @@ suspend fun , F : Feedback> Fuzzing.f random: Random = Random(0), configuration: Configuration = Configuration() ) { + class StatImpl( + override var totalRuns: Long = 0, + val startTime: Long = System.nanoTime(), + override var missedTypes: MissedSeed = MissedSeed(), + ) : Statistic { + override val elapsedTime: Long + get() = System.nanoTime() - startTime + } + val userStatistic = StatImpl() val fuzzing = this val typeCache = hashMapOf>>() fun fuzzOne(): Node = fuzz( @@ -254,7 +281,7 @@ suspend fun , F : Feedback> Fuzzing.f random = random, configuration = configuration, builder = PassRoutine("Main Routine"), - state = State(1, typeCache), + state = State(1, typeCache, userStatistic.missedTypes), ) val dynamicallyGenerated = mutableListOf>() val seeds = Statistics() @@ -275,13 +302,16 @@ suspend fun , F : Feedback> Fuzzing.f fuzzing, random, configuration, - State(1, typeCache) + State(1, typeCache, userStatistic.missedTypes) ) } } } }.forEach execution@ { values -> yield() + fuzzing.update(description, userStatistic.apply { + totalRuns++ + }, configuration) check(values.parameters.size == values.result.size) { "Cannot create value for ${values.parameters}" } val valuesCache = mutableMapOf, R>() val result = values.result.map { valuesCache.computeIfAbsent(it) { r -> create(r) } } @@ -318,6 +348,7 @@ private fun , FEEDBACK : Feedback< val result = parameters.map { type -> val results = typeCache.computeIfAbsent(type) { mutableListOf() } if (results.isNotEmpty() && random.flipCoin(configuration.probReuseGeneratedValueForSameType)) { + // we need to check cases when one value is passed for different arguments results.random(random) } else { produce(type, fuzzing, description, random, configuration, state).also { @@ -347,7 +378,7 @@ private fun , FEEDBACK : Feedback< } } if (candidates.isEmpty()) { - error("No seed candidates generated for type: $type") + throw NoSeedValueException(type) } return candidates.random(random) } @@ -366,11 +397,10 @@ private fun , FEEDBACK : Feedback< ): Result { return if (state.recursionTreeDepth > configuration.recursionTreeDepth) { Result.Empty { task.construct.builder(0) } - } else { - val iterations = if (state.iterations >= 0 && random.flipCoin(configuration.probCreateRectangleCollectionInsteadSawLike)) { - state.iterations - } else { - random.nextInt(0, configuration.collectionIterations + 1) + } else try { + val iterations = when { + state.iterations >= 0 && random.flipCoin(configuration.probCreateRectangleCollectionInsteadSawLike) -> state.iterations + else -> random.nextInt(0, configuration.collectionIterations + 1) } Result.Collection( construct = fuzz( @@ -380,10 +410,10 @@ private fun , FEEDBACK : Feedback< random, configuration, task.construct, - State(state.recursionTreeDepth + 1, state.cache, iterations) + State(state.recursionTreeDepth + 1, state.cache, state.missedTypes, iterations) ), modify = if (random.flipCoin(configuration.probCollectionMutationInsteadCreateNew)) { - val result = fuzz(task.modify.types, fuzzing, description, random, configuration, task.modify, State(state.recursionTreeDepth + 1, state.cache, iterations)) + val result = fuzz(task.modify.types, fuzzing, description, random, configuration, task.modify, State(state.recursionTreeDepth + 1, state.cache, state.missedTypes, iterations)) arrayListOf(result).apply { (1 until iterations).forEach { _ -> add(mutate(result, fuzzing, random, configuration, state)) @@ -391,11 +421,19 @@ private fun , FEEDBACK : Feedback< } } else { (0 until iterations).map { - fuzz(task.modify.types, fuzzing, description, random, configuration, task.modify, State(state.recursionTreeDepth + 1, state.cache, iterations)) + fuzz(task.modify.types, fuzzing, description, random, configuration, task.modify, State(state.recursionTreeDepth + 1, state.cache, state.missedTypes, iterations)) } }, iterations = iterations ) + } catch (nsv: NoSeedValueException) { + @Suppress("UNCHECKED_CAST") + state.missedTypes[nsv.type as TYPE] = task + if (configuration.generateEmptyCollectionsForMissedTypes) { + Result.Empty { task.construct.builder(0) } + } else { + throw nsv + } } } @@ -413,7 +451,7 @@ private fun , FEEDBACK : Feedback< ): Result { return if (state.recursionTreeDepth > configuration.recursionTreeDepth) { Result.Empty { task.empty.builder() } - } else { + } else try { Result.Recursive( construct = fuzz( task.construct.types, @@ -422,7 +460,7 @@ private fun , FEEDBACK : Feedback< random, configuration, task.construct, - State(state.recursionTreeDepth + 1, state.cache) + State(state.recursionTreeDepth + 1, state.cache, state.missedTypes) ), modify = task.modify .shuffled(random) @@ -434,12 +472,20 @@ private fun , FEEDBACK : Feedback< random, configuration, routine, - State(state.recursionTreeDepth + 1, state.cache) + State(state.recursionTreeDepth + 1, state.cache, state.missedTypes) ) }.transformIfNotEmpty { take(random.nextInt(size + 1).coerceAtLeast(1)) } ) + } catch (nsv: NoSeedValueException) { + @Suppress("UNCHECKED_CAST") + state.missedTypes[nsv.type as TYPE] = task + if (configuration.generateEmptyRecursiveForMissedTypes) { + Result.Empty { task.empty() } + } else { + throw nsv + } } } @@ -472,7 +518,7 @@ private fun , FEEDBACK : Feedback< is Result.Recursive -> { if (resultToMutate.modify.isEmpty() || random.flipCoin(configuration.probConstructorMutationInsteadModificationMutation)) { Result.Recursive( - construct = mutate(resultToMutate.construct, fuzzing, random, configuration, State(state.recursionTreeDepth + 1, state.cache)), + construct = mutate(resultToMutate.construct, fuzzing, random, configuration, State(state.recursionTreeDepth + 1, state.cache, state.missedTypes)), modify = resultToMutate.modify ) } else if (random.flipCoin(configuration.probShuffleAndCutRecursiveObjectModificationMutation)) { @@ -485,7 +531,7 @@ private fun , FEEDBACK : Feedback< construct = resultToMutate.construct, modify = resultToMutate.modify.toMutableList().apply { val i = random.nextInt(0, resultToMutate.modify.size) - set(i, mutate(resultToMutate.modify[i], fuzzing, random, configuration, State(state.recursionTreeDepth + 1, state.cache))) + set(i, mutate(resultToMutate.modify[i], fuzzing, random, configuration, State(state.recursionTreeDepth + 1, state.cache, state.missedTypes))) } ) } @@ -496,7 +542,7 @@ private fun , FEEDBACK : Feedback< if (isNotEmpty()) { if (random.flipCoin(100 - configuration.probCollectionShuffleInsteadResultMutation)) { val i = random.nextInt(0, resultToMutate.modify.size) - set(i, mutate(resultToMutate.modify[i], fuzzing, random, configuration, State(state.recursionTreeDepth + 1, state.cache))) + set(i, mutate(resultToMutate.modify[i], fuzzing, random, configuration, State(state.recursionTreeDepth + 1, state.cache, state.missedTypes))) } else { shuffle(random) } @@ -578,7 +624,8 @@ private data class PassRoutine(val description: String) : Routine(em private class State( val recursionTreeDepth: Int = 1, val cache: MutableMap>>, - val iterations: Int = -1 + val missedTypes: MissedSeed, + val iterations: Int = -1, ) /** diff --git a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Configuration.kt b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Configuration.kt index 77cd6650b0..29cf61b5a3 100644 --- a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Configuration.kt +++ b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Configuration.kt @@ -71,5 +71,17 @@ class Configuration( /** * Probability of reusing same generated value when 2 or more parameters have the same type. */ - var probReuseGeneratedValueForSameType: Int = 1 + var probReuseGeneratedValueForSameType: Int = 1, + + /** + * When true any [Seed.Collection] will not try + * to generate modification if a current type is already known to fail to generate values. + */ + var generateEmptyCollectionsForMissedTypes: Boolean = true, + + /** + * When true any [Seed.Recursive] will not try + * to generate a recursive object, but will use [Seed.Recursive.empty] instead. + */ + var generateEmptyRecursiveForMissedTypes: Boolean = true, ) \ No newline at end of file diff --git a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Providers.kt b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Providers.kt index f2b1e9ca1c..319dfecb64 100644 --- a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Providers.kt +++ b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Providers.kt @@ -29,6 +29,8 @@ class BaseFuzzing, F : Feedback>( val exec: suspend (description: D, values: List) -> F ) : Fuzzing { + var update: suspend (D, Statistic, Configuration) -> Unit = { d, s, c -> super.update(d, s, c) } + constructor(vararg providers: ValueProvider, exec: suspend (description: D, values: List) -> F) : this(providers.toList(), exec) override fun generate(description: D, type: T): Sequence> { @@ -49,6 +51,10 @@ class BaseFuzzing, F : Feedback>( override suspend fun handle(description: D, values: List): F { return exec(description, values) } + + override suspend fun update(description: D, statistic: Statistic, configuration: Configuration) { + update.invoke(description, statistic, configuration) + } } /** diff --git a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Statistic.kt b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Statistic.kt new file mode 100644 index 0000000000..3641d3b064 --- /dev/null +++ b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Statistic.kt @@ -0,0 +1,14 @@ +package org.utbot.fuzzing + +import org.utbot.fuzzing.utils.MissedSeed + +/** + * User class that holds data about current fuzzing running. + * + * Concrete implementation is passed to the [Fuzzing.update]. + */ +interface Statistic { + val totalRuns: Long + val elapsedTime: Long + val missedTypes: MissedSeed +} \ No newline at end of file diff --git a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/utils/MissedSeed.kt b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/utils/MissedSeed.kt new file mode 100644 index 0000000000..2618238522 --- /dev/null +++ b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/utils/MissedSeed.kt @@ -0,0 +1,27 @@ +package org.utbot.fuzzing.utils + +import org.utbot.fuzzing.Seed + +class MissedSeed : Iterable { + + private val values = hashMapOf>() + + operator fun set(value: T, seed: Seed) { + values[value] = seed + } + + operator fun get(value: T): Seed? { + return values[value] + } + + fun isEmpty(): Boolean { + return values.size == 0 + } + + fun isNotEmpty(): Boolean = !isEmpty() + + override fun iterator(): Iterator { + return values.keys.iterator() + } + +} \ No newline at end of file