diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt index 7c91ada79d..4eb56f7ba2 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt @@ -88,9 +88,13 @@ import org.utbot.fuzzer.FuzzedValue import org.utbot.fuzzer.ModelProvider import org.utbot.fuzzer.ReferencePreservingIntIdGenerator import org.utbot.fuzzer.Trie +import org.utbot.fuzzer.TrieBasedFuzzerStatistics import org.utbot.fuzzer.UtFuzzedExecution +import org.utbot.fuzzer.withMutations import org.utbot.fuzzer.collectConstantsForFuzzer import org.utbot.fuzzer.defaultModelProviders +import org.utbot.fuzzer.defaultModelMutators +import org.utbot.fuzzer.flipCoin import org.utbot.fuzzer.fuzz import org.utbot.fuzzer.providers.ObjectModelProvider import org.utbot.instrumentation.ConcreteExecutor @@ -392,6 +396,7 @@ class UtBotSymbolicEngine( val fallbackModelProvider = FallbackModelProvider(defaultIdGenerator) val constantValues = collectConstantsForFuzzer(graph) + val random = Random(0) val thisInstance = when { methodUnderTest.isStatic -> null methodUnderTest.isConstructor -> if ( @@ -405,7 +410,7 @@ class UtBotSymbolicEngine( else -> { ObjectModelProvider(defaultIdGenerator).withFallback(fallbackModelProvider).generate( FuzzedMethodDescription("thisInstance", voidClassId, listOf(methodUnderTest.clazz.id), constantValues) - ).take(10).shuffled(Random(0)).map { it.value.model }.first().apply { + ).take(10).shuffled(random).map { it.value.model }.first().apply { if (this is UtNullModel) { // it will definitely fail because of NPE, return@flow } @@ -421,8 +426,9 @@ class UtBotSymbolicEngine( parameterNameMap = { index -> names?.getOrNull(index) } } val coveredInstructionTracker = Trie(Instruction::id) - val coveredInstructionValues = mutableMapOf, List>() - var attempts = UtSettings.fuzzingMaxAttempts + val coveredInstructionValues = linkedMapOf, List>() + var attempts = 0 + val attemptsLimit = UtSettings.fuzzingMaxAttempts val hasMethodUnderTestParametersToFuzz = executableId.parameters.isNotEmpty() val fuzzedValues = if (hasMethodUnderTestParametersToFuzz) { fuzz(methodUnderTestDescription, modelProvider(defaultModelProviders(defaultIdGenerator))) @@ -435,7 +441,9 @@ class UtBotSymbolicEngine( fuzz(thisMethodDescription, ObjectModelProvider(defaultIdGenerator).apply { limitValuesCreatedByFieldAccessors = 500 }) - } + }.withMutations( + TrieBasedFuzzerStatistics(coveredInstructionValues), methodUnderTestDescription, *defaultModelMutators().toTypedArray() + ) fuzzedValues.forEach { values -> if (controller.job?.isActive == false || System.currentTimeMillis() >= until) { logger.info { "Fuzzing overtime: $methodUnderTest" } @@ -473,14 +481,19 @@ class UtBotSymbolicEngine( val coveredInstructions = concreteExecutionResult.coverage.coveredInstructions if (coveredInstructions.isNotEmpty()) { - val count = coveredInstructionTracker.add(coveredInstructions) - if (count.count > 1) { - if (--attempts < 0) { + val coverageKey = coveredInstructionTracker.add(coveredInstructions) + if (coverageKey.count > 1) { + if (++attempts >= attemptsLimit) { return@flow } + // Update the seeded values sometimes + // This is necessary because some values cannot do a good values in mutation in any case + if (random.flipCoin(probability = 50)) { + coveredInstructionValues[coverageKey] = values + } return@forEach } - coveredInstructionValues[count] = values + coveredInstructionValues[coverageKey] = values } else { logger.error { "Coverage is empty for $methodUnderTest with ${values.map { it.model }}" } } diff --git a/utbot-framework/src/main/kotlin/org/utbot/fuzzer/FuzzerFunctions.kt b/utbot-framework/src/main/kotlin/org/utbot/fuzzer/FuzzerFunctions.kt index 9efdcb7ad4..3c32087b6b 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/fuzzer/FuzzerFunctions.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/fuzzer/FuzzerFunctions.kt @@ -204,7 +204,7 @@ private object StringConstant: ConstantsFinder { // if string constant is called from String class let's pass it as modification if (value.method.declaringClass.name == "java.lang.String") { val stringConstantWasPassedAsArg = unit.useBoxes.findFirstInstanceOf()?.plainValue - if (stringConstantWasPassedAsArg != null) { + if (stringConstantWasPassedAsArg != null && stringConstantWasPassedAsArg is String) { return listOf(FuzzedConcreteValue(stringClassId, stringConstantWasPassedAsArg, FuzzedOp.CH)) } val stringConstantWasPassedAsThis = graph.getPredsOf(unit) @@ -213,7 +213,7 @@ private object StringConstant: ConstantsFinder { ?.useBoxes ?.findFirstInstanceOf() ?.plainValue - if (stringConstantWasPassedAsThis != null) { + if (stringConstantWasPassedAsThis != null && stringConstantWasPassedAsThis is String) { return listOf(FuzzedConcreteValue(stringClassId, stringConstantWasPassedAsThis, FuzzedOp.CH)) } } @@ -250,4 +250,4 @@ private fun sootIfToFuzzedOp(unit: JIfStmt) = when (unit.condition) { else -> FuzzedOp.NONE } -private fun nextDirectUnit(graph: ExceptionalUnitGraph, unit: Unit): Unit? = graph.getSuccsOf(unit).takeIf { it.size == 1 }?.first() \ No newline at end of file +private fun nextDirectUnit(graph: ExceptionalUnitGraph, unit: Unit): Unit? = graph.getSuccsOf(unit).takeIf { it.size == 1 }?.first() diff --git a/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/FuzzedValue.kt b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/FuzzedValue.kt index 3919f111cd..0414fc99fe 100644 --- a/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/FuzzedValue.kt +++ b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/FuzzedValue.kt @@ -8,7 +8,7 @@ import org.utbot.framework.plugin.api.UtModel */ class FuzzedValue( val model: UtModel, - val createdBy: ModelProvider? = null + val createdBy: ModelProvider? = null, ) { /** diff --git a/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/Fuzzer.kt b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/Fuzzer.kt index c0dde7d1c5..35a5c67ca8 100644 --- a/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/Fuzzer.kt +++ b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/Fuzzer.kt @@ -1,18 +1,19 @@ package org.utbot.fuzzer -import org.utbot.fuzzer.providers.ConstantsModelProvider -import org.utbot.fuzzer.providers.ObjectModelProvider -import org.utbot.fuzzer.providers.PrimitivesModelProvider -import org.utbot.fuzzer.providers.StringConstantModelProvider import mu.KotlinLogging +import org.utbot.fuzzer.mutators.NumberRandomMutator +import org.utbot.fuzzer.mutators.StringRandomMutator import org.utbot.fuzzer.providers.ArrayModelProvider import org.utbot.fuzzer.providers.CharToStringModelProvider import org.utbot.fuzzer.providers.CollectionModelProvider -import org.utbot.fuzzer.providers.PrimitiveDefaultsModelProvider +import org.utbot.fuzzer.providers.ConstantsModelProvider import org.utbot.fuzzer.providers.EnumModelProvider +import org.utbot.fuzzer.providers.ObjectModelProvider +import org.utbot.fuzzer.providers.PrimitiveDefaultsModelProvider import org.utbot.fuzzer.providers.PrimitiveWrapperModelProvider -import java.lang.IllegalArgumentException -import java.util.IdentityHashMap +import org.utbot.fuzzer.providers.PrimitivesModelProvider +import org.utbot.fuzzer.providers.StringConstantModelProvider +import java.util.* import java.util.concurrent.atomic.AtomicInteger import kotlin.random.Random @@ -95,6 +96,9 @@ class ReferencePreservingIntIdGenerator(lowerBound: Int = DEFAULT_LOWER_BOUND) : } } +/** + * Generated by fuzzer sequence of values which can be passed into the method. + */ fun fuzz(description: FuzzedMethodDescription, vararg modelProviders: ModelProvider): Sequence> { if (modelProviders.isEmpty()) { throw IllegalArgumentException("At least one model provider is required") @@ -116,6 +120,29 @@ fun fuzz(description: FuzzedMethodDescription, vararg modelProviders: ModelProvi return CartesianProduct(values, Random(0L)).asSequence() } +/** + * Wraps sequence of values, iterates through them and mutates. + * + * Mutation when possible is generated after every value of source sequence and then supplies values until it needed. + * [statistics] should not be updated by this method, but can be changed by caller. + */ +fun Sequence>.withMutations(statistics: FuzzerStatistics, description: FuzzedMethodDescription, vararg mutators: ModelMutator) = sequence { + val fvi = iterator() + val mutatorList = mutators.toList() + val random = Random(0L) + while (fvi.hasNext()) { + // Takes a value that was generated by model providers and submits it + yield(fvi.next()) + // Fuzzing can generate values that don't recover new paths. + // So, fuzzing tries to mutate values on each loop + // if there are too many attempts to find new paths without mutations. + yieldMutated(statistics, description, mutatorList, random) + } + // try mutations if fuzzer tried all combinations if any seeds are available + @Suppress("ControlFlowWithEmptyBody") + while (yieldMutated(statistics, description, mutatorList, random)) {} +} + /** * Creates a model provider from a list of default providers. */ @@ -148,3 +175,52 @@ fun objectModelProviders(idGenerator: IdentityPreservingIdGenerator): Model PrimitiveWrapperModelProvider, ) } + +fun defaultModelMutators(): List = listOf(StringRandomMutator, NumberRandomMutator) + +/** + * Tries to mutate a random value from the seed. + * + * Returns `null` if didn't try to do any mutation. + */ +fun mutateRandomValueOrNull( + statistics: FuzzerStatistics, + description: FuzzedMethodDescription, + mutators: List = defaultModelMutators(), + random: Random = Random, +): List? { + if (mutators.isEmpty()) return null + val values = statistics.takeIf { it.isNotEmpty() }?.randomValues(random) ?: return null + var newValues :MutableList? = null + mutators.asSequence() + .forEach { mut -> + mut.mutate(description, values, random).forEach { (index, value) -> + newValues = (newValues ?: values.toMutableList()) + newValues?.set(index, value) + } + } + return newValues +} + +/** + * Run mutations and yields values into the sequence. + * + * Mutations are supplied infinitely until [repeat] returns true. [repeat] is run before mutation. + * + * @param statistics coverage-based seed + * @param description method description + * @param mutators mutators which are applied to the random value + * @param random instance that is used to choose random index from the [statistics] + */ +suspend fun SequenceScope>.yieldMutated( + statistics: FuzzerStatistics, + description: FuzzedMethodDescription, + mutators: List, + random: Random +) : Boolean { + mutateRandomValueOrNull(statistics, description, mutators, random)?.let { + yield(it) + return true + } + return false +} \ No newline at end of file diff --git a/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/FuzzerStatistics.kt b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/FuzzerStatistics.kt new file mode 100644 index 0000000000..521fa7457d --- /dev/null +++ b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/FuzzerStatistics.kt @@ -0,0 +1,71 @@ +package org.utbot.fuzzer + +import kotlin.math.pow +import kotlin.random.Random + +/** + * Stores information that can be useful for fuzzing such as coverage, run count, etc. + */ +interface FuzzerStatistics { + + val seeds: Collection + + /** + * Returns a random seed to process. + */ + fun randomSeed(random: Random): K? + + fun randomValues(random: Random): List? + + fun executions(seed: K): Int + + operator fun get(seed: K): List? + + fun isEmpty(): Boolean + + fun isNotEmpty(): Boolean { + return !isEmpty() + } +} + +class TrieBasedFuzzerStatistics( + private val values: LinkedHashMap, List> = linkedMapOf() +) : FuzzerStatistics> { + + override val seeds: Collection> + get() = values.keys + + override fun randomSeed(random: Random): Trie.Node? { + return values.keys.elementAtOrNull(randomIndex(random)) + } + + override fun isEmpty(): Boolean { + return values.isEmpty() + } + + override fun isNotEmpty(): Boolean { + return values.isNotEmpty() + } + + override fun randomValues(random: Random): List? { + return values.values.elementAtOrNull(randomIndex(random)) + } + + private fun randomIndex(random: Random): Int { + val frequencies = DoubleArray(values.size).also { f -> + values.keys.forEachIndexed { index, key -> + f[index] = 1 / key.count.toDouble().pow(2) + } + } + return random.chooseOne(frequencies) + } + + override fun get(seed: Trie.Node): List? { + return values[seed] + } + + override fun executions(seed: Trie.Node): Int { + return seed.count + } + +} \ No newline at end of file diff --git a/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/ModelMutator.kt b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/ModelMutator.kt new file mode 100644 index 0000000000..8f9db6f05f --- /dev/null +++ b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/ModelMutator.kt @@ -0,0 +1,46 @@ +package org.utbot.fuzzer + +import org.utbot.framework.plugin.api.UtModel +import kotlin.random.Random + +/** + * Mutates values and returns it. + */ +interface ModelMutator { + + /** + * Mutates values set. + * + * Default implementation iterates through values and delegates to `mutate(FuzzedMethodDescription, Int, Random)`. + */ + fun mutate( + description: FuzzedMethodDescription, + parameters: List, + random: Random, + ) : List { + return parameters + .asSequence() + .mapIndexedNotNull { index, fuzzedValue -> + mutate(description, index, fuzzedValue, random)?.let { mutated -> + FuzzedParameter(index, mutated) + } + } + .toList() + } + + /** + * Mutate a single value if it is possible. + */ + fun mutate( + description: FuzzedMethodDescription, + index: Int, + value: FuzzedValue, + random: Random + ) : FuzzedValue? { + return null + } + + fun UtModel.mutatedFrom(template: FuzzedValue, block: FuzzedValue.() -> Unit = {}): FuzzedValue { + return FuzzedValue(this, template.createdBy).apply(block) + } +} \ No newline at end of file diff --git a/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/RandomExtensions.kt b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/RandomExtensions.kt new file mode 100644 index 0000000000..61e022bb4d --- /dev/null +++ b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/RandomExtensions.kt @@ -0,0 +1,36 @@ +package org.utbot.fuzzer + +import kotlin.random.Random + +/** + * Chooses a random value using frequencies. + * + * If value has greater frequency value then it would be chosen with greater probability. + * + * @return the index of the chosen item. + */ +fun Random.chooseOne(frequencies: DoubleArray): Int { + val total = frequencies.sum() + val value = nextDouble(total) + var nextBound = 0.0 + frequencies.forEachIndexed { index, bound -> + check(bound >= 0) { "Frequency must not be negative" } + nextBound += bound + if (value < nextBound) return index + } + error("Cannot find next index") +} + +/** + * Tries a value. + * + * If a random value is less than [probability] returns true. + */ +fun Random.flipCoin(probability: Int): Boolean { + check(probability in 0 .. 100) { "probability must in range [0, 100] but $probability is provided" } + return nextInt(1, 101) <= probability +} + +fun Long.invertBit(bitIndex: Int): Long { + return this xor (1L shl bitIndex) +} \ No newline at end of file diff --git a/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/mutators/NumberRandomMutator.kt b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/mutators/NumberRandomMutator.kt new file mode 100644 index 0000000000..9634d46112 --- /dev/null +++ b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/mutators/NumberRandomMutator.kt @@ -0,0 +1,60 @@ +package org.utbot.fuzzer.mutators + +import org.utbot.framework.plugin.api.UtPrimitiveModel +import org.utbot.fuzzer.FuzzedMethodDescription +import org.utbot.fuzzer.FuzzedValue +import org.utbot.fuzzer.ModelMutator +import org.utbot.fuzzer.invertBit +import kotlin.random.Random + +/** + * Mutates any [Number] changing random bit. + */ +object NumberRandomMutator : ModelMutator { + + override fun mutate( + description: FuzzedMethodDescription, + index: Int, + value: FuzzedValue, + random: Random + ): FuzzedValue? { + val model = value.model + return if (model is UtPrimitiveModel && model.value is Number) { + val newValue = changeRandomBit(random, model.value as Number) + UtPrimitiveModel(newValue).mutatedFrom(value) { + summary = "%var% = $newValue (mutated from ${model.value})" + } + } else null + } + + private fun changeRandomBit(random: Random, number: Number): Number { + val size = when (number) { + is Byte -> Byte.SIZE_BITS + is Short -> Short.SIZE_BITS + is Int -> Int.SIZE_BITS + is Float -> Float.SIZE_BITS + is Long -> Long.SIZE_BITS + is Double -> Double.SIZE_BITS + else -> error("Unknown type: ${number.javaClass}") + } + val asLong = when (number) { + is Byte, is Short, is Int -> number.toLong() + is Long -> number + is Float -> number.toRawBits().toLong() + is Double -> number.toRawBits() + else -> error("Unknown type: ${number.javaClass}") + } + val bitIndex = random.nextInt(size) + val mutated = asLong.invertBit(bitIndex) + return when (number) { + is Byte -> mutated.toByte() + is Short -> mutated.toShort() + is Int -> mutated.toInt() + is Float -> Float.fromBits(mutated.toInt()) + is Long -> mutated + is Double -> Double.fromBits(mutated) + else -> error("Unknown type: ${number.javaClass}") + } + } +} + diff --git a/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/mutators/StringRandomMutator.kt b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/mutators/StringRandomMutator.kt new file mode 100644 index 0000000000..1b05cf7826 --- /dev/null +++ b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/mutators/StringRandomMutator.kt @@ -0,0 +1,74 @@ +package org.utbot.fuzzer.mutators + +import org.utbot.framework.plugin.api.UtPrimitiveModel +import org.utbot.framework.plugin.api.util.stringClassId +import org.utbot.fuzzer.FuzzedMethodDescription +import org.utbot.fuzzer.FuzzedValue +import org.utbot.fuzzer.ModelMutator +import org.utbot.fuzzer.flipCoin +import kotlin.random.Random + +/** + * Mutates string by adding and/or removal symbol at random position. + */ +object StringRandomMutator : ModelMutator { + + override fun mutate( + description: FuzzedMethodDescription, + index: Int, + value: FuzzedValue, + random: Random + ): FuzzedValue? { + return (value.model as? UtPrimitiveModel) + ?.takeIf { it.classId == stringClassId } + ?.let { model -> + mutate(random, model.value as String).let { + UtPrimitiveModel(it).mutatedFrom(value) { + summary = "%var% = mutated string" + } + } + } + } + + private fun mutate(random: Random, string: String): String { + // we can miss some mutation for a purpose + val position = random.nextInt(string.length + 1) + var result: String = string + if (random.flipCoin(probability = 50)) { + result = tryRemoveChar(random, result, position) ?: string + } + if (random.flipCoin(probability = 50)) { + result = tryAddChar(random, result, position) + } + return result + } + + private fun tryAddChar(random: Random, value: String, position: Int): String { + val charToMutate = if (value.isNotEmpty()) { + value.random(random) + } else { + // use any meaningful character from the ascii table + random.nextInt(33, 127).toChar() + } + return buildString { + append(value.substring(0, position)) + // try to change char to some that is close enough to origin char + val charTableSpread = 64 + if (random.nextBoolean()) { + append(charToMutate - random.nextInt(1, charTableSpread)) + } else { + append(charToMutate + random.nextInt(1, charTableSpread)) + } + append(value.substring(position, value.length)) + } + } + + private fun tryRemoveChar(random: Random, value: String, position: Int): String? { + if (position >= value.length) return null + val toRemove = random.nextInt(value.length) + return buildString { + append(value.substring(0, toRemove)) + append(value.substring(toRemove + 1, value.length)) + } + } +} \ No newline at end of file diff --git a/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/providers/StringConstantModelProvider.kt b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/providers/StringConstantModelProvider.kt index cc2e22dba4..1f9a47238d 100644 --- a/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/providers/StringConstantModelProvider.kt +++ b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/providers/StringConstantModelProvider.kt @@ -1,48 +1,33 @@ package org.utbot.fuzzer.providers import org.utbot.framework.plugin.api.UtPrimitiveModel +import org.utbot.framework.plugin.api.util.charClassId import org.utbot.framework.plugin.api.util.stringClassId import org.utbot.fuzzer.FuzzedMethodDescription -import org.utbot.fuzzer.FuzzedOp import org.utbot.fuzzer.FuzzedParameter import org.utbot.fuzzer.ModelProvider +import org.utbot.fuzzer.ModelProvider.Companion.yieldAllValues import org.utbot.fuzzer.ModelProvider.Companion.yieldValue -import kotlin.random.Random object StringConstantModelProvider : ModelProvider { override fun generate(description: FuzzedMethodDescription): Sequence = sequence { - val random = Random(72923L) description.concreteValues .asSequence() .filter { (classId, _) -> classId == stringClassId } - .forEach { (_, value, op) -> - listOf(value, mutate(random, value as? String, op)) - .asSequence() - .filterNotNull() - .map { UtPrimitiveModel(it) }.forEach { model -> - description.parametersMap.getOrElse(model.classId) { emptyList() }.forEach { index -> - yieldValue(index, model.fuzzed { summary = "%var% = string" }) - } - } + .forEach { (_, value, _) -> + description.parametersMap.getOrElse(stringClassId) { emptyList() }.forEach { index -> + yieldValue(index, UtPrimitiveModel(value).fuzzed { summary = "%var% = string" }) + } } - } - - private fun mutate(random: Random, value: String?, op: FuzzedOp): String? { - if (value == null || value.isEmpty() || op != FuzzedOp.CH) return null - val indexOfMutation = random.nextInt(value.length) - return value.replaceRange(indexOfMutation, indexOfMutation + 1, SingleCharacterSequence(value[indexOfMutation] - random.nextInt(1, 128))) - } - - private class SingleCharacterSequence(private val character: Char) : CharSequence { - override val length: Int - get() = 1 - - override fun get(index: Int): Char = character - - override fun subSequence(startIndex: Int, endIndex: Int): CharSequence { - throw UnsupportedOperationException() - } - + val charsAsStrings = description.concreteValues + .asSequence() + .filter { (classId, _) -> classId == charClassId } + .map { (_, value, _) -> + UtPrimitiveModel((value as Char).toString()).fuzzed { + summary = "%var% = $value" + } + } + yieldAllValues(description.parametersMap.getOrElse(stringClassId) { emptyList() }, charsAsStrings) } } \ No newline at end of file diff --git a/utbot-fuzzers/src/test/kotlin/org/utbot/framework/plugin/api/ModelMutatorTest.kt b/utbot-fuzzers/src/test/kotlin/org/utbot/framework/plugin/api/ModelMutatorTest.kt new file mode 100644 index 0000000000..35bc58d284 --- /dev/null +++ b/utbot-fuzzers/src/test/kotlin/org/utbot/framework/plugin/api/ModelMutatorTest.kt @@ -0,0 +1,37 @@ +package org.utbot.framework.plugin.api + +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.Test +import org.utbot.fuzzer.invertBit +import kotlin.random.Random + +class ModelMutatorTest { + + @Test + fun `invert bit works for long`() { + var attempts = 100_000 + val random = Random(2210) + sequence { + while (true) { + yield(random.nextLong()) + } + }.forEach { value -> + if (attempts-- <= 0) { return } + for (bit in 0 until Long.SIZE_BITS) { + val newValue = value.invertBit(bit) + val oldBinary = value.toBinaryString() + val newBinary = newValue.toBinaryString() + assertEquals(oldBinary.length, newBinary.length) + for (test in Long.SIZE_BITS - 1 downTo 0) { + if (test != Long.SIZE_BITS - 1 - bit) { + assertEquals(oldBinary[test], newBinary[test]) { "$oldBinary : $newBinary for value $value" } + } else { + assertNotEquals(oldBinary[test], newBinary[test]) { "$oldBinary : $newBinary for value $value" } + } + } + } + } + } + + private fun Long.toBinaryString() = java.lang.Long.toBinaryString(this).padStart(Long.SIZE_BITS, '0') +} \ No newline at end of file diff --git a/utbot-fuzzers/src/test/kotlin/org/utbot/framework/plugin/api/ModelProviderTest.kt b/utbot-fuzzers/src/test/kotlin/org/utbot/framework/plugin/api/ModelProviderTest.kt index 782ce6ee38..dbe3225fde 100644 --- a/utbot-fuzzers/src/test/kotlin/org/utbot/framework/plugin/api/ModelProviderTest.kt +++ b/utbot-fuzzers/src/test/kotlin/org/utbot/framework/plugin/api/ModelProviderTest.kt @@ -160,27 +160,12 @@ class ModelProviderTest { ) assertEquals(1, models.size) - assertEquals(2, models[0]!!.size) - listOf("nonemptystring", "nonemptystr`ng").forEach { + assertEquals(1, models[0]!!.size) + listOf("nonemptystring").forEach { assertTrue( models[0]!!.contains(UtPrimitiveModel(it))) { "Failed to find string $it in list ${models[0]}" } } } - @Test - fun `test mutation creates the same values between different runs`() { - repeat(10) { - val models = collect(StringConstantModelProvider, - parameters = listOf(stringClassId), - constants = listOf( - FuzzedConcreteValue(stringClassId, "anotherstring", FuzzedOp.CH), - ) - ) - listOf("anotherstring", "anotherskring").forEach { - assertTrue( models[0]!!.contains(UtPrimitiveModel(it))) { "Failed to find string $it in list ${models[0]}" } - } - } - } - @Test @Suppress("unused", "UNUSED_PARAMETER", "RemoveEmptySecondaryConstructorBody") fun `test default object model creation for simple constructors`() { diff --git a/utbot-fuzzers/src/test/kotlin/org/utbot/framework/plugin/api/RandomExtensionsTest.kt b/utbot-fuzzers/src/test/kotlin/org/utbot/framework/plugin/api/RandomExtensionsTest.kt new file mode 100644 index 0000000000..764d13219d --- /dev/null +++ b/utbot-fuzzers/src/test/kotlin/org/utbot/framework/plugin/api/RandomExtensionsTest.kt @@ -0,0 +1,74 @@ +package org.utbot.framework.plugin.api + +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.Test +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.ValueSource +import org.utbot.fuzzer.chooseOne +import org.utbot.fuzzer.flipCoin +import kotlin.math.abs +import kotlin.random.Random + +class RandomExtensionsTest { + + @Test + fun `default implementation always returns 0`() { + val frequencies = doubleArrayOf(1.0) + (0 until 1000).forEach { _ -> + assertEquals(0, Random.chooseOne(frequencies)) + } + } + + @ParameterizedTest(name = "seed{arguments}") + @ValueSource(ints = [0, 100, -123, 99999, 84]) + fun `with default forEach function frequencies is equal`(seed: Int) { + val random = Random(seed) + val frequencies = doubleArrayOf(10.0, 20.0, 30.0, 40.0) + val result = IntArray(frequencies.size) + assertEquals(100.0, frequencies.sum()) { "In this test every frequency value represents a percent. The sum must be equal to 100" } + val tries = 100_000 + val errors = tries / 100 // 1% + (0 until tries).forEach { _ -> + result[random.chooseOne(frequencies)]++ + } + val expected = frequencies.map { tries * it / 100} + result.forEachIndexed { index, value -> + assertTrue(abs(expected[index] - value) < errors) { + "The error should not extent $errors for $tries cases, but ${expected[index]} and $value too far" + } + } + } + + @Test + fun `inverting probabilities from the documentation`() { + val frequencies = doubleArrayOf(20.0, 80.0) + val random = Random(0) + val result = IntArray(frequencies.size) + val tries = 10_000 + val errors = tries / 100 // 1% + (0 until tries).forEach { _ -> + result[random.chooseOne(DoubleArray(frequencies.size) { 100.0 - frequencies[it] })]++ + } + result.forEachIndexed { index, value -> + val expected = frequencies[frequencies.size - 1 - index] * errors + assertTrue(abs(value - expected) < errors) { + "The error should not extent 100 for 10 000 cases, but $expected and $value too far" + } + } + } + + @Test + fun `flip the coin is fair enough`() { + val random = Random(0) + var result = 0 + val probability = 20 + val experiments = 1_000_000 + for (i in 0 until experiments) { + if (random.flipCoin(probability)) { + result++ + } + } + val error = experiments / 1000 // 0.1 % + assertTrue(abs(result - experiments * probability / 100) < error) + } +} \ No newline at end of file