Skip to content

Commit 1c44797

Browse files
MarkoutteMarkoutte
authored
Mutate values to improve code coverage #213 (#713)
1 parent c8018a5 commit 1c44797

File tree

13 files changed

+523
-66
lines changed

13 files changed

+523
-66
lines changed

‎utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -88,9 +88,13 @@ import org.utbot.fuzzer.FuzzedValue
8888
import org.utbot.fuzzer.ModelProvider
8989
import org.utbot.fuzzer.ReferencePreservingIntIdGenerator
9090
import org.utbot.fuzzer.Trie
91+
import org.utbot.fuzzer.TrieBasedFuzzerStatistics
9192
import org.utbot.fuzzer.UtFuzzedExecution
93+
import org.utbot.fuzzer.withMutations
9294
import org.utbot.fuzzer.collectConstantsForFuzzer
9395
import org.utbot.fuzzer.defaultModelProviders
96+
import org.utbot.fuzzer.defaultModelMutators
97+
import org.utbot.fuzzer.flipCoin
9498
import org.utbot.fuzzer.fuzz
9599
import org.utbot.fuzzer.providers.ObjectModelProvider
96100
import org.utbot.instrumentation.ConcreteExecutor
@@ -392,6 +396,7 @@ class UtBotSymbolicEngine(
392396
val fallbackModelProvider = FallbackModelProvider(defaultIdGenerator)
393397
val constantValues = collectConstantsForFuzzer(graph)
394398

399+
val random = Random(0)
395400
val thisInstance = when {
396401
methodUnderTest.isStatic -> null
397402
methodUnderTest.isConstructor -> if (
@@ -405,7 +410,7 @@ class UtBotSymbolicEngine(
405410
else -> {
406411
ObjectModelProvider(defaultIdGenerator).withFallback(fallbackModelProvider).generate(
407412
FuzzedMethodDescription("thisInstance", voidClassId, listOf(methodUnderTest.clazz.id), constantValues)
408-
).take(10).shuffled(Random(0)).map { it.value.model }.first().apply {
413+
).take(10).shuffled(random).map { it.value.model }.first().apply {
409414
if (this is UtNullModel) { // it will definitely fail because of NPE,
410415
return@flow
411416
}
@@ -421,8 +426,9 @@ class UtBotSymbolicEngine(
421426
parameterNameMap = { index -> names?.getOrNull(index) }
422427
}
423428
val coveredInstructionTracker = Trie(Instruction::id)
424-
val coveredInstructionValues = mutableMapOf<Trie.Node<Instruction>, List<FuzzedValue>>()
425-
var attempts = UtSettings.fuzzingMaxAttempts
429+
val coveredInstructionValues = linkedMapOf<Trie.Node<Instruction>, List<FuzzedValue>>()
430+
var attempts = 0
431+
val attemptsLimit = UtSettings.fuzzingMaxAttempts
426432
val hasMethodUnderTestParametersToFuzz = executableId.parameters.isNotEmpty()
427433
val fuzzedValues = if (hasMethodUnderTestParametersToFuzz) {
428434
fuzz(methodUnderTestDescription, modelProvider(defaultModelProviders(defaultIdGenerator)))
@@ -435,7 +441,9 @@ class UtBotSymbolicEngine(
435441
fuzz(thisMethodDescription, ObjectModelProvider(defaultIdGenerator).apply {
436442
limitValuesCreatedByFieldAccessors = 500
437443
})
438-
}
444+
}.withMutations(
445+
TrieBasedFuzzerStatistics(coveredInstructionValues), methodUnderTestDescription, *defaultModelMutators().toTypedArray()
446+
)
439447
fuzzedValues.forEach { values ->
440448
if (controller.job?.isActive == false || System.currentTimeMillis() >= until) {
441449
logger.info { "Fuzzing overtime: $methodUnderTest" }
@@ -473,14 +481,19 @@ class UtBotSymbolicEngine(
473481

474482
val coveredInstructions = concreteExecutionResult.coverage.coveredInstructions
475483
if (coveredInstructions.isNotEmpty()) {
476-
val count = coveredInstructionTracker.add(coveredInstructions)
477-
if (count.count > 1) {
478-
if (--attempts < 0) {
484+
val coverageKey = coveredInstructionTracker.add(coveredInstructions)
485+
if (coverageKey.count > 1) {
486+
if (++attempts >= attemptsLimit) {
479487
return@flow
480488
}
489+
// Update the seeded values sometimes
490+
// This is necessary because some values cannot do a good values in mutation in any case
491+
if (random.flipCoin(probability = 50)) {
492+
coveredInstructionValues[coverageKey] = values
493+
}
481494
return@forEach
482495
}
483-
coveredInstructionValues[count] = values
496+
coveredInstructionValues[coverageKey] = values
484497
} else {
485498
logger.error { "Coverage is empty for $methodUnderTest with ${values.map { it.model }}" }
486499
}

‎utbot-framework/src/main/kotlin/org/utbot/fuzzer/FuzzerFunctions.kt

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -204,7 +204,7 @@ private object StringConstant: ConstantsFinder {
204204
// if string constant is called from String class let's pass it as modification
205205
if (value.method.declaringClass.name == "java.lang.String") {
206206
val stringConstantWasPassedAsArg = unit.useBoxes.findFirstInstanceOf<Constant>()?.plainValue
207-
if (stringConstantWasPassedAsArg != null) {
207+
if (stringConstantWasPassedAsArg != null && stringConstantWasPassedAsArg is String) {
208208
return listOf(FuzzedConcreteValue(stringClassId, stringConstantWasPassedAsArg, FuzzedOp.CH))
209209
}
210210
val stringConstantWasPassedAsThis = graph.getPredsOf(unit)
@@ -213,7 +213,7 @@ private object StringConstant: ConstantsFinder {
213213
?.useBoxes
214214
?.findFirstInstanceOf<Constant>()
215215
?.plainValue
216-
if (stringConstantWasPassedAsThis != null) {
216+
if (stringConstantWasPassedAsThis != null && stringConstantWasPassedAsThis is String) {
217217
return listOf(FuzzedConcreteValue(stringClassId, stringConstantWasPassedAsThis, FuzzedOp.CH))
218218
}
219219
}
@@ -250,4 +250,4 @@ private fun sootIfToFuzzedOp(unit: JIfStmt) = when (unit.condition) {
250250
else -> FuzzedOp.NONE
251251
}
252252

253-
private fun nextDirectUnit(graph: ExceptionalUnitGraph, unit: Unit): Unit? = graph.getSuccsOf(unit).takeIf { it.size == 1 }?.first()
253+
private fun nextDirectUnit(graph: ExceptionalUnitGraph, unit: Unit): Unit? = graph.getSuccsOf(unit).takeIf { it.size == 1 }?.first()

‎utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/FuzzedValue.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import org.utbot.framework.plugin.api.UtModel
88
*/
99
class FuzzedValue(
1010
val model: UtModel,
11-
val createdBy: ModelProvider? = null
11+
val createdBy: ModelProvider? = null,
1212
) {
1313

1414
/**

‎utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/Fuzzer.kt

Lines changed: 83 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,19 @@
11
package org.utbot.fuzzer
22

3-
import org.utbot.fuzzer.providers.ConstantsModelProvider
4-
import org.utbot.fuzzer.providers.ObjectModelProvider
5-
import org.utbot.fuzzer.providers.PrimitivesModelProvider
6-
import org.utbot.fuzzer.providers.StringConstantModelProvider
73
import mu.KotlinLogging
4+
import org.utbot.fuzzer.mutators.NumberRandomMutator
5+
import org.utbot.fuzzer.mutators.StringRandomMutator
86
import org.utbot.fuzzer.providers.ArrayModelProvider
97
import org.utbot.fuzzer.providers.CharToStringModelProvider
108
import org.utbot.fuzzer.providers.CollectionModelProvider
11-
import org.utbot.fuzzer.providers.PrimitiveDefaultsModelProvider
9+
import org.utbot.fuzzer.providers.ConstantsModelProvider
1210
import org.utbot.fuzzer.providers.EnumModelProvider
11+
import org.utbot.fuzzer.providers.ObjectModelProvider
12+
import org.utbot.fuzzer.providers.PrimitiveDefaultsModelProvider
1313
import org.utbot.fuzzer.providers.PrimitiveWrapperModelProvider
14-
import java.lang.IllegalArgumentException
15-
import java.util.IdentityHashMap
14+
import org.utbot.fuzzer.providers.PrimitivesModelProvider
15+
import org.utbot.fuzzer.providers.StringConstantModelProvider
16+
import java.util.*
1617
import java.util.concurrent.atomic.AtomicInteger
1718
import kotlin.random.Random
1819

@@ -95,6 +96,9 @@ class ReferencePreservingIntIdGenerator(lowerBound: Int = DEFAULT_LOWER_BOUND) :
9596
}
9697
}
9798

99+
/**
100+
* Generated by fuzzer sequence of values which can be passed into the method.
101+
*/
98102
fun fuzz(description: FuzzedMethodDescription, vararg modelProviders: ModelProvider): Sequence<List<FuzzedValue>> {
99103
if (modelProviders.isEmpty()) {
100104
throw IllegalArgumentException("At least one model provider is required")
@@ -116,6 +120,29 @@ fun fuzz(description: FuzzedMethodDescription, vararg modelProviders: ModelProvi
116120
return CartesianProduct(values, Random(0L)).asSequence()
117121
}
118122

123+
/**
124+
* Wraps sequence of values, iterates through them and mutates.
125+
*
126+
* Mutation when possible is generated after every value of source sequence and then supplies values until it needed.
127+
* [statistics] should not be updated by this method, but can be changed by caller.
128+
*/
129+
fun <T> Sequence<List<FuzzedValue>>.withMutations(statistics: FuzzerStatistics<T>, description: FuzzedMethodDescription, vararg mutators: ModelMutator) = sequence {
130+
val fvi = iterator()
131+
val mutatorList = mutators.toList()
132+
val random = Random(0L)
133+
while (fvi.hasNext()) {
134+
// Takes a value that was generated by model providers and submits it
135+
yield(fvi.next())
136+
// Fuzzing can generate values that don't recover new paths.
137+
// So, fuzzing tries to mutate values on each loop
138+
// if there are too many attempts to find new paths without mutations.
139+
yieldMutated(statistics, description, mutatorList, random)
140+
}
141+
// try mutations if fuzzer tried all combinations if any seeds are available
142+
@Suppress("ControlFlowWithEmptyBody")
143+
while (yieldMutated(statistics, description, mutatorList, random)) {}
144+
}
145+
119146
/**
120147
* Creates a model provider from a list of default providers.
121148
*/
@@ -148,3 +175,52 @@ fun objectModelProviders(idGenerator: IdentityPreservingIdGenerator<Int>): Model
148175
PrimitiveWrapperModelProvider,
149176
)
150177
}
178+
179+
fun defaultModelMutators(): List<ModelMutator> = listOf(StringRandomMutator, NumberRandomMutator)
180+
181+
/**
182+
* Tries to mutate a random value from the seed.
183+
*
184+
* Returns `null` if didn't try to do any mutation.
185+
*/
186+
fun <T> mutateRandomValueOrNull(
187+
statistics: FuzzerStatistics<T>,
188+
description: FuzzedMethodDescription,
189+
mutators: List<ModelMutator> = defaultModelMutators(),
190+
random: Random = Random,
191+
): List<FuzzedValue>? {
192+
if (mutators.isEmpty()) return null
193+
val values = statistics.takeIf { it.isNotEmpty() }?.randomValues(random) ?: return null
194+
var newValues :MutableList<FuzzedValue>? = null
195+
mutators.asSequence()
196+
.forEach { mut ->
197+
mut.mutate(description, values, random).forEach { (index, value) ->
198+
newValues = (newValues ?: values.toMutableList())
199+
newValues?.set(index, value)
200+
}
201+
}
202+
return newValues
203+
}
204+
205+
/**
206+
* Run mutations and yields values into the sequence.
207+
*
208+
* Mutations are supplied infinitely until [repeat] returns true. [repeat] is run before mutation.
209+
*
210+
* @param statistics coverage-based seed
211+
* @param description method description
212+
* @param mutators mutators which are applied to the random value
213+
* @param random instance that is used to choose random index from the [statistics]
214+
*/
215+
suspend fun <T> SequenceScope<List<FuzzedValue>>.yieldMutated(
216+
statistics: FuzzerStatistics<T>,
217+
description: FuzzedMethodDescription,
218+
mutators: List<ModelMutator>,
219+
random: Random
220+
) : Boolean {
221+
mutateRandomValueOrNull(statistics, description, mutators, random)?.let {
222+
yield(it)
223+
return true
224+
}
225+
return false
226+
}

‎utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/FuzzerStatistics.kt

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
package org.utbot.fuzzer
2+
3+
import kotlin.math.pow
4+
import kotlin.random.Random
5+
6+
/**
7+
* Stores information that can be useful for fuzzing such as coverage, run count, etc.
8+
*/
9+
interface FuzzerStatistics<K> {
10+
11+
val seeds: Collection<K>
12+
13+
/**
14+
* Returns a random seed to process.
15+
*/
16+
fun randomSeed(random: Random): K?
17+
18+
fun randomValues(random: Random): List<FuzzedValue>?
19+
20+
fun executions(seed: K): Int
21+
22+
operator fun get(seed: K): List<FuzzedValue>?
23+
24+
fun isEmpty(): Boolean
25+
26+
fun isNotEmpty(): Boolean {
27+
return !isEmpty()
28+
}
29+
}
30+
31+
class TrieBasedFuzzerStatistics<V>(
32+
private val values: LinkedHashMap<Trie.Node<V>, List<FuzzedValue>> = linkedMapOf()
33+
) : FuzzerStatistics<Trie.Node<V>> {
34+
35+
override val seeds: Collection<Trie.Node<V>>
36+
get() = values.keys
37+
38+
override fun randomSeed(random: Random): Trie.Node<V>? {
39+
return values.keys.elementAtOrNull(randomIndex(random))
40+
}
41+
42+
override fun isEmpty(): Boolean {
43+
return values.isEmpty()
44+
}
45+
46+
override fun isNotEmpty(): Boolean {
47+
return values.isNotEmpty()
48+
}
49+
50+
override fun randomValues(random: Random): List<FuzzedValue>? {
51+
return values.values.elementAtOrNull(randomIndex(random))
52+
}
53+
54+
private fun randomIndex(random: Random): Int {
55+
val frequencies = DoubleArray(values.size).also { f ->
56+
values.keys.forEachIndexed { index, key ->
57+
f[index] = 1 / key.count.toDouble().pow(2)
58+
}
59+
}
60+
return random.chooseOne(frequencies)
61+
}
62+
63+
override fun get(seed: Trie.Node<V>): List<FuzzedValue>? {
64+
return values[seed]
65+
}
66+
67+
override fun executions(seed: Trie.Node<V>): Int {
68+
return seed.count
69+
}
70+
71+
}

‎utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/ModelMutator.kt

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
package org.utbot.fuzzer
2+
3+
import org.utbot.framework.plugin.api.UtModel
4+
import kotlin.random.Random
5+
6+
/**
7+
* Mutates values and returns it.
8+
*/
9+
interface ModelMutator {
10+
11+
/**
12+
* Mutates values set.
13+
*
14+
* Default implementation iterates through values and delegates to `mutate(FuzzedMethodDescription, Int, Random)`.
15+
*/
16+
fun mutate(
17+
description: FuzzedMethodDescription,
18+
parameters: List<FuzzedValue>,
19+
random: Random,
20+
) : List<FuzzedParameter> {
21+
return parameters
22+
.asSequence()
23+
.mapIndexedNotNull { index, fuzzedValue ->
24+
mutate(description, index, fuzzedValue, random)?.let { mutated ->
25+
FuzzedParameter(index, mutated)
26+
}
27+
}
28+
.toList()
29+
}
30+
31+
/**
32+
* Mutate a single value if it is possible.
33+
*/
34+
fun mutate(
35+
description: FuzzedMethodDescription,
36+
index: Int,
37+
value: FuzzedValue,
38+
random: Random
39+
) : FuzzedValue? {
40+
return null
41+
}
42+
43+
fun UtModel.mutatedFrom(template: FuzzedValue, block: FuzzedValue.() -> Unit = {}): FuzzedValue {
44+
return FuzzedValue(this, template.createdBy).apply(block)
45+
}
46+
}

‎utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/RandomExtensions.kt

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
package org.utbot.fuzzer
2+
3+
import kotlin.random.Random
4+
5+
/**
6+
* Chooses a random value using frequencies.
7+
*
8+
* If value has greater frequency value then it would be chosen with greater probability.
9+
*
10+
* @return the index of the chosen item.
11+
*/
12+
fun Random.chooseOne(frequencies: DoubleArray): Int {
13+
val total = frequencies.sum()
14+
val value = nextDouble(total)
15+
var nextBound = 0.0
16+
frequencies.forEachIndexed { index, bound ->
17+
check(bound >= 0) { "Frequency must not be negative" }
18+
nextBound += bound
19+
if (value < nextBound) return index
20+
}
21+
error("Cannot find next index")
22+
}
23+
24+
/**
25+
* Tries a value.
26+
*
27+
* If a random value is less than [probability] returns true.
28+
*/
29+
fun Random.flipCoin(probability: Int): Boolean {
30+
check(probability in 0 .. 100) { "probability must in range [0, 100] but $probability is provided" }
31+
return nextInt(1, 101) <= probability
32+
}
33+
34+
fun Long.invertBit(bitIndex: Int): Long {
35+
return this xor (1L shl bitIndex)
36+
}

0 commit comments

Comments
 (0)