Skip to content

Commit 48d6e4b

Browse files
MarkoutteMarkoutte
committed
Mutate values to improve code coverage #213 
1 parent 8b12b88 commit 48d6e4b

File tree

11 files changed

+400
-48
lines changed

11 files changed

+400
-48
lines changed

‎utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -276,6 +276,13 @@ object UtSettings {
276276
*/
277277
var fuzzingMaxAttempts: Int by getIntProperty(Int.MAX_VALUE)
278278

279+
/**
280+
* Fuzzing tries to mutate values using this factor.
281+
*
282+
* If any mutation is successful then counter is reset.
283+
*/
284+
var fuzzingRandomMutationsFactor: Int by getIntProperty(10_000)
285+
279286
/**
280287
* Fuzzer tries to generate and run tests during this time.
281288
*/

‎utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt

Lines changed: 42 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ import org.utbot.framework.plugin.api.util.description
8383
import org.utbot.framework.util.jimpleBody
8484
import org.utbot.framework.plugin.api.util.voidClassId
8585
import org.utbot.fuzzer.FallbackModelProvider
86+
import org.utbot.fuzzer.FrequencyRandom
8687
import org.utbot.fuzzer.FuzzedMethodDescription
8788
import org.utbot.fuzzer.FuzzedValue
8889
import org.utbot.fuzzer.ModelProvider
@@ -91,6 +92,7 @@ import org.utbot.fuzzer.Trie
9192
import org.utbot.fuzzer.UtFuzzedExecution
9293
import org.utbot.fuzzer.collectConstantsForFuzzer
9394
import org.utbot.fuzzer.defaultModelProviders
95+
import org.utbot.fuzzer.findNextMutatedValue
9496
import org.utbot.fuzzer.fuzz
9597
import org.utbot.fuzzer.providers.ObjectModelProvider
9698
import org.utbot.instrumentation.ConcreteExecutor
@@ -421,8 +423,9 @@ class UtBotSymbolicEngine(
421423
parameterNameMap = { index -> names?.getOrNull(index) }
422424
}
423425
val coveredInstructionTracker = Trie(Instruction::id)
424-
val coveredInstructionValues = mutableMapOf<Trie.Node<Instruction>, List<FuzzedValue>>()
425-
var attempts = UtSettings.fuzzingMaxAttempts
426+
val coveredInstructionValues = linkedMapOf<Trie.Node<Instruction>, List<FuzzedValue>>()
427+
var attempts = 0
428+
val attemptsLimit = UtSettings.fuzzingMaxAttempts
426429
val hasMethodUnderTestParametersToFuzz = executableId.parameters.isNotEmpty()
427430
val fuzzedValues = if (hasMethodUnderTestParametersToFuzz) {
428431
fuzz(methodUnderTestDescription, modelProvider(defaultModelProviders(defaultIdGenerator)))
@@ -436,7 +439,34 @@ class UtBotSymbolicEngine(
436439
limitValuesCreatedByFieldAccessors = 500
437440
})
438441
}
439-
fuzzedValues.forEach { values ->
442+
443+
val frequencyRandom = FrequencyRandom(Random(221))
444+
val factor = maxOf(0, UtSettings.fuzzingRandomMutationsFactor)
445+
val fuzzedValueSequence: Sequence<List<FuzzedValue>> = sequence {
446+
val fvi = fuzzedValues.iterator()
447+
while (fvi.hasNext()) {
448+
yield(fvi.next())
449+
// if we stuck switch to "next + several mutations" mode
450+
if ((attempts + 1) % (factor / 100) == 0 && coveredInstructionValues.isNotEmpty()) {
451+
for (i in 0 until (factor / 100)) {
452+
findNextMutatedValue(frequencyRandom, coveredInstructionValues, methodUnderTestDescription)?.let { yield(it) }
453+
}
454+
}
455+
}
456+
// try mutations if fuzzer tried all combinations
457+
var tryLocal = factor
458+
while (tryLocal-- >= 0) {
459+
val value = findNextMutatedValue(frequencyRandom, coveredInstructionValues, methodUnderTestDescription)
460+
if (value != null) {
461+
val before = coveredInstructionValues.size
462+
yield(value)
463+
if (coveredInstructionValues.size != before) {
464+
tryLocal = factor
465+
}
466+
}
467+
}
468+
}
469+
fuzzedValueSequence.forEach { values ->
440470
if (controller.job?.isActive == false || System.currentTimeMillis() >= until) {
441471
logger.info { "Fuzzing overtime: $methodUnderTest" }
442472
return@flow
@@ -473,14 +503,19 @@ class UtBotSymbolicEngine(
473503

474504
val coveredInstructions = concreteExecutionResult.coverage.coveredInstructions
475505
if (coveredInstructions.isNotEmpty()) {
476-
val count = coveredInstructionTracker.add(coveredInstructions)
477-
if (count.count > 1) {
478-
if (--attempts < 0) {
506+
val coverageKey = coveredInstructionTracker.add(coveredInstructions)
507+
if (coverageKey.count > 1) {
508+
if (++attempts >= attemptsLimit) {
479509
return@flow
480510
}
511+
// Update the seeded values sometimes
512+
// This is necessary because some values cannot do a good values in mutation in any case
513+
if (frequencyRandom.random.nextInt(1, 101) <= 50) {
514+
coveredInstructionValues[coverageKey] = values
515+
}
481516
return@forEach
482517
}
483-
coveredInstructionValues[count] = values
518+
coveredInstructionValues[coverageKey] = values
484519
} else {
485520
logger.error { "Coverage is empty for $methodUnderTest with ${values.map { it.model }}" }
486521
}

‎utbot-framework/src/main/kotlin/org/utbot/fuzzer/FuzzerFunctions.kt

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import org.utbot.framework.plugin.api.util.longClassId
1111
import org.utbot.framework.plugin.api.util.shortClassId
1212
import org.utbot.framework.plugin.api.util.stringClassId
1313
import mu.KotlinLogging
14+
import org.utbot.framework.plugin.api.Instruction
1415
import soot.BooleanType
1516
import soot.ByteType
1617
import soot.CharType
@@ -42,6 +43,7 @@ import soot.jimple.internal.JNeExpr
4243
import soot.jimple.internal.JTableSwitchStmt
4344
import soot.jimple.internal.JVirtualInvokeExpr
4445
import soot.toolkits.graph.ExceptionalUnitGraph
46+
import kotlin.math.pow
4547

4648
private val logger = KotlinLogging.logger {}
4749

@@ -204,7 +206,7 @@ private object StringConstant: ConstantsFinder {
204206
// if string constant is called from String class let's pass it as modification
205207
if (value.method.declaringClass.name == "java.lang.String") {
206208
val stringConstantWasPassedAsArg = unit.useBoxes.findFirstInstanceOf<Constant>()?.plainValue
207-
if (stringConstantWasPassedAsArg != null) {
209+
if (stringConstantWasPassedAsArg != null && stringConstantWasPassedAsArg is String) {
208210
return listOf(FuzzedConcreteValue(stringClassId, stringConstantWasPassedAsArg, FuzzedOp.CH))
209211
}
210212
val stringConstantWasPassedAsThis = graph.getPredsOf(unit)
@@ -213,7 +215,7 @@ private object StringConstant: ConstantsFinder {
213215
?.useBoxes
214216
?.findFirstInstanceOf<Constant>()
215217
?.plainValue
216-
if (stringConstantWasPassedAsThis != null) {
218+
if (stringConstantWasPassedAsThis != null && stringConstantWasPassedAsThis is String) {
217219
return listOf(FuzzedConcreteValue(stringClassId, stringConstantWasPassedAsThis, FuzzedOp.CH))
218220
}
219221
}
@@ -250,4 +252,23 @@ private fun sootIfToFuzzedOp(unit: JIfStmt) = when (unit.condition) {
250252
else -> FuzzedOp.NONE
251253
}
252254

253-
private fun nextDirectUnit(graph: ExceptionalUnitGraph, unit: Unit): Unit? = graph.getSuccsOf(unit).takeIf { it.size == 1 }?.first()
255+
private fun nextDirectUnit(graph: ExceptionalUnitGraph, unit: Unit): Unit? = graph.getSuccsOf(unit).takeIf { it.size == 1 }?.first()
256+
257+
fun findNextMutatedValue(
258+
frequencyRandom: FrequencyRandom,
259+
coverage: LinkedHashMap<Trie.Node<Instruction>, List<FuzzedValue>>,
260+
description: FuzzedMethodDescription,
261+
): List<FuzzedValue>? {
262+
if (coverage.isEmpty()) return null
263+
frequencyRandom.prepare(coverage.map { it.key.count }) { 1 / it.toDouble().pow(2) }
264+
val values = coverage.values.drop(frequencyRandom.nextIndex()).firstOrNull()
265+
if (values != null) {
266+
val mutatedValue = defaultMutators().fold(values) { v, mut ->
267+
mut.mutate(description, v, frequencyRandom.random)
268+
}
269+
if (mutatedValue != values) {
270+
return mutatedValue
271+
}
272+
}
273+
return null
274+
}

‎utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/FrequencyRandom.kt

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
package org.utbot.fuzzer
2+
3+
import kotlin.random.Random
4+
5+
/**
6+
* Frequency random returns [nextIndex] with given probabilities.
7+
*/
8+
class FrequencyRandom(var random: Random = Random) {
9+
private var total = 1.0
10+
private var bound: DoubleArray = DoubleArray(1) { 1.0 }
11+
val size: Int
12+
get() = bound.size
13+
14+
/**
15+
* Updates internals to generate values using [frequencies] and custom function [forEach].
16+
*
17+
* For example, it is possible implement logic where the source list of [frequencies] is inverted.
18+
*
19+
* ```
20+
* prepare(listOf(20., 80.), forEach = { 100 - it })
21+
*
22+
* In this case second value (that has 80.)in the [frequencies] list will be found in 20% of all cases.
23+
*/
24+
fun <T : Number> prepare(frequencies: List<T>, forEach: (T) -> Double = { it.toDouble() }) {
25+
bound = DoubleArray(frequencies.size) { forEach(frequencies[it]) }
26+
for (i in 1 until bound.size) {
27+
bound[i] = bound[i] + bound[i - 1]
28+
}
29+
total = if (bound.isEmpty()) 0.0 else bound.last()
30+
}
31+
32+
fun nextIndex(): Int {
33+
val value = random.nextDouble(total)
34+
for (index in bound.indices) {
35+
if (value < bound[index]) {
36+
return index
37+
}
38+
}
39+
error("Cannot find next index")
40+
}
41+
}

‎utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/FuzzedValue.kt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@ import org.utbot.framework.plugin.api.UtModel
88
*/
99
class FuzzedValue(
1010
val model: UtModel,
11-
val createdBy: ModelProvider? = null
11+
val createdBy: ModelProvider? = null,
12+
val mutatedBy: List<ModelMutator> = emptyList(),
1213
) {
1314

1415
/**

‎utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/Fuzzer.kt

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,19 @@
11
package org.utbot.fuzzer
22

3-
import org.utbot.fuzzer.providers.ConstantsModelProvider
4-
import org.utbot.fuzzer.providers.ObjectModelProvider
5-
import org.utbot.fuzzer.providers.PrimitivesModelProvider
6-
import org.utbot.fuzzer.providers.StringConstantModelProvider
73
import mu.KotlinLogging
4+
import org.utbot.fuzzer.mutators.NumberRandomMutator
5+
import org.utbot.fuzzer.mutators.StringRandomMutator
86
import org.utbot.fuzzer.providers.ArrayModelProvider
97
import org.utbot.fuzzer.providers.CharToStringModelProvider
108
import org.utbot.fuzzer.providers.CollectionModelProvider
11-
import org.utbot.fuzzer.providers.PrimitiveDefaultsModelProvider
9+
import org.utbot.fuzzer.providers.ConstantsModelProvider
1210
import org.utbot.fuzzer.providers.EnumModelProvider
11+
import org.utbot.fuzzer.providers.ObjectModelProvider
12+
import org.utbot.fuzzer.providers.PrimitiveDefaultsModelProvider
1313
import org.utbot.fuzzer.providers.PrimitiveWrapperModelProvider
14-
import java.lang.IllegalArgumentException
15-
import java.util.IdentityHashMap
14+
import org.utbot.fuzzer.providers.PrimitivesModelProvider
15+
import org.utbot.fuzzer.providers.StringConstantModelProvider
16+
import java.util.*
1617
import java.util.concurrent.atomic.AtomicInteger
1718
import kotlin.random.Random
1819

@@ -148,3 +149,5 @@ fun objectModelProviders(idGenerator: IdentityPreservingIdGenerator<Int>): Model
148149
PrimitiveWrapperModelProvider,
149150
)
150151
}
152+
153+
fun defaultMutators(): List<ModelMutator> = listOf(StringRandomMutator(50), NumberRandomMutator(50))

‎utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/ModelMutator.kt

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
package org.utbot.fuzzer
2+
3+
import org.utbot.framework.plugin.api.UtModel
4+
import kotlin.random.Random
5+
6+
/**
7+
* Mutates values and returns it.
8+
*
9+
* Mutator can be not applied using [probability] as pivot.
10+
* In this case unchanged values is returned.
11+
*/
12+
interface ModelMutator {
13+
14+
/**
15+
* The probability of applying this mutator. Can be ignored in some implementations.
16+
*/
17+
val probability: Int
18+
19+
/**
20+
* Mutates values set.
21+
*
22+
* Default implementation iterates through values and delegates to `mutate(FuzzedMethodDescription, Int, Random)`.
23+
*/
24+
fun mutate(
25+
description: FuzzedMethodDescription,
26+
parameters: List<FuzzedValue>,
27+
random: Random,
28+
) : List<FuzzedValue> {
29+
return parameters.mapIndexed { index, fuzzedValue ->
30+
mutate(description, index, fuzzedValue, random) ?: fuzzedValue
31+
}
32+
}
33+
34+
/**
35+
* Mutate a single value if it is possible.
36+
*
37+
* Default implementation mutates the value with given [probability] or returns `null`.
38+
*/
39+
fun mutate(
40+
description: FuzzedMethodDescription,
41+
index: Int,
42+
value: FuzzedValue,
43+
random: Random
44+
) : FuzzedValue? {
45+
return if (random.nextInt(1, 101) < probability) value else null
46+
}
47+
48+
fun UtModel.mutatedFrom(template: FuzzedValue, block: FuzzedValue.() -> Unit = {}): FuzzedValue {
49+
return FuzzedValue(this, template.createdBy, template.mutatedBy + this@ModelMutator).apply(block)
50+
}
51+
}

‎utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/mutators/NumberRandomMutator.kt

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
package org.utbot.fuzzer.mutators
2+
3+
import org.utbot.framework.plugin.api.UtPrimitiveModel
4+
import org.utbot.fuzzer.FuzzedMethodDescription
5+
import org.utbot.fuzzer.FuzzedValue
6+
import org.utbot.fuzzer.ModelMutator
7+
import kotlin.random.Random
8+
9+
/**
10+
* Mutates any [Number] changing random bit.
11+
*/
12+
class NumberRandomMutator(override val probability: Int) : ModelMutator {
13+
14+
init {
15+
check(probability in 0 .. 100) { "Probability must be in range 0..100" }
16+
}
17+
18+
override fun mutate(
19+
description: FuzzedMethodDescription,
20+
index: Int,
21+
value: FuzzedValue,
22+
random: Random
23+
): FuzzedValue? {
24+
if (random.nextInt(1, 101) <= probability) return null
25+
val model = value.model
26+
return if (model is UtPrimitiveModel && model.value is Number) {
27+
val newValue = changeRandomBit(random, model.value as Number)
28+
UtPrimitiveModel(newValue).mutatedFrom(value) {
29+
summary = "%var% = $newValue (mutated from ${model.value})"
30+
}
31+
} else null
32+
}
33+
34+
private fun changeRandomBit(random: Random, number: Number): Number {
35+
val size = when (number) {
36+
is Byte -> Byte.SIZE_BITS
37+
is Short -> Short.SIZE_BITS
38+
is Int -> Int.SIZE_BITS
39+
is Float -> Float.SIZE_BITS
40+
is Long -> Long.SIZE_BITS
41+
is Double -> Double.SIZE_BITS
42+
else -> error("Unknown type: ${number.javaClass}")
43+
}
44+
val asLong = when (number) {
45+
is Byte, is Short, is Int -> number.toLong()
46+
is Long -> number
47+
is Float -> number.toRawBits().toLong()
48+
is Double -> number.toRawBits()
49+
else -> error("Unknown type: ${number.javaClass}")
50+
}
51+
val bitIndex = random.nextInt(size)
52+
val mutated = invertBit(asLong, bitIndex)
53+
return when (number) {
54+
is Byte -> mutated.toByte()
55+
is Short -> mutated.toShort()
56+
is Int -> mutated.toInt()
57+
is Float -> Float.fromBits(mutated.toInt())
58+
is Long -> mutated
59+
is Double -> Double.fromBits(mutated)
60+
else -> error("Unknown type: ${number.javaClass}")
61+
}
62+
}
63+
64+
private fun invertBit(value: Long, bitIndex: Int): Long {
65+
val b = ((value shr bitIndex) and 1).inv()
66+
val mask = 1L shl bitIndex
67+
return value and mask.inv() or (b shl bitIndex and mask)
68+
}
69+
}
70+

0 commit comments

Comments
 (0)