Skip to content

Commit d3b10e2

Browse files
MarkoutteMarkoutte
authored
Generate test by fuzzing for methods with no parameters #511 (#515)
1 parent 894a379 commit d3b10e2

19 files changed

+164
-105
lines changed

‎utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ import org.utbot.framework.plugin.api.EnvironmentModels
6464
import org.utbot.framework.plugin.api.Instruction
6565
import org.utbot.framework.plugin.api.MissingState
6666
import org.utbot.framework.plugin.api.Step
67+
import org.utbot.framework.plugin.api.UtAssembleModel
6768
import org.utbot.framework.plugin.api.UtConcreteExecutionFailure
6869
import org.utbot.framework.plugin.api.UtError
6970
import org.utbot.framework.plugin.api.UtExecution
@@ -79,6 +80,7 @@ import org.utbot.framework.plugin.api.util.id
7980
import org.utbot.framework.plugin.api.util.utContext
8081
import org.utbot.framework.plugin.api.util.description
8182
import org.utbot.framework.util.jimpleBody
83+
import org.utbot.framework.plugin.api.util.voidClassId
8284
import org.utbot.fuzzer.FallbackModelProvider
8385
import org.utbot.fuzzer.FuzzedMethodDescription
8486
import org.utbot.fuzzer.FuzzedValue
@@ -89,10 +91,12 @@ import org.utbot.fuzzer.defaultModelProviders
8991
import org.utbot.fuzzer.fuzz
9092
import org.utbot.fuzzer.names.MethodBasedNameSuggester
9193
import org.utbot.fuzzer.names.ModelBasedNameSuggester
94+
import org.utbot.fuzzer.providers.ObjectModelProvider
9295
import org.utbot.instrumentation.ConcreteExecutor
9396
import soot.jimple.Stmt
9497
import soot.tagkit.ParamNamesTag
9598
import java.lang.reflect.Method
99+
import kotlin.random.Random
96100
import kotlin.system.measureTimeMillis
97101

98102
val logger = KotlinLogging.logger {}
@@ -384,6 +388,7 @@ class UtBotSymbolicEngine(
384388
}
385389

386390
val fallbackModelProvider = FallbackModelProvider { nextDefaultModelId++ }
391+
val constantValues = collectConstantsForFuzzer(graph)
387392

388393
val thisInstance = when {
389394
methodUnderTest.isStatic -> null
@@ -396,7 +401,9 @@ class UtBotSymbolicEngine(
396401
null
397402
}
398403
else -> {
399-
fallbackModelProvider.toModel(methodUnderTest.clazz).apply {
404+
ObjectModelProvider { nextDefaultModelId++ }.withFallback(fallbackModelProvider).generate(
405+
FuzzedMethodDescription("thisInstance", voidClassId, listOf(methodUnderTest.clazz.id), constantValues)
406+
).take(10).shuffled(Random(0)).map { it.value.model }.first().apply {
400407
if (this is UtNullModel) { // it will definitely fail because of NPE,
401408
return@flow
402409
}
@@ -411,17 +418,34 @@ class UtBotSymbolicEngine(
411418
val names = graph.body.method.tags.filterIsInstance<ParamNamesTag>().firstOrNull()?.names
412419
parameterNameMap = { index -> names?.getOrNull(index) }
413420
}
414-
val modelProviderWithFallback = modelProvider(defaultModelProviders { nextDefaultModelId++ }).withFallback(fallbackModelProvider::toModel)
415421
val coveredInstructionTracker = Trie(Instruction::id)
416422
val coveredInstructionValues = mutableMapOf<Trie.Node<Instruction>, List<FuzzedValue>>()
417423
var attempts = UtSettings.fuzzingMaxAttempts
418-
fuzz(methodUnderTestDescription, modelProviderWithFallback).forEach { values ->
424+
val hasMethodUnderTestParametersToFuzz = executableId.parameters.isNotEmpty()
425+
val fuzzedValues = if (hasMethodUnderTestParametersToFuzz) {
426+
fuzz(methodUnderTestDescription, modelProvider(defaultModelProviders { nextDefaultModelId++ }))
427+
} else {
428+
// in case a method with no parameters is passed fuzzing tries to fuzz this instance with different constructors, setters and field mutators
429+
val thisMethodDescription = FuzzedMethodDescription("thisInstance", voidClassId, listOf(methodUnderTest.clazz.id), constantValues).apply {
430+
className = executableId.classId.simpleName
431+
packageName = executableId.classId.packageName
432+
}
433+
fuzz(thisMethodDescription, ObjectModelProvider { nextDefaultModelId++ }.apply {
434+
limitValuesCreatedByFieldAccessors = 500
435+
})
436+
}
437+
fuzzedValues.forEach { values ->
419438
if (System.currentTimeMillis() >= until) {
420439
logger.info { "Fuzzing overtime: $methodUnderTest" }
421440
return@flow
422441
}
423442

424-
val initialEnvironmentModels = EnvironmentModels(thisInstance, values.map { it.model }, mapOf())
443+
val initialEnvironmentModels = if (hasMethodUnderTestParametersToFuzz) {
444+
EnvironmentModels(thisInstance, values.map { it.model }, mapOf())
445+
} else {
446+
check(values.size == 1 && values.first().model is UtAssembleModel)
447+
EnvironmentModels(values.first().model, emptyList(), mapOf())
448+
}
425449

426450
try {
427451
val concreteExecutionResult =
@@ -462,7 +486,7 @@ class UtBotSymbolicEngine(
462486
fullPath = emptyList(),
463487
coverage = concreteExecutionResult.coverage,
464488
testMethodName = testMethodName?.testName,
465-
displayName = testMethodName?.displayName
489+
displayName = testMethodName?.takeIf { hasMethodUnderTestParametersToFuzz }?.displayName
466490
)
467491
)
468492
} catch (e: CancellationException) {

‎utbot-framework/src/main/kotlin/org/utbot/external/api/UtBotJavaApi.kt

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import org.utbot.framework.plugin.api.util.withUtContext
3232
import org.utbot.framework.plugin.api.util.wrapperByPrimitive
3333
import org.utbot.fuzzer.FuzzedValue
3434
import org.utbot.fuzzer.ModelProvider
35+
import org.utbot.fuzzer.ModelProvider.Companion.yieldValue
3536
import org.utbot.instrumentation.ConcreteExecutor
3637
import org.utbot.instrumentation.execute
3738
import java.lang.reflect.Method
@@ -175,11 +176,13 @@ object UtBotJavaApi {
175176
}
176177
?.map { UtPrimitiveModel(it) } ?: emptySequence()
177178

178-
val customModelProvider = ModelProvider { description, consumer ->
179-
description.parametersMap.forEach { (classId, indices) ->
180-
createPrimitiveModels(primitiveValuesSupplier, classId).forEach { model ->
181-
indices.forEach { index ->
182-
consumer.accept(index, FuzzedValue(model))
179+
val customModelProvider = ModelProvider { description ->
180+
sequence {
181+
description.parametersMap.forEach { (classId, indices) ->
182+
createPrimitiveModels(primitiveValuesSupplier, classId).forEach { model ->
183+
indices.forEach { index ->
184+
yieldValue(index, FuzzedValue(model))
185+
}
183186
}
184187
}
185188
}

‎utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/FuzzedParameter.kt

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
package org.utbot.fuzzer
2+
3+
/**
4+
* Fuzzed parameter of a method.
5+
*
6+
* @param index of the parameter in method signature
7+
* @param value fuzzed values
8+
*/
9+
class FuzzedParameter(
10+
val index: Int,
11+
val value: FuzzedValue
12+
) {
13+
operator fun component1() = index
14+
operator fun component2() = value
15+
}

‎utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/Fuzzer.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ private val logger = KotlinLogging.logger {}
2020
fun fuzz(description: FuzzedMethodDescription, vararg modelProviders: ModelProvider): Sequence<List<FuzzedValue>> {
2121
val values = List<MutableList<FuzzedValue>>(description.parameters.size) { mutableListOf() }
2222
modelProviders.forEach { fuzzingProvider ->
23-
fuzzingProvider.generate(description) { index, model ->
23+
fuzzingProvider.generate(description).forEach { (index, model) ->
2424
values[index].add(model)
2525
}
2626
}

‎utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/ModelProvider.kt

Lines changed: 41 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,16 @@ package org.utbot.fuzzer
22

33
import org.utbot.framework.plugin.api.UtModel
44
import org.utbot.framework.plugin.api.ClassId
5-
import java.util.function.BiConsumer
65

76
fun interface ModelProvider {
87

98
/**
109
* Generates values for the method.
1110
*
1211
* @param description a fuzzed method description
13-
* @param consumer accepts index in the parameter list and [UtModel] for this parameter.
12+
* @return sequence that produces [FuzzedParameter].
1413
*/
15-
fun generate(description: FuzzedMethodDescription, consumer: BiConsumer<Int, FuzzedValue>)
14+
fun generate(description: FuzzedMethodDescription): Sequence<FuzzedParameter>
1615

1716
/**
1817
* Combines this model provider with `anotherModelProvider` into one instance.
@@ -51,25 +50,24 @@ fun interface ModelProvider {
5150
* @param modelProvider is called and every value of [ClassId] is collected which wasn't created by this model provider.
5251
*/
5352
fun withFallback(modelProvider: ModelProvider) : ModelProvider {
54-
return ModelProvider { description, consumer ->
55-
val providedByDelegateMethodParameters = mutableMapOf<Int, MutableList<FuzzedValue>>()
56-
this@ModelProvider.generate(description) { index, model ->
57-
providedByDelegateMethodParameters.computeIfAbsent(index) { mutableListOf() }.add(model)
58-
}
59-
providedByDelegateMethodParameters.forEach { (index, models) ->
60-
models.forEach { model ->
61-
consumer.accept(index, model)
53+
val thisModelProvider = this
54+
return ModelProvider { description ->
55+
sequence {
56+
val providedByDelegateMethodParameters = mutableSetOf<Int>()
57+
thisModelProvider.generate(description).forEach { (index, model) ->
58+
providedByDelegateMethodParameters += index
59+
yieldValue(index, model)
6260
}
63-
}
64-
val missingParameters =
65-
(0 until description.parameters.size).filter { !providedByDelegateMethodParameters.containsKey(it) }
66-
if (missingParameters.isNotEmpty()) {
67-
val values = mutableMapOf<Int, MutableList<FuzzedValue>>()
68-
modelProvider.generate(description) { i, m -> values.computeIfAbsent(i) { mutableListOf() }.add(m) }
69-
missingParameters.forEach { index ->
70-
values[index]?.let { models ->
71-
models.forEach { model ->
72-
consumer.accept(index, model)
61+
val missingParameters =
62+
(0 until description.parameters.size).filter { !providedByDelegateMethodParameters.contains(it) }
63+
if (missingParameters.isNotEmpty()) {
64+
val values = mutableMapOf<Int, MutableList<FuzzedValue>>()
65+
modelProvider.generate(description).forEach { (i, m) -> values.computeIfAbsent(i) { mutableListOf() }.add(m) }
66+
missingParameters.forEach { index ->
67+
values[index]?.let { models ->
68+
models.forEach { model ->
69+
yieldValue(index, model)
70+
}
7371
}
7472
}
7573
}
@@ -86,15 +84,17 @@ fun interface ModelProvider {
8684
* @param fallbackModelSupplier is called for every [ClassId] which wasn't created by this model provider.
8785
*/
8886
fun withFallback(fallbackModelSupplier: (ClassId) -> UtModel?) : ModelProvider {
89-
return withFallback { description, consumer ->
90-
description.parametersMap.forEach { (classId, indices) ->
91-
fallbackModelSupplier(classId)?.let { model ->
92-
indices.forEach { index ->
93-
consumer.accept(index, model.fuzzed())
87+
return withFallback( ModelProvider { description ->
88+
sequence {
89+
description.parametersMap.forEach { (classId, indices) ->
90+
fallbackModelSupplier(classId)?.let { model ->
91+
indices.forEach { index ->
92+
yieldValue(index, model.fuzzed())
93+
}
9494
}
9595
}
9696
}
97-
}
97+
})
9898
}
9999

100100
companion object {
@@ -103,26 +103,32 @@ fun interface ModelProvider {
103103
return Combined(providers.toList())
104104
}
105105

106-
fun BiConsumer<Int, FuzzedValue>.consumeAll(indices: List<Int>, models: Sequence<FuzzedValue>) {
107-
models.forEach { model ->
108-
indices.forEach { index ->
109-
accept(index, model)
106+
suspend fun SequenceScope<FuzzedParameter>.yieldValue(index: Int, value: FuzzedValue) {
107+
yield(FuzzedParameter(index, value))
108+
}
109+
110+
suspend fun SequenceScope<FuzzedParameter>.yieldAllValues(indices: List<Int>, models: Sequence<FuzzedValue>) {
111+
indices.forEach { index ->
112+
models.forEach { model ->
113+
yieldValue(index, model)
110114
}
111115
}
112116
}
113117

114-
fun BiConsumer<Int, FuzzedValue>.consumeAll(indices: List<Int>, models: List<FuzzedValue>) {
115-
consumeAll(indices, models.asSequence())
118+
suspend fun SequenceScope<FuzzedParameter>.yieldAllValues(indices: List<Int>, models: List<FuzzedValue>) {
119+
yieldAllValues(indices, models.asSequence())
116120
}
117121
}
118122

119123
/**
120124
* Wrapper class that delegates implementation to the [providers].
121125
*/
122126
private class Combined(val providers: List<ModelProvider>): ModelProvider {
123-
override fun generate(description: FuzzedMethodDescription, consumer: BiConsumer<Int, FuzzedValue>) {
127+
override fun generate(description: FuzzedMethodDescription): Sequence<FuzzedParameter> = sequence {
124128
providers.forEach { provider ->
125-
provider.generate(description, consumer)
129+
provider.generate(description).forEach {
130+
yieldValue(it.index, it.value)
131+
}
126132
}
127133
}
128134
}

‎utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/providers/AbstractModelProvider.kt

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,20 +2,20 @@ package org.utbot.fuzzer.providers
22

33
import org.utbot.framework.plugin.api.*
44
import org.utbot.fuzzer.FuzzedMethodDescription
5-
import org.utbot.fuzzer.FuzzedValue
5+
import org.utbot.fuzzer.FuzzedParameter
66
import org.utbot.fuzzer.ModelProvider
7-
import java.util.function.BiConsumer
7+
import org.utbot.fuzzer.ModelProvider.Companion.yieldValue
88

99
/**
1010
* Simple model implementation.
1111
*/
1212
@Suppress("unused")
1313
abstract class AbstractModelProvider: ModelProvider {
14-
override fun generate(description: FuzzedMethodDescription, consumer: BiConsumer<Int, FuzzedValue>) {
14+
override fun generate(description: FuzzedMethodDescription): Sequence<FuzzedParameter> = sequence{
1515
description.parametersMap.forEach { (classId, indices) ->
1616
toModel(classId)?.let { defaultModel ->
1717
indices.forEach { index ->
18-
consumer.accept(index, defaultModel.fuzzed())
18+
yieldValue(index, defaultModel.fuzzed())
1919
}
2020
}
2121
}

‎utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/providers/ArrayModelProvider.kt

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,21 +4,20 @@ import org.utbot.framework.plugin.api.UtArrayModel
44
import org.utbot.framework.plugin.api.util.defaultValueModel
55
import org.utbot.framework.plugin.api.util.isArray
66
import org.utbot.fuzzer.FuzzedMethodDescription
7-
import org.utbot.fuzzer.FuzzedValue
7+
import org.utbot.fuzzer.FuzzedParameter
88
import org.utbot.fuzzer.ModelProvider
9-
import org.utbot.fuzzer.ModelProvider.Companion.consumeAll
10-
import java.util.function.BiConsumer
9+
import org.utbot.fuzzer.ModelProvider.Companion.yieldAllValues
1110
import java.util.function.IntSupplier
1211

1312
class ArrayModelProvider(
1413
private val idGenerator: IntSupplier
1514
) : ModelProvider {
16-
override fun generate(description: FuzzedMethodDescription, consumer: BiConsumer<Int, FuzzedValue>) {
15+
override fun generate(description: FuzzedMethodDescription): Sequence<FuzzedParameter> = sequence {
1716
description.parametersMap
1817
.asSequence()
1918
.filter { (classId, _) -> classId.isArray }
2019
.forEach { (arrayClassId, indices) ->
21-
consumer.consumeAll(indices, listOf(0, 10).map { arraySize ->
20+
yieldAllValues(indices, listOf(0, 10).map { arraySize ->
2221
UtArrayModel(
2322
id = idGenerator.asInt,
2423
arrayClassId,

‎utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/providers/CharToStringModelProvider.kt

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,26 +4,28 @@ import org.utbot.framework.plugin.api.UtPrimitiveModel
44
import org.utbot.framework.plugin.api.util.charClassId
55
import org.utbot.framework.plugin.api.util.stringClassId
66
import org.utbot.fuzzer.FuzzedMethodDescription
7-
import org.utbot.fuzzer.FuzzedValue
7+
import org.utbot.fuzzer.FuzzedParameter
88
import org.utbot.fuzzer.ModelProvider
9-
import java.util.function.BiConsumer
9+
import org.utbot.fuzzer.ModelProvider.Companion.yieldValue
1010

1111
/**
1212
* Collects all char constants and creates string with them.
1313
*/
1414
object CharToStringModelProvider : ModelProvider {
15-
override fun generate(description: FuzzedMethodDescription, consumer: BiConsumer<Int, FuzzedValue>) {
16-
val indices = description.parametersMap[stringClassId] ?: return
15+
override fun generate(description: FuzzedMethodDescription): Sequence<FuzzedParameter> = sequence {
16+
val indices = description.parametersMap[stringClassId] ?: return@sequence
1717
if (indices.isNotEmpty()) {
1818
val string = description.concreteValues.asSequence()
1919
.filter { it.classId == charClassId }
2020
.map { it.value }
2121
.filterIsInstance<Char>()
2222
.joinToString(separator = "")
2323
if (string.isNotEmpty()) {
24-
val model = UtPrimitiveModel(string).fuzzed()
25-
indices.forEach {
26-
consumer.accept(it, model)
24+
sequenceOf(string.reversed(), string).forEach { str ->
25+
val model = UtPrimitiveModel(str).fuzzed()
26+
indices.forEach {
27+
yieldValue(it, model)
28+
}
2729
}
2830
}
2931
}

‎utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/providers/CollectionModelProvider.kt

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,9 @@ import org.utbot.framework.plugin.api.UtStatementModel
1010
import org.utbot.framework.plugin.api.util.id
1111
import org.utbot.framework.plugin.api.util.jClass
1212
import org.utbot.fuzzer.FuzzedMethodDescription
13-
import org.utbot.fuzzer.FuzzedValue
13+
import org.utbot.fuzzer.FuzzedParameter
1414
import org.utbot.fuzzer.ModelProvider
15-
import org.utbot.fuzzer.ModelProvider.Companion.consumeAll
16-
import java.util.function.BiConsumer
15+
import org.utbot.fuzzer.ModelProvider.Companion.yieldAllValues
1716
import java.util.function.IntSupplier
1817

1918
/**
@@ -36,12 +35,12 @@ class CollectionModelProvider(
3635
java.util.Iterator::class.java to ::createIteratorModels,
3736
)
3837

39-
override fun generate(description: FuzzedMethodDescription, consumer: BiConsumer<Int, FuzzedValue>) {
38+
override fun generate(description: FuzzedMethodDescription): Sequence<FuzzedParameter> = sequence {
4039
description.parametersMap
4140
.asSequence()
4241
.forEach { (classId, indices) ->
4342
generators[classId.jClass]?.let { createModels ->
44-
consumer.consumeAll(indices, createModels().map { it.fuzzed() })
43+
yieldAllValues(indices, createModels().map { it.fuzzed() })
4544
}
4645
}
4746
}

0 commit comments

Comments
 (0)