Skip to content

Commit 4d5575b

Browse files
authored
Add new fuzzing update method to control execution (#1740)
1 parent 28e6636 commit 4d5575b

File tree

8 files changed

+189
-33
lines changed

8 files changed

+189
-33
lines changed

utbot-fuzzers/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ import org.utbot.fuzzer.*
99
import org.utbot.fuzzing.providers.*
1010
import org.utbot.fuzzing.utils.Trie
1111
import java.lang.reflect.*
12+
import java.util.concurrent.CancellationException
1213
import java.util.concurrent.TimeUnit
13-
import kotlin.system.measureNanoTime
1414
import kotlin.random.Random
1515

1616
private val logger = KotlinLogging.logger {}
@@ -105,23 +105,31 @@ suspend fun runJavaFuzzing(
105105
val tracer = Trie(Instruction::id)
106106
val descriptionWithOptionalThisInstance = FuzzedDescription(createFuzzedMethodDescription(thisInstance), tracer, typeCache, random)
107107
val descriptionWithOnlyParameters = FuzzedDescription(createFuzzedMethodDescription(null), tracer, typeCache, random)
108+
val start = System.nanoTime()
108109
try {
109110
logger.info { "Starting fuzzing for method: $methodUnderTest" }
110111
logger.info { "\tuse thisInstance = ${thisInstance != null}" }
111112
logger.info { "\tparameters = $parameters" }
112113
var totalExecutionCalled = 0
113-
val totalFuzzingTime = measureNanoTime {
114-
runFuzzing(ValueProvider.of(providers), descriptionWithOptionalThisInstance, random) { _, t ->
115-
totalExecutionCalled++
116-
if (thisInstance == null) {
117-
exec(null, descriptionWithOnlyParameters, t)
118-
} else {
119-
exec(t.first(), descriptionWithOnlyParameters, t.drop(1))
120-
}
114+
runFuzzing(
115+
provider = ValueProvider.of(providers),
116+
description = descriptionWithOptionalThisInstance, random,
117+
configuration = Configuration()
118+
) { _, t ->
119+
totalExecutionCalled++
120+
if (thisInstance == null) {
121+
exec(null, descriptionWithOnlyParameters, t)
122+
} else {
123+
exec(t.first(), descriptionWithOnlyParameters, t.drop(1))
121124
}
122125
}
126+
val totalFuzzingTime = System.nanoTime() - start
123127
logger.info { "Finishing fuzzing for method: $methodUnderTest in ${TimeUnit.NANOSECONDS.toMillis(totalFuzzingTime)} ms" }
124128
logger.info { "\tTotal execution called: $totalExecutionCalled" }
129+
} catch (ce: CancellationException) {
130+
val totalFuzzingTime = System.nanoTime() - start
131+
logger.info { "Fuzzing is stopped because of timeout. Total execution time: ${TimeUnit.NANOSECONDS.toMillis(totalFuzzingTime)} ms" }
132+
logger.debug(ce) { "\tStacktrace:" }
125133
} catch (t: Throwable) {
126134
logger.info(t) { "Fuzzing is stopped because of an error" }
127135
}
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
package org.utbot.fuzzing.samples;
2+
3+
import java.util.List;
4+
5+
@SuppressWarnings("unused")
6+
public class FailToGenerateListGeneric {
7+
8+
interface Something {}
9+
10+
int func(List<Something> x) {
11+
return x.size();
12+
}
13+
14+
}

utbot-fuzzers/src/test/kotlin/org/utbot/fuzzing/JavaFuzzingTest.kt

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,19 @@
11
package org.utbot.fuzzing
22

33
import kotlinx.coroutines.runBlocking
4+
import kotlinx.coroutines.withTimeout
45
import org.junit.jupiter.api.Assertions.*
56
import org.junit.jupiter.api.Test
6-
import org.junit.jupiter.api.assertDoesNotThrow
77
import org.utbot.framework.plugin.api.MethodId
88
import org.utbot.framework.plugin.api.TestIdentityPreservingIdGenerator
9+
import org.utbot.framework.plugin.api.UtAssembleModel
910
import org.utbot.framework.plugin.api.UtPrimitiveModel
1011
import org.utbot.framework.plugin.api.util.*
1112
import org.utbot.fuzzer.FuzzedConcreteValue
1213
import org.utbot.fuzzing.samples.DeepNested
1314
import org.utbot.fuzzer.FuzzedType
1415
import org.utbot.fuzzing.samples.AccessibleObjects
16+
import org.utbot.fuzzing.samples.FailToGenerateListGeneric
1517
import org.utbot.fuzzing.samples.Stubs
1618
import org.utbot.fuzzing.utils.Trie
1719
import java.lang.reflect.GenericArrayType
@@ -214,10 +216,36 @@ class JavaFuzzingTest {
214216
assertEquals(0, exec) { "Fuzzer should not create any values of private classes" }
215217
}
216218

219+
@Test
220+
fun `fuzzing generate single test in case of collection with fail-to-generate generic type`() {
221+
val size = 100
222+
var exec = size
223+
val collections = ArrayList<Any?>(exec)
224+
runBlockingWithContext {
225+
runJavaFuzzing(
226+
TestIdentityPreservingIdGenerator,
227+
methodUnderTest = FailToGenerateListGeneric::class.java.declaredMethods.first { it.name == "func" }.executableId,
228+
constants = emptyList(),
229+
names = emptyList()
230+
) { _, _, v ->
231+
collections.add(v.first().model as? UtAssembleModel)
232+
BaseFeedback(Trie.emptyNode(), if (--exec > 0) Control.CONTINUE else Control.STOP)
233+
}
234+
}
235+
assertEquals(0, exec) { "Total fuzzer run number must be 0" }
236+
assertEquals(size, collections.size) { "Total generated values number must be $size" }
237+
assertEquals(size, collections.count { it is UtAssembleModel }) { "Total assemble models size must be $size" }
238+
collections.filterIsInstance<UtAssembleModel>().forEach {
239+
assertEquals(0, it.modificationsChain.size)
240+
}
241+
}
242+
217243
private fun <T> runBlockingWithContext(block: suspend () -> T) : T {
218244
return withUtContext(UtContext(this::class.java.classLoader)) {
219245
runBlocking {
220-
block()
246+
withTimeout(10000) {
247+
block()
248+
}
221249
}
222250
}
223251
}

utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt

Lines changed: 68 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ package org.utbot.fuzzing
44
import kotlinx.coroutines.yield
55
import mu.KotlinLogging
66
import org.utbot.fuzzing.seeds.KnownValue
7+
import org.utbot.fuzzing.utils.MissedSeed
78
import org.utbot.fuzzing.utils.chooseOne
89
import org.utbot.fuzzing.utils.flipCoin
910
import org.utbot.fuzzing.utils.transformIfNotEmpty
@@ -36,18 +37,30 @@ interface Fuzzing<TYPE, RESULT, DESCRIPTION : Description<TYPE>, FEEDBACK : Feed
3637
*
3738
* Fuzzing combines, randomize and mutates values using the seeds.
3839
* Then it generates values and runs them with this method. This method should provide some feedback,
39-
* which is the most important part for good fuzzing result. [emptyFeedback] can be provided only for test
40-
* or infinite loops. Consider to implement own implementation of [Feedback] to provide more correct data or
40+
* which is the most important part for a good fuzzing result. [emptyFeedback] can be provided only for test
41+
* or infinite loops. Consider implementing own implementation of [Feedback] to provide more correct data or
4142
* use [BaseFeedback] to generate key based feedback. In this case, the key is used to analyse what value should be next.
4243
*
4344
* @param description contains user-defined information about current run. Can be used as a state of the run.
4445
* @param values current values to process.
4546
*/
4647
suspend fun handle(description: DESCRIPTION, values: List<RESULT>): FEEDBACK
48+
49+
/**
50+
* This method is called before every fuzzing attempt.
51+
*
52+
* Usually, is used to update configuration or manipulate some data in the statistics
53+
* (for example, clear some [Statistic.missedTypes] types).
54+
*
55+
* @param description contains user-defined information about current run. Can be used as a state of the run.
56+
* @param statistic statistic about fuzzing generation like elapsed time or number of the runs.
57+
* @param configuration current used configuration; it can be changes for tuning fuzzing.
58+
*/
59+
suspend fun update(description: DESCRIPTION, statistic: Statistic<TYPE, RESULT>, configuration: Configuration) {}
4760
}
4861

4962
/**
50-
* Some description of current fuzzing run. Usually, contains name of the target method and its parameter list.
63+
* Some description of current fuzzing run. Usually, it contains the name of the target method and its parameter list.
5164
*/
5265
open class Description<TYPE>(
5366
val parameters: List<TYPE>
@@ -235,6 +248,11 @@ private object EmptyFeedback : Feedback<Nothing, Nothing> {
235248
}
236249
}
237250

251+
private class NoSeedValueException(
252+
// this type cannot be generalized because Java forbids types for [Throwable].
253+
val type: Any?
254+
) : Exception("No seed candidates generated for type: $type")
255+
238256
/**
239257
* Starts fuzzing for this [Fuzzing] object.
240258
*
@@ -245,6 +263,15 @@ suspend fun <T, R, D : Description<T>, F : Feedback<T, R>> Fuzzing<T, R, D, F>.f
245263
random: Random = Random(0),
246264
configuration: Configuration = Configuration()
247265
) {
266+
class StatImpl(
267+
override var totalRuns: Long = 0,
268+
val startTime: Long = System.nanoTime(),
269+
override var missedTypes: MissedSeed<T, R> = MissedSeed(),
270+
) : Statistic<T, R> {
271+
override val elapsedTime: Long
272+
get() = System.nanoTime() - startTime
273+
}
274+
val userStatistic = StatImpl()
248275
val fuzzing = this
249276
val typeCache = hashMapOf<T, List<Seed<T, R>>>()
250277
fun fuzzOne(): Node<T, R> = fuzz(
@@ -254,7 +281,7 @@ suspend fun <T, R, D : Description<T>, F : Feedback<T, R>> Fuzzing<T, R, D, F>.f
254281
random = random,
255282
configuration = configuration,
256283
builder = PassRoutine("Main Routine"),
257-
state = State(1, typeCache),
284+
state = State(1, typeCache, userStatistic.missedTypes),
258285
)
259286
val dynamicallyGenerated = mutableListOf<Node<T, R>>()
260287
val seeds = Statistics<T, R, F>()
@@ -275,13 +302,16 @@ suspend fun <T, R, D : Description<T>, F : Feedback<T, R>> Fuzzing<T, R, D, F>.f
275302
fuzzing,
276303
random,
277304
configuration,
278-
State(1, typeCache)
305+
State(1, typeCache, userStatistic.missedTypes)
279306
)
280307
}
281308
}
282309
}
283310
}.forEach execution@ { values ->
284311
yield()
312+
fuzzing.update(description, userStatistic.apply {
313+
totalRuns++
314+
}, configuration)
285315
check(values.parameters.size == values.result.size) { "Cannot create value for ${values.parameters}" }
286316
val valuesCache = mutableMapOf<Result<T, R>, R>()
287317
val result = values.result.map { valuesCache.computeIfAbsent(it) { r -> create(r) } }
@@ -318,6 +348,7 @@ private fun <TYPE, RESULT, DESCRIPTION : Description<TYPE>, FEEDBACK : Feedback<
318348
val result = parameters.map { type ->
319349
val results = typeCache.computeIfAbsent(type) { mutableListOf() }
320350
if (results.isNotEmpty() && random.flipCoin(configuration.probReuseGeneratedValueForSameType)) {
351+
// we need to check cases when one value is passed for different arguments
321352
results.random(random)
322353
} else {
323354
produce(type, fuzzing, description, random, configuration, state).also {
@@ -347,7 +378,7 @@ private fun <TYPE, RESULT, DESCRIPTION : Description<TYPE>, FEEDBACK : Feedback<
347378
}
348379
}
349380
if (candidates.isEmpty()) {
350-
error("No seed candidates generated for type: $type")
381+
throw NoSeedValueException(type)
351382
}
352383
return candidates.random(random)
353384
}
@@ -366,11 +397,10 @@ private fun <TYPE, RESULT, DESCRIPTION : Description<TYPE>, FEEDBACK : Feedback<
366397
): Result<TYPE, RESULT> {
367398
return if (state.recursionTreeDepth > configuration.recursionTreeDepth) {
368399
Result.Empty { task.construct.builder(0) }
369-
} else {
370-
val iterations = if (state.iterations >= 0 && random.flipCoin(configuration.probCreateRectangleCollectionInsteadSawLike)) {
371-
state.iterations
372-
} else {
373-
random.nextInt(0, configuration.collectionIterations + 1)
400+
} else try {
401+
val iterations = when {
402+
state.iterations >= 0 && random.flipCoin(configuration.probCreateRectangleCollectionInsteadSawLike) -> state.iterations
403+
else -> random.nextInt(0, configuration.collectionIterations + 1)
374404
}
375405
Result.Collection(
376406
construct = fuzz(
@@ -380,22 +410,30 @@ private fun <TYPE, RESULT, DESCRIPTION : Description<TYPE>, FEEDBACK : Feedback<
380410
random,
381411
configuration,
382412
task.construct,
383-
State(state.recursionTreeDepth + 1, state.cache, iterations)
413+
State(state.recursionTreeDepth + 1, state.cache, state.missedTypes, iterations)
384414
),
385415
modify = if (random.flipCoin(configuration.probCollectionMutationInsteadCreateNew)) {
386-
val result = fuzz(task.modify.types, fuzzing, description, random, configuration, task.modify, State(state.recursionTreeDepth + 1, state.cache, iterations))
416+
val result = fuzz(task.modify.types, fuzzing, description, random, configuration, task.modify, State(state.recursionTreeDepth + 1, state.cache, state.missedTypes, iterations))
387417
arrayListOf(result).apply {
388418
(1 until iterations).forEach { _ ->
389419
add(mutate(result, fuzzing, random, configuration, state))
390420
}
391421
}
392422
} else {
393423
(0 until iterations).map {
394-
fuzz(task.modify.types, fuzzing, description, random, configuration, task.modify, State(state.recursionTreeDepth + 1, state.cache, iterations))
424+
fuzz(task.modify.types, fuzzing, description, random, configuration, task.modify, State(state.recursionTreeDepth + 1, state.cache, state.missedTypes, iterations))
395425
}
396426
},
397427
iterations = iterations
398428
)
429+
} catch (nsv: NoSeedValueException) {
430+
@Suppress("UNCHECKED_CAST")
431+
state.missedTypes[nsv.type as TYPE] = task
432+
if (configuration.generateEmptyCollectionsForMissedTypes) {
433+
Result.Empty { task.construct.builder(0) }
434+
} else {
435+
throw nsv
436+
}
399437
}
400438
}
401439

@@ -413,7 +451,7 @@ private fun <TYPE, RESULT, DESCRIPTION : Description<TYPE>, FEEDBACK : Feedback<
413451
): Result<TYPE, RESULT> {
414452
return if (state.recursionTreeDepth > configuration.recursionTreeDepth) {
415453
Result.Empty { task.empty.builder() }
416-
} else {
454+
} else try {
417455
Result.Recursive(
418456
construct = fuzz(
419457
task.construct.types,
@@ -422,7 +460,7 @@ private fun <TYPE, RESULT, DESCRIPTION : Description<TYPE>, FEEDBACK : Feedback<
422460
random,
423461
configuration,
424462
task.construct,
425-
State(state.recursionTreeDepth + 1, state.cache)
463+
State(state.recursionTreeDepth + 1, state.cache, state.missedTypes)
426464
),
427465
modify = task.modify
428466
.shuffled(random)
@@ -434,12 +472,20 @@ private fun <TYPE, RESULT, DESCRIPTION : Description<TYPE>, FEEDBACK : Feedback<
434472
random,
435473
configuration,
436474
routine,
437-
State(state.recursionTreeDepth + 1, state.cache)
475+
State(state.recursionTreeDepth + 1, state.cache, state.missedTypes)
438476
)
439477
}.transformIfNotEmpty {
440478
take(random.nextInt(size + 1).coerceAtLeast(1))
441479
}
442480
)
481+
} catch (nsv: NoSeedValueException) {
482+
@Suppress("UNCHECKED_CAST")
483+
state.missedTypes[nsv.type as TYPE] = task
484+
if (configuration.generateEmptyRecursiveForMissedTypes) {
485+
Result.Empty { task.empty() }
486+
} else {
487+
throw nsv
488+
}
443489
}
444490
}
445491

@@ -472,7 +518,7 @@ private fun <TYPE, RESULT, DESCRIPTION : Description<TYPE>, FEEDBACK : Feedback<
472518
is Result.Recursive<TYPE, RESULT> -> {
473519
if (resultToMutate.modify.isEmpty() || random.flipCoin(configuration.probConstructorMutationInsteadModificationMutation)) {
474520
Result.Recursive(
475-
construct = mutate(resultToMutate.construct, fuzzing, random, configuration, State(state.recursionTreeDepth + 1, state.cache)),
521+
construct = mutate(resultToMutate.construct, fuzzing, random, configuration, State(state.recursionTreeDepth + 1, state.cache, state.missedTypes)),
476522
modify = resultToMutate.modify
477523
)
478524
} else if (random.flipCoin(configuration.probShuffleAndCutRecursiveObjectModificationMutation)) {
@@ -485,7 +531,7 @@ private fun <TYPE, RESULT, DESCRIPTION : Description<TYPE>, FEEDBACK : Feedback<
485531
construct = resultToMutate.construct,
486532
modify = resultToMutate.modify.toMutableList().apply {
487533
val i = random.nextInt(0, resultToMutate.modify.size)
488-
set(i, mutate(resultToMutate.modify[i], fuzzing, random, configuration, State(state.recursionTreeDepth + 1, state.cache)))
534+
set(i, mutate(resultToMutate.modify[i], fuzzing, random, configuration, State(state.recursionTreeDepth + 1, state.cache, state.missedTypes)))
489535
}
490536
)
491537
}
@@ -496,7 +542,7 @@ private fun <TYPE, RESULT, DESCRIPTION : Description<TYPE>, FEEDBACK : Feedback<
496542
if (isNotEmpty()) {
497543
if (random.flipCoin(100 - configuration.probCollectionShuffleInsteadResultMutation)) {
498544
val i = random.nextInt(0, resultToMutate.modify.size)
499-
set(i, mutate(resultToMutate.modify[i], fuzzing, random, configuration, State(state.recursionTreeDepth + 1, state.cache)))
545+
set(i, mutate(resultToMutate.modify[i], fuzzing, random, configuration, State(state.recursionTreeDepth + 1, state.cache, state.missedTypes)))
500546
} else {
501547
shuffle(random)
502548
}
@@ -578,7 +624,8 @@ private data class PassRoutine<T, R>(val description: String) : Routine<T, R>(em
578624
private class State<TYPE, RESULT>(
579625
val recursionTreeDepth: Int = 1,
580626
val cache: MutableMap<TYPE, List<Seed<TYPE, RESULT>>>,
581-
val iterations: Int = -1
627+
val missedTypes: MissedSeed<TYPE, RESULT>,
628+
val iterations: Int = -1,
582629
)
583630

584631
/**

utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Configuration.kt

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,5 +71,17 @@ class Configuration(
7171
/**
7272
* Probability of reusing same generated value when 2 or more parameters have the same type.
7373
*/
74-
var probReuseGeneratedValueForSameType: Int = 1
74+
var probReuseGeneratedValueForSameType: Int = 1,
75+
76+
/**
77+
* When true any [Seed.Collection] will not try
78+
* to generate modification if a current type is already known to fail to generate values.
79+
*/
80+
var generateEmptyCollectionsForMissedTypes: Boolean = true,
81+
82+
/**
83+
* When true any [Seed.Recursive] will not try
84+
* to generate a recursive object, but will use [Seed.Recursive.empty] instead.
85+
*/
86+
var generateEmptyRecursiveForMissedTypes: Boolean = true,
7587
)

0 commit comments

Comments
 (0)