From ec720a72d10d351242d83990efddb66c471f5e79 Mon Sep 17 00:00:00 2001 From: Maksim Pelevin Date: Mon, 26 Jun 2023 11:38:54 +0300 Subject: [PATCH 1/7] Add an option to enable 'always generate value' in fuzzing --- .../kotlin/org/utbot/framework/UtSettings.kt | 5 ++ .../org/utbot/fuzzer/FuzzerFunctions.kt | 41 ++++------- .../kotlin/org/utbot/fuzzing/JavaLanguage.kt | 5 +- .../org/utbot/fuzzing/providers/Objects.kt | 69 +++++++++++++++++-- 4 files changed, 89 insertions(+), 31 deletions(-) diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt index b6ea38e511..66bccdc6bc 100644 --- a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt +++ b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt @@ -236,6 +236,11 @@ object UtSettings : AbstractSettings(logger, defaultKeyForSettingsPath, defaultS */ var fuzzingTimeoutInMillis: Long by getLongProperty(3_000L, 0, Long.MAX_VALUE) + /** + * Fuzzer will create objects when they cannot be created by other providers (e.g. private classes) + */ + var fuzzObjectWhenTheyCannotBeCreatedClean: Boolean by getBooleanProperty(false) + /** * Generate tests that treat possible overflows in arithmetic operations as errors * that throw Arithmetic Exception. diff --git a/utbot-framework/src/main/kotlin/org/utbot/fuzzer/FuzzerFunctions.kt b/utbot-framework/src/main/kotlin/org/utbot/fuzzer/FuzzerFunctions.kt index a6e82fb5b1..4f2de61f8b 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/fuzzer/FuzzerFunctions.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/fuzzer/FuzzerFunctions.kt @@ -1,17 +1,11 @@ package org.utbot.fuzzer import mu.KotlinLogging +import org.utbot.framework.UtSettings import org.utbot.framework.plugin.api.classId -import org.utbot.framework.plugin.api.util.booleanClassId -import org.utbot.framework.plugin.api.util.byteClassId -import org.utbot.framework.plugin.api.util.charClassId -import org.utbot.framework.plugin.api.util.doubleClassId -import org.utbot.framework.plugin.api.util.floatClassId -import org.utbot.framework.plugin.api.util.intClassId -import org.utbot.framework.plugin.api.util.longClassId -import org.utbot.framework.plugin.api.util.shortClassId -import org.utbot.framework.plugin.api.util.stringClassId +import org.utbot.framework.plugin.api.util.* import org.utbot.framework.util.executableId +import org.utbot.fuzzing.providers.CreateObjectAnywayValueProvider import soot.BooleanType import soot.ByteType import soot.CharType @@ -28,23 +22,7 @@ import soot.jimple.Constant import soot.jimple.IntConstant import soot.jimple.InvokeExpr import soot.jimple.NullConstant -import soot.jimple.internal.AbstractSwitchStmt -import soot.jimple.internal.ImmediateBox -import soot.jimple.internal.JAssignStmt -import soot.jimple.internal.JCastExpr -import soot.jimple.internal.JEqExpr -import soot.jimple.internal.JGeExpr -import soot.jimple.internal.JGtExpr -import soot.jimple.internal.JIfStmt -import soot.jimple.internal.JInvokeStmt -import soot.jimple.internal.JLeExpr -import soot.jimple.internal.JLookupSwitchStmt -import soot.jimple.internal.JLtExpr -import soot.jimple.internal.JNeExpr -import soot.jimple.internal.JSpecialInvokeExpr -import soot.jimple.internal.JStaticInvokeExpr -import soot.jimple.internal.JTableSwitchStmt -import soot.jimple.internal.JVirtualInvokeExpr +import soot.jimple.internal.* import soot.toolkits.graph.ExceptionalUnitGraph private val logger = KotlinLogging.logger {} @@ -70,6 +48,7 @@ fun collectConstantsForFuzzer(graph: ExceptionalUnitGraph): Set try { finder.find(graph, unit, value) @@ -270,6 +249,16 @@ private object DateFormatByVarStringConstant: ConstantsFinder { } } +private object AbstractMethodIsCalled: ConstantsFinder { + override fun find(graph: ExceptionalUnitGraph, unit: Unit, value: Value): List { + if (UtSettings.fuzzObjectWhenTheyCannotBeCreatedClean && value is JInterfaceInvokeExpr) { + // todo do a better way to add information about virtual method calls to providers + return listOf(FuzzedConcreteValue(value.method.javaClass.id, CreateObjectAnywayValueProvider::class, FuzzedContext.Call(value.method.executableId))) + } + return emptyList() + } +} + private object ConstantsAsIs: ConstantsFinder { override fun find(graph: ExceptionalUnitGraph, unit: Unit, value: Value): List { if (value !is Constant || value is NullConstant) return emptyList() diff --git a/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt b/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt index 78332a0f8b..1275106765 100644 --- a/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt +++ b/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt @@ -1,6 +1,7 @@ package org.utbot.fuzzing import mu.KotlinLogging +import org.utbot.framework.UtSettings import org.utbot.framework.plugin.api.ClassId import org.utbot.framework.plugin.api.ExecutableId import org.utbot.framework.plugin.api.Instruction @@ -42,7 +43,9 @@ fun defaultValueProviders(idGenerator: IdentityPreservingIdGenerator) = lis FloatValueProvider, StringValueProvider, NumberValueProvider, - ObjectValueProvider(idGenerator), + ObjectValueProvider(idGenerator).letIf(UtSettings.fuzzObjectWhenTheyCannotBeCreatedClean) { ovp -> + ovp.withFallback(CreateObjectAnywayValueProvider(idGenerator, useMock = true)) + }, ArrayValueProvider(idGenerator), EnumValueProvider(idGenerator), ListSetValueProvider(idGenerator), diff --git a/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/providers/Objects.kt b/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/providers/Objects.kt index b34b92473a..94eb658e54 100644 --- a/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/providers/Objects.kt +++ b/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/providers/Objects.kt @@ -1,11 +1,9 @@ package org.utbot.fuzzing.providers +import org.utbot.common.isAbstract import org.utbot.framework.plugin.api.* import org.utbot.framework.plugin.api.util.* -import org.utbot.fuzzer.FuzzedType -import org.utbot.fuzzer.FuzzedValue -import org.utbot.fuzzer.IdGenerator -import org.utbot.fuzzer.fuzzed +import org.utbot.fuzzer.* import org.utbot.fuzzing.* import org.utbot.fuzzing.utils.hex import java.lang.reflect.Field @@ -134,6 +132,69 @@ object NullValueProvider : ValueProvider, + val useMock: Boolean = true, +) : ValueProvider { + + override fun accept(type: FuzzedType) = type.classId.isRefType + + override fun generate(description: FuzzedDescription, type: FuzzedType) = sequence> { + val methodCalls = description.constants.filter { + it.value == CreateObjectAnywayValueProvider::class + }.mapNotNull { + it.fuzzedContext as? FuzzedContext.Call + }.map { + it.method + }.toSet() + + yield(Seed.Recursive( + construct = Routine.Create(emptyList()) { + UtCompositeModel(idGenerator.createId(), type.classId, useMock).fuzzed { + summary = "some object" + } + }, + modify = sequence { + // generate all fields + generateSequence(type.classId.jClass) { + it.superclass + }.flatMap { javaClass -> + javaClass.declaredFields.toList() + }.forEach { field -> + yield(Routine.Call(listOf(toFuzzerType(field.type, description.typeCache))) { instance, args -> + (instance.model as UtCompositeModel).fields[field.fieldId] = args.first().model + }) + } + + generateSequence(listOf(type.classId.jClass)) { classList -> + classList.flatMap { listOf(it.superclass) + it.interfaces }.filterNotNull().takeIf { it.isNotEmpty() } + }.flatten().filter { + isAccessible(it, description.description.packageName) + }.flatMap { javaClass -> + javaClass.declaredMethods.filter { + javaClass.isInterface || it.isAbstract + }.filter { + isAccessible(it, description.description.packageName) + } + // todo filter by methods seen in code + }.forEach { method -> + val executableId = method.executableId + if (methodCalls.contains(executableId)) { + yield(Routine.Call(listOf(toFuzzerType(method.returnType, description.typeCache))) { instance, args -> + (instance.model as UtCompositeModel).mocks[executableId] = args.map(FuzzedValue::model) + }) + } + } + }, + empty = Routine.Empty { + UtCompositeModel(idGenerator.createId(), type.classId, useMock).fuzzed { + summary = "some object" + } + } + )) + } +} + internal class PublicSetterGetter( val setter: Method, val getter: Method, From 8fb0ee64aa3af6dc034a4c5af605457230acf6a6 Mon Sep 17 00:00:00 2001 From: Maksim Pelevin Date: Thu, 6 Jul 2023 09:14:36 +0300 Subject: [PATCH 2/7] Fix various problems with fuzzing in ContestEstimator --- .../main/kotlin/org/utbot/framework/plugin/api/Api.kt | 6 +++++- .../src/main/kotlin/org/utbot/fuzzing/Api.kt | 11 ++++++----- .../src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt | 2 +- .../kotlin/org/utbot/fuzzing/providers/Objects.kt | 2 +- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt index 99d71bc9da..80f0f070d2 100644 --- a/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt +++ b/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt @@ -881,6 +881,8 @@ val Type.classId: ClassId else -> error("Unknown type $this") } +private val logger = KotlinLogging.logger {} + /** * Class id. Contains name, not a full qualified name. * @@ -905,7 +907,9 @@ open class ClassId @JvmOverloads constructor( get() = jClass.modifiers open val canonicalName: String - get() = jClass.canonicalName ?: error("ClassId $name does not have canonical name") + get() = jClass.canonicalName ?: name.also { + logger.error("ClassId $name does not have canonical name") + } open val simpleName: String get() = jClass.simpleName diff --git a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt index 083e8be67b..705484e531 100644 --- a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt +++ b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt @@ -7,6 +7,7 @@ import org.utbot.fuzzing.seeds.KnownValue import org.utbot.fuzzing.utils.MissedSeed import org.utbot.fuzzing.utils.chooseOne import org.utbot.fuzzing.utils.flipCoin +import org.utbot.fuzzing.utils.transformIfNotEmpty import kotlin.random.Random private val logger by lazy { KotlinLogging.logger {} } @@ -515,11 +516,11 @@ private fun , FEEDBACK : Feedback< } ), modify = task.modify -// .toMutableList() -// .transformIfNotEmpty { -// shuffle(random) -// take(random.nextInt(size + 1)) -// } + .toMutableList() + .transformIfNotEmpty { + shuffle(random) + take(configuration.maxNumberOfRecursiveSeedModifications) + } .mapTo(arrayListOf()) { routine -> fuzz( routine.types, diff --git a/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt b/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt index 1275106765..0c6b1de657 100644 --- a/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt +++ b/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt @@ -44,7 +44,7 @@ fun defaultValueProviders(idGenerator: IdentityPreservingIdGenerator) = lis StringValueProvider, NumberValueProvider, ObjectValueProvider(idGenerator).letIf(UtSettings.fuzzObjectWhenTheyCannotBeCreatedClean) { ovp -> - ovp.withFallback(CreateObjectAnywayValueProvider(idGenerator, useMock = true)) + ovp.withFallback(CreateObjectAnywayValueProvider(idGenerator, useMock = false)) }, ArrayValueProvider(idGenerator), EnumValueProvider(idGenerator), diff --git a/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/providers/Objects.kt b/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/providers/Objects.kt index 94eb658e54..d047128404 100644 --- a/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/providers/Objects.kt +++ b/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/providers/Objects.kt @@ -134,7 +134,7 @@ object NullValueProvider : ValueProvider, - val useMock: Boolean = true, + val useMock: Boolean = false, ) : ValueProvider { override fun accept(type: FuzzedType) = type.classId.isRefType From 70125802e04ad869a970d42a95981287ac4687d2 Mon Sep 17 00:00:00 2001 From: Maksim Pelevin Date: Thu, 6 Jul 2023 12:07:39 +0300 Subject: [PATCH 3/7] JaCoCo can be run with test goal --- .../kotlin/org/utbot/fuzzing/Configuration.kt | 5 ++ utbot-junit-contest/build.gradle | 47 +++++++++++++++---- .../main/kotlin/org/utbot/contest/Contest.kt | 6 +-- 3 files changed, 44 insertions(+), 14 deletions(-) diff --git a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Configuration.kt b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Configuration.kt index 5675d70cf1..a6c19926c9 100644 --- a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Configuration.kt +++ b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Configuration.kt @@ -90,4 +90,9 @@ data class Configuration( * to generate a recursive object, but will use [Seed.Recursive.empty] instead. */ var generateEmptyRecursiveForMissedTypes: Boolean = true, + + /** + * Limits maximum number of recursive seed modifications + */ + var maxNumberOfRecursiveSeedModifications: Int = 10, ) \ No newline at end of file diff --git a/utbot-junit-contest/build.gradle b/utbot-junit-contest/build.gradle index b1319b02aa..189d36e5f6 100644 --- a/utbot-junit-contest/build.gradle +++ b/utbot-junit-contest/build.gradle @@ -13,22 +13,30 @@ compileJava { compileTestJava { options.fork = true - options.forkOptions.executable = 'javac' options.compilerArgs << "-XDignore.symbol.file" } +def testProjects = [ + 'build/output/test/antlr', + 'build/output/test/codeforces', + 'build/output/test/fastjson-1.2.50', + 'build/output/test/fescar', + 'build/output/test/guava', + 'build/output/test/guava-26.0', + 'build/output/test/guava-30.0', + 'build/output/test/pdfbox', + 'build/output/test/seata', + 'build/output/test/seata-core-0.5.0', + 'build/output/test/spoon', + 'build/output/test/spoon-core-7.0.0', +] + sourceSets { test { java { - srcDir('build/output/test/antlr') - srcDir('build/output/test/custom') - srcDir('build/output/test/guava') - srcDir('build/output/test/fescar') - srcDir('build/output/test/pdfbox') - srcDir('build/output/test/seata') - srcDir('build/output/test/spoon') - srcDir('build/output/test/samples') - srcDir('build/output/test/utbottest') + testProjects.forEach { + srcDir(it) + } } } } @@ -39,7 +47,26 @@ test { } jacocoTestReport { + afterEvaluate { + def r = testProjects.collect { + fileTree(dir: it) + }.findAll { + it.dir.exists() + } + sourceDirectories.setFrom(r.collect {files(it) }) + classDirectories.setFrom( + r.collect { + fileTree(dir: it.dir.toPath().parent.resolveSibling("unzipped").resolve(it.dir.name)) + }.findAll { + it.dir.exists() + }.collect { + files(it) + } + ) + } + reports { + csv.enabled = true html.enabled = true } } diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt index 8d317ed4f2..e0812048a0 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt @@ -10,9 +10,6 @@ import org.utbot.common.isAbstract import org.utbot.engine.EngineController import org.utbot.framework.TestSelectionStrategyType import org.utbot.framework.UtSettings -import org.utbot.framework.codegen.domain.ForceStaticMocking -import org.utbot.framework.codegen.domain.StaticsMocking -import org.utbot.framework.codegen.domain.junitByVersion import org.utbot.framework.plugin.api.util.UtContext import org.utbot.framework.plugin.api.util.executableId import org.utbot.framework.plugin.api.util.id @@ -51,7 +48,7 @@ import kotlinx.coroutines.newSingleThreadContext import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withTimeoutOrNull import org.utbot.framework.SummariesGenerationType -import org.utbot.framework.codegen.domain.ProjectType +import org.utbot.framework.codegen.domain.* import org.utbot.framework.codegen.generator.CodeGenerator import org.utbot.framework.codegen.services.language.CgLanguageAssistant import org.utbot.framework.minimization.minimizeExecutions @@ -228,6 +225,7 @@ fun runGeneration( forceStaticMocking = forceStaticMocking, generateWarningsForStaticMocking = false, cgLanguageAssistant = CgLanguageAssistant.getByCodegenLanguage(CodegenLanguage.defaultItem), + runtimeExceptionTestsBehaviour = RuntimeExceptionTestsBehaviour.PASS, ) logger.info().measureTime({ "class ${cut.fqn}" }, { statsForClass }) { From 0656354d1bf1bd315d3928cd1b6c5cd1c6e81b5e Mon Sep 17 00:00:00 2001 From: Maksim Pelevin Date: Thu, 6 Jul 2023 12:32:16 +0300 Subject: [PATCH 4/7] Fallback provider should not prevent providers filtering when applied --- .../kotlin/org/utbot/fuzzing/Providers.kt | 61 ++++++++++++------- .../kotlin/org/utbot/fuzzing/ProvidersTest.kt | 21 +++++++ .../org/utbot/fuzzing/providers/Objects.kt | 4 +- 3 files changed, 62 insertions(+), 24 deletions(-) diff --git a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Providers.kt b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Providers.kt index 85ad2bbe5e..f64fb7d996 100644 --- a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Providers.kt +++ b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Providers.kt @@ -96,9 +96,9 @@ fun interface ValueProvider> { */ fun except(filter: (ValueProvider) -> Boolean): ValueProvider { return if (this is Combined) { - Combined(providers.filterNot(filter)) + Combined(providers.map { it.unwrapIfFallback() }.filterNot(filter)) } else { - Combined(if (filter(this)) emptyList() else listOf(this)) + Combined(if (filter(unwrapIfFallback())) emptyList() else listOf(this)) } } @@ -117,26 +117,7 @@ fun interface ValueProvider> { * Uses fallback value provider in case when 'this' one failed to generate any value. */ fun withFallback(fallback: ValueProvider) : ValueProvider { - val thisProvider = this - return object : ValueProvider { - override fun enrich(description: D, type: T, scope: Scope) { - thisProvider.enrich(description, type, scope) - // Enriching scope by fallback value provider in this point is not quite right, - // but it doesn't look as a problem right now. - fallback.enrich(description, type, scope) - } - - override fun generate(description: D, type: T): Sequence> { - val default = if (thisProvider.accept(type)) thisProvider.generate(description, type) else emptySequence() - return if (default.iterator().hasNext()) { - default - } else if (fallback.accept(type)) { - fallback.generate(description, type) - } else { - emptySequence() - } - } - } + return Fallback(this, fallback) } /** @@ -152,6 +133,42 @@ fun interface ValueProvider> { return if (flag) block(this) else this } + /** + * Checks if current provider has fallback and return the initial provider. + * + * If the initial provider is also fallback, then it is unwrapped too. + * + * @return unwrapped provider or this if it is not fallback + */ + fun unwrapIfFallback(): ValueProvider { + return if (this is Fallback) { provider.unwrapIfFallback() } else { this } + } + + private class Fallback>( + val provider: ValueProvider, + val fallback: ValueProvider, + ): ValueProvider { + + override fun enrich(description: D, type: T, scope: Scope) { + provider.enrich(description, type, scope) + // Enriching scope by fallback value provider in this point is not quite right, + // but it doesn't look as a problem right now. + fallback.enrich(description, type, scope) + } + + override fun generate(description: D, type: T): Sequence> { + val default = if (provider.accept(type)) provider.generate(description, type) else emptySequence() + return if (default.iterator().hasNext()) { + default + } else if (fallback.accept(type)) { + fallback.generate(description, type) + } else { + emptySequence() + } + } + + } + /** * Wrapper class that delegates implementation to the [providers]. */ diff --git a/utbot-fuzzing/src/test/kotlin/org/utbot/fuzzing/ProvidersTest.kt b/utbot-fuzzing/src/test/kotlin/org/utbot/fuzzing/ProvidersTest.kt index 9d306b9ad7..74b2d8c204 100644 --- a/utbot-fuzzing/src/test/kotlin/org/utbot/fuzzing/ProvidersTest.kt +++ b/utbot-fuzzing/src/test/kotlin/org/utbot/fuzzing/ProvidersTest.kt @@ -111,6 +111,27 @@ class ProvidersTest { Assertions.assertEquals(3, (seq.drop(1).first() as Seed.Simple).value) } + @Test + fun `test fallback unwrapping from providers`() { + val provider1 = p { 2 } + val provider2 = p { 3 } + val fallback = p { 4 } + val providers1 = ValueProvider.of(listOf( + provider1.withFallback(fallback), + provider2 + )) + val seq1 = providers1.generate(description, Unit).toSet() + Assertions.assertEquals(2, seq1.count()) + Assertions.assertEquals(2, (seq1.first() as Seed.Simple).value) + Assertions.assertEquals(3, (seq1.drop(1).first() as Seed.Simple).value) + + val providers2 = providers1.except(provider1) + + val seq2 = providers2.generate(description, Unit).toSet() + Assertions.assertEquals(1, seq2.count()) + Assertions.assertEquals(3, (seq2.first() as Seed.Simple).value) + } + @Test fun `provider is not called when accept-method returns false`() { val seq = ValueProvider.of(listOf( diff --git a/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/providers/Objects.kt b/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/providers/Objects.kt index d047128404..f5fe5c3e92 100644 --- a/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/providers/Objects.kt +++ b/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/providers/Objects.kt @@ -151,7 +151,7 @@ class CreateObjectAnywayValueProvider( yield(Seed.Recursive( construct = Routine.Create(emptyList()) { UtCompositeModel(idGenerator.createId(), type.classId, useMock).fuzzed { - summary = "some object" + summary = "Unsafe object" } }, modify = sequence { @@ -188,7 +188,7 @@ class CreateObjectAnywayValueProvider( }, empty = Routine.Empty { UtCompositeModel(idGenerator.createId(), type.classId, useMock).fuzzed { - summary = "some object" + summary = "Unsafe object" } } )) From 46ce93011ed63eb2c965cacf4feadae6dc24a80e Mon Sep 17 00:00:00 2001 From: Maksim Pelevin Date: Thu, 6 Jul 2023 13:16:31 +0300 Subject: [PATCH 5/7] Fix a problem with creation of some types --- .../org/utbot/fuzzing/providers/Objects.kt | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/providers/Objects.kt b/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/providers/Objects.kt index f5fe5c3e92..75030b32a1 100644 --- a/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/providers/Objects.kt +++ b/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/providers/Objects.kt @@ -11,16 +11,21 @@ import java.lang.reflect.Member import java.lang.reflect.Method import java.lang.reflect.Modifier +private fun isIgnored(type: ClassId): Boolean { + return type == stringClassId + || type == dateClassId + || type == NumberValueProvider.classId + || type.isCollectionOrMap + || type.isPrimitiveWrapper + || type.isEnum + || type.isAbstract + || (type.isInner && !type.isStatic) +} + class ObjectValueProvider( val idGenerator: IdGenerator, ) : ValueProvider { - private val unwantedConstructorsClasses = listOf( - stringClassId, - dateClassId, - NumberValueProvider.classId - ) - override fun accept(type: FuzzedType) = !isIgnored(type.classId) override fun generate( @@ -88,15 +93,6 @@ class ObjectValueProvider( ) } - private fun isIgnored(type: ClassId): Boolean { - return unwantedConstructorsClasses.contains(type) - || type.isCollectionOrMap - || type.isPrimitiveWrapper - || type.isEnum - || type.isAbstract - || (type.isInner && !type.isStatic) - } - private fun findTypesOfNonRecursiveConstructor(type: FuzzedType, packageName: String?): List { return type.classId.allConstructors .filter { isAccessible(it.constructor, packageName) } @@ -137,7 +133,7 @@ class CreateObjectAnywayValueProvider( val useMock: Boolean = false, ) : ValueProvider { - override fun accept(type: FuzzedType) = type.classId.isRefType + override fun accept(type: FuzzedType) = type.classId.isRefType && !isIgnored(type.classId) override fun generate(description: FuzzedDescription, type: FuzzedType) = sequence> { val methodCalls = description.constants.filter { From 5c82da0df180338f55a0b57e4602495353268b5a Mon Sep 17 00:00:00 2001 From: Maksim Pelevin Date: Mon, 17 Jul 2023 09:05:43 +0300 Subject: [PATCH 6/7] Fuzz objects for interfaces and abstract classes --- .../kotlin/org/utbot/framework/UtSettings.kt | 4 +- .../org/utbot/fuzzer/FuzzerFunctions.kt | 13 -- .../src/main/kotlin/org/utbot/fuzzing/Api.kt | 9 +- .../kotlin/org/utbot/fuzzing/JavaLanguage.kt | 4 +- .../org/utbot/fuzzing/providers/Objects.kt | 113 ++++++++---------- 5 files changed, 59 insertions(+), 84 deletions(-) diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt index 66bccdc6bc..ea7353a2b0 100644 --- a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt +++ b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt @@ -237,9 +237,9 @@ object UtSettings : AbstractSettings(logger, defaultKeyForSettingsPath, defaultS var fuzzingTimeoutInMillis: Long by getLongProperty(3_000L, 0, Long.MAX_VALUE) /** - * Fuzzer will create objects when they cannot be created by other providers (e.g. private classes) + * Find implementations of interfaces and abstract classes to fuzz. */ - var fuzzObjectWhenTheyCannotBeCreatedClean: Boolean by getBooleanProperty(false) + var fuzzingImplementationOfAbstractClasses: Boolean by getBooleanProperty(true) /** * Generate tests that treat possible overflows in arithmetic operations as errors diff --git a/utbot-framework/src/main/kotlin/org/utbot/fuzzer/FuzzerFunctions.kt b/utbot-framework/src/main/kotlin/org/utbot/fuzzer/FuzzerFunctions.kt index 4f2de61f8b..5f7fc57d9e 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/fuzzer/FuzzerFunctions.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/fuzzer/FuzzerFunctions.kt @@ -1,11 +1,9 @@ package org.utbot.fuzzer import mu.KotlinLogging -import org.utbot.framework.UtSettings import org.utbot.framework.plugin.api.classId import org.utbot.framework.plugin.api.util.* import org.utbot.framework.util.executableId -import org.utbot.fuzzing.providers.CreateObjectAnywayValueProvider import soot.BooleanType import soot.ByteType import soot.CharType @@ -48,7 +46,6 @@ fun collectConstantsForFuzzer(graph: ExceptionalUnitGraph): Set try { finder.find(graph, unit, value) @@ -249,16 +246,6 @@ private object DateFormatByVarStringConstant: ConstantsFinder { } } -private object AbstractMethodIsCalled: ConstantsFinder { - override fun find(graph: ExceptionalUnitGraph, unit: Unit, value: Value): List { - if (UtSettings.fuzzObjectWhenTheyCannotBeCreatedClean && value is JInterfaceInvokeExpr) { - // todo do a better way to add information about virtual method calls to providers - return listOf(FuzzedConcreteValue(value.method.javaClass.id, CreateObjectAnywayValueProvider::class, FuzzedContext.Call(value.method.executableId))) - } - return emptyList() - } -} - private object ConstantsAsIs: ConstantsFinder { override fun find(graph: ExceptionalUnitGraph, unit: Unit, value: Value): List { if (value !is Constant || value is NullConstant) return emptyList() diff --git a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt index 705484e531..6911dcb8dd 100644 --- a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt +++ b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt @@ -289,7 +289,14 @@ private object EmptyFeedback : Feedback { class NoSeedValueException internal constructor( // this type cannot be generalized because Java forbids types for [Throwable]. val type: Any? -) : Exception("No seed candidates generated for type: $type") +) : Exception() { + override fun fillInStackTrace(): Throwable { + return this + } + + override val message: String + get() = "No seed candidates generated for type: $type" +} suspend fun , F : Feedback> Fuzzing.fuzz( description: D, diff --git a/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt b/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt index 0c6b1de657..b55e567c68 100644 --- a/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt +++ b/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt @@ -43,8 +43,8 @@ fun defaultValueProviders(idGenerator: IdentityPreservingIdGenerator) = lis FloatValueProvider, StringValueProvider, NumberValueProvider, - ObjectValueProvider(idGenerator).letIf(UtSettings.fuzzObjectWhenTheyCannotBeCreatedClean) { ovp -> - ovp.withFallback(CreateObjectAnywayValueProvider(idGenerator, useMock = false)) + ObjectValueProvider(idGenerator).letIf(UtSettings.fuzzingImplementationOfAbstractClasses) { ovp -> + ovp.withFallback(AbstractsObjectValueProvider(idGenerator)) }, ArrayValueProvider(idGenerator), EnumValueProvider(idGenerator), diff --git a/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/providers/Objects.kt b/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/providers/Objects.kt index 75030b32a1..b356df138f 100644 --- a/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/providers/Objects.kt +++ b/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/providers/Objects.kt @@ -1,23 +1,28 @@ package org.utbot.fuzzing.providers -import org.utbot.common.isAbstract import org.utbot.framework.plugin.api.* import org.utbot.framework.plugin.api.util.* import org.utbot.fuzzer.* import org.utbot.fuzzing.* import org.utbot.fuzzing.utils.hex +import soot.Scene +import soot.SootClass import java.lang.reflect.Field import java.lang.reflect.Member import java.lang.reflect.Method import java.lang.reflect.Modifier -private fun isIgnored(type: ClassId): Boolean { +private fun isKnownTypes(type: ClassId): Boolean { return type == stringClassId || type == dateClassId || type == NumberValueProvider.classId || type.isCollectionOrMap || type.isPrimitiveWrapper || type.isEnum +} + +private fun isIgnored(type: ClassId): Boolean { + return isKnownTypes(type) || type.isAbstract || (type.isInner && !type.isStatic) } @@ -33,10 +38,8 @@ class ObjectValueProvider( type: FuzzedType ) = sequence { val classId = type.classId - val constructors = findTypesOfNonRecursiveConstructor(type, description.description.packageName) - .takeIf { it.isNotEmpty() } - ?.asSequence() - ?: classId.allConstructors.filter { + val constructors = classId.allConstructors + .filter { isAccessible(it.constructor, description.description.packageName) } constructors.forEach { constructorId -> @@ -92,14 +95,6 @@ class ObjectValueProvider( empty = nullRoutine(classId) ) } - - private fun findTypesOfNonRecursiveConstructor(type: FuzzedType, packageName: String?): List { - return type.classId.allConstructors - .filter { isAccessible(it.constructor, packageName) } - .filter { c -> - c.parameters.all { it.isPrimitive || it == stringClassId || it.isArray } - }.toList() - } } @Suppress("unused") @@ -128,66 +123,52 @@ object NullValueProvider : ValueProvider, - val useMock: Boolean = false, ) : ValueProvider { - override fun accept(type: FuzzedType) = type.classId.isRefType && !isIgnored(type.classId) + override fun accept(type: FuzzedType) = type.classId.isRefType && !isKnownTypes(type.classId) override fun generate(description: FuzzedDescription, type: FuzzedType) = sequence> { - val methodCalls = description.constants.filter { - it.value == CreateObjectAnywayValueProvider::class - }.mapNotNull { - it.fuzzedContext as? FuzzedContext.Call - }.map { - it.method - }.toSet() - - yield(Seed.Recursive( - construct = Routine.Create(emptyList()) { - UtCompositeModel(idGenerator.createId(), type.classId, useMock).fuzzed { - summary = "Unsafe object" + val t = Scene.v().getRefType(type.classId.canonicalName).sootClass + fun canCreateClass(sc: SootClass): Boolean { + try { + if (!sc.isConcrete) return false + val packageName = sc.packageName + if (packageName != null) { + if (packageName.startsWith("jdk.internal") || + packageName.startsWith("org.utbot") || + packageName.startsWith("sun.")) + return false } - }, - modify = sequence { - // generate all fields - generateSequence(type.classId.jClass) { - it.superclass - }.flatMap { javaClass -> - javaClass.declaredFields.toList() - }.forEach { field -> - yield(Routine.Call(listOf(toFuzzerType(field.type, description.typeCache))) { instance, args -> - (instance.model as UtCompositeModel).fields[field.fieldId] = args.first().model - }) - } - - generateSequence(listOf(type.classId.jClass)) { classList -> - classList.flatMap { listOf(it.superclass) + it.interfaces }.filterNotNull().takeIf { it.isNotEmpty() } - }.flatten().filter { - isAccessible(it, description.description.packageName) - }.flatMap { javaClass -> - javaClass.declaredMethods.filter { - javaClass.isInterface || it.isAbstract - }.filter { - isAccessible(it, description.description.packageName) - } - // todo filter by methods seen in code - }.forEach { method -> - val executableId = method.executableId - if (methodCalls.contains(executableId)) { - yield(Routine.Call(listOf(toFuzzerType(method.returnType, description.typeCache))) { instance, args -> - (instance.model as UtCompositeModel).mocks[executableId] = args.map(FuzzedValue::model) - }) - } - } - }, - empty = Routine.Empty { - UtCompositeModel(idGenerator.createId(), type.classId, useMock).fuzzed { - summary = "Unsafe object" + val isAnonymousClass = sc.name.matches(""".*\$\d+$""".toRegex()) + if (isAnonymousClass) { + return false } + val jClass = sc.id.jClass + return isAccessible(jClass, description.description.packageName) && + jClass.declaredConstructors.any { isAccessible(it, description.description.packageName) } + } catch (ignore: Throwable) { + return false } - )) + } + + val implementations = when { + t.isInterface -> Scene.v().fastHierarchy.getAllImplementersOfInterface(t).filter(::canCreateClass) + t.isAbstract -> Scene.v().fastHierarchy.getSubclassesOf(t).filter(::canCreateClass) + else -> emptyList() + } + implementations.shuffled(description.random).take(10).forEach { concrete -> + yield(Seed.Recursive( + construct = Routine.Create(listOf(toFuzzerType(concrete.id.jClass, description.typeCache))) { + it.first() + }, + empty = nullRoutine(type.classId) + )) + } } } From d0d93ec37f2ed655542317b793b7bd60fa5d0f49 Mon Sep 17 00:00:00 2001 From: Maksim Pelevin Date: Mon, 17 Jul 2023 13:26:43 +0300 Subject: [PATCH 7/7] Fix test in case when Soot is not available --- .../main/kotlin/org/utbot/fuzzing/providers/Objects.kt | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/providers/Objects.kt b/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/providers/Objects.kt index b356df138f..68ab56e678 100644 --- a/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/providers/Objects.kt +++ b/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/providers/Objects.kt @@ -1,5 +1,6 @@ package org.utbot.fuzzing.providers +import mu.KotlinLogging import org.utbot.framework.plugin.api.* import org.utbot.framework.plugin.api.util.* import org.utbot.fuzzer.* @@ -12,6 +13,8 @@ import java.lang.reflect.Member import java.lang.reflect.Method import java.lang.reflect.Modifier +private val logger = KotlinLogging.logger {} + private fun isKnownTypes(type: ClassId): Boolean { return type == stringClassId || type == dateClassId @@ -133,7 +136,12 @@ class AbstractsObjectValueProvider( override fun accept(type: FuzzedType) = type.classId.isRefType && !isKnownTypes(type.classId) override fun generate(description: FuzzedDescription, type: FuzzedType) = sequence> { - val t = Scene.v().getRefType(type.classId.canonicalName).sootClass + val t = try { + Scene.v().getRefType(type.classId.canonicalName).sootClass + } catch (ignore: NoClassDefFoundError) { + logger.error(ignore) { "Soot may be not initialized" } + return@sequence + } fun canCreateClass(sc: SootClass): Boolean { try { if (!sc.isConcrete) return false