From bd21af8252b8c7fd28784d0cefa2b34489788285 Mon Sep 17 00:00:00 2001 From: IlyaMuravjov Date: Thu, 31 Aug 2023 18:50:39 +0300 Subject: [PATCH 1/2] Rerun Spring integration tests on clean context after minimization --- .../org/utbot/engine/UtBotSymbolicEngine.kt | 8 +-- .../org/utbot/external/api/UtBotJavaApi.kt | 3 +- .../context/ConcreteExecutionContext.kt | 9 +++- ...verageFilteringConcreteExecutionContext.kt | 8 ++- .../RerunningConcreteExecutionContext.kt | 53 +++++++++++++++++++ .../simple/SimpleConcreteExecutionContext.kt | 9 +++- .../framework/plugin/api/TestCaseGenerator.kt | 32 +++++++---- .../execution/UtExecutionInstrumentation.kt | 6 ++- .../SpringUtExecutionInstrumentation.kt | 3 ++ .../kotlin/org/utbot/spring/api/SpringApi.kt | 5 ++ .../kotlin/org/utbot/spring/SpringApiImpl.kt | 5 ++ .../spring/SpringApplicationContextImpl.kt | 14 +++-- ...IntegrationTestConcreteExecutionContext.kt | 8 +-- .../testing/TestSpecificTestCaseGenerator.kt | 7 ++- 14 files changed, 140 insertions(+), 30 deletions(-) create mode 100644 utbot-framework/src/main/kotlin/org/utbot/framework/context/custom/RerunningConcreteExecutionContext.kt diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt index 1cdde48e30..494e189a99 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt @@ -782,11 +782,12 @@ private fun ResolvedModels.constructStateForMethod(methodUnderTest: ExecutableId return EnvironmentModels(thisInstanceBefore, paramsBefore, statics, methodUnderTest) } -private suspend fun ConcreteExecutor>.executeConcretely( +suspend fun ConcreteExecutor>.executeConcretely( methodUnderTest: ExecutableId, stateBefore: EnvironmentModels, instrumentation: List, - timeoutInMillis: Long + timeoutInMillis: Long, + isRerun: Boolean = false, ): UtConcreteExecutionResult = executeAsync( methodUnderTest.classId.name, methodUnderTest.signature, @@ -794,7 +795,8 @@ private suspend fun ConcreteExecutor, - classUnderTestId: ClassId + methodUnderTest: ExecutableId, + ): List + + fun transformExecutionsAfterMinimization( + executions: List, + methodUnderTest: ExecutableId, + rerunExecutor: ConcreteExecutor, ): List fun tryCreateFuzzingContext( diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/context/custom/CoverageFilteringConcreteExecutionContext.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/context/custom/CoverageFilteringConcreteExecutionContext.kt index b7ff805157..8fa66a3f7f 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/framework/context/custom/CoverageFilteringConcreteExecutionContext.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/framework/context/custom/CoverageFilteringConcreteExecutionContext.kt @@ -5,6 +5,7 @@ import org.utbot.common.hasOnClasspath import org.utbot.common.tryLoadClass import org.utbot.framework.context.ConcreteExecutionContext import org.utbot.framework.plugin.api.ClassId +import org.utbot.framework.plugin.api.ExecutableId import org.utbot.framework.plugin.api.UtExecution import org.utbot.framework.plugin.api.util.utContext import java.io.File @@ -39,8 +40,13 @@ class CoverageFilteringConcreteExecutionContext( override fun transformExecutionsBeforeMinimization( executions: List, - classUnderTestId: ClassId + methodUnderTest: ExecutableId, ): List { + @Suppress("NAME_SHADOWING") + val executions = delegateContext.transformExecutionsBeforeMinimization(executions, methodUnderTest) + + val classUnderTestId = methodUnderTest.classId + val annotationsToIgnoreCoverage = annotationsToIgnoreCoverage.mapNotNull { utContext.classLoader.tryLoadClass(it.name) } diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/context/custom/RerunningConcreteExecutionContext.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/context/custom/RerunningConcreteExecutionContext.kt new file mode 100644 index 0000000000..a4b33910e3 --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/framework/context/custom/RerunningConcreteExecutionContext.kt @@ -0,0 +1,53 @@ +package org.utbot.framework.context.custom + +import kotlinx.coroutines.runBlocking +import mu.KotlinLogging +import org.utbot.engine.executeConcretely +import org.utbot.framework.UtSettings +import org.utbot.framework.context.ConcreteExecutionContext +import org.utbot.framework.plugin.api.ExecutableId +import org.utbot.framework.plugin.api.UtExecution +import org.utbot.instrumentation.ConcreteExecutor +import org.utbot.instrumentation.instrumentation.execution.UtConcreteExecutionResult +import org.utbot.instrumentation.instrumentation.execution.UtExecutionInstrumentation + +class RerunningConcreteExecutionContext( + private val delegateContext: ConcreteExecutionContext, + private val rerunTimeoutInMillis: Long = 10L * UtSettings.concreteExecutionDefaultTimeoutInInstrumentedProcessMillis, +) : ConcreteExecutionContext by delegateContext { + companion object { + private val logger = KotlinLogging.logger {} + } + + override fun transformExecutionsAfterMinimization( + executions: List, + methodUnderTest: ExecutableId, + rerunExecutor: ConcreteExecutor, + ): List = delegateContext.transformExecutionsAfterMinimization( + executions, + methodUnderTest, + rerunExecutor + ).map { execution -> + runBlocking { + val result = try { + rerunExecutor.executeConcretely( + methodUnderTest = methodUnderTest, + stateBefore = execution.stateBefore, + instrumentation = emptyList(), + timeoutInMillis = rerunTimeoutInMillis, + isRerun = true, + ) + } catch (e: Throwable) { + // we can't update execution result if we don't have a result + logger.warn(e) { "Rerun failed, keeping original result for execution [$execution]" } + return@runBlocking execution + } + execution.copy( + stateBefore = result.stateBefore, + stateAfter = result.stateAfter, + result = result.result, + coverage = result.coverage, + ) + } + } +} \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/context/simple/SimpleConcreteExecutionContext.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/context/simple/SimpleConcreteExecutionContext.kt index 503b6fee93..0e411d9d32 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/framework/context/simple/SimpleConcreteExecutionContext.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/framework/context/simple/SimpleConcreteExecutionContext.kt @@ -4,6 +4,7 @@ import org.utbot.framework.context.ConcreteExecutionContext import org.utbot.framework.context.JavaFuzzingContext import org.utbot.framework.plugin.api.ClassId import org.utbot.framework.plugin.api.ConcreteContextLoadingResult +import org.utbot.framework.plugin.api.ExecutableId import org.utbot.framework.plugin.api.UtExecution import org.utbot.fuzzer.IdentityPreservingIdGenerator import org.utbot.instrumentation.ConcreteExecutor @@ -22,7 +23,13 @@ class SimpleConcreteExecutionContext(fullClassPath: String) : ConcreteExecutionC override fun transformExecutionsBeforeMinimization( executions: List, - classUnderTestId: ClassId + methodUnderTest: ExecutableId, + ): List = executions + + override fun transformExecutionsAfterMinimization( + executions: List, + methodUnderTest: ExecutableId, + rerunExecutor: ConcreteExecutor, ): List = executions override fun tryCreateFuzzingContext( diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/plugin/api/TestCaseGenerator.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/plugin/api/TestCaseGenerator.kt index a04cad2149..634a66f6e8 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/framework/plugin/api/TestCaseGenerator.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/framework/plugin/api/TestCaseGenerator.kt @@ -39,6 +39,8 @@ import org.utbot.framework.util.SootUtils import org.utbot.framework.util.jimpleBody import org.utbot.framework.util.toModel import org.utbot.instrumentation.ConcreteExecutor +import org.utbot.instrumentation.instrumentation.execution.UtConcreteExecutionResult +import org.utbot.instrumentation.instrumentation.execution.UtExecutionInstrumentation import org.utbot.instrumentation.warmup import org.utbot.taint.TaintConfigurationProvider import java.io.File @@ -73,12 +75,12 @@ open class TestCaseGenerator( ) { private val logger: KLogger = KotlinLogging.logger {} private val timeoutLogger: KLogger = KotlinLogging.logger(logger.name + ".timeout") - private val concreteExecutionContext = applicationContext.createConcreteExecutionContext( + protected val concreteExecutionContext = applicationContext.createConcreteExecutionContext( fullClasspath = classpathForEngine, classpathWithoutDependencies = buildDirs.joinToString(File.pathSeparator) ) - private val classpathForEngine: String + protected val classpathForEngine: String get() = (buildDirs + listOfNotNull(classpath)).joinToString(File.pathSeparator) init { @@ -110,16 +112,24 @@ open class TestCaseGenerator( } } - fun minimizeExecutions(classUnderTestId: ClassId, executions: List): List = + fun minimizeExecutions( + methodUnderTest: ExecutableId, + executions: List, + rerunExecutor: ConcreteExecutor, + ): List = when (UtSettings.testMinimizationStrategyType) { TestSelectionStrategyType.DO_NOT_MINIMIZE_STRATEGY -> executions TestSelectionStrategyType.COVERAGE_STRATEGY -> - minimizeTestCase( - concreteExecutionContext.transformExecutionsBeforeMinimization( - executions, - classUnderTestId + concreteExecutionContext.transformExecutionsAfterMinimization( + minimizeTestCase( + concreteExecutionContext.transformExecutionsBeforeMinimization( + executions, + methodUnderTest, + ), + executionToTestSuite = { it.result::class.java } ), - executionToTestSuite = { it.result::class.java } + methodUnderTest, + rerunExecutor = rerunExecutor, ) } @@ -283,7 +293,11 @@ open class TestCaseGenerator( return@use methods.map { method -> UtMethodTestSet( method, - minimizeExecutions(method.classId, method2executions.getValue(method)), + minimizeExecutions( + method, + method2executions.getValue(method), + rerunExecutor = ConcreteExecutor(concreteExecutionContext.instrumentationFactory, classpathForEngine) + ), jimpleBody(method), method2errors.getValue(method) ) diff --git a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/UtExecutionInstrumentation.kt b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/UtExecutionInstrumentation.kt index 9b5217df55..3986bef965 100644 --- a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/UtExecutionInstrumentation.kt +++ b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/UtExecutionInstrumentation.kt @@ -16,12 +16,16 @@ import org.utbot.instrumentation.instrumentation.execution.phases.PhasesControll * @property [stateBefore] is necessary for construction of parameters of a concrete call. * @property [instrumentation] is necessary for mocking static methods and new instances. * @property [timeout] is timeout for specific concrete execution (in milliseconds). + * @property [isRerun] reruns can be used to obtain more reproducible results (e.g. on clean Spring application context), + * rerun can take more time due to context reinitialisation. + * * By default is initialized from [UtSettings.concreteExecutionDefaultTimeoutInInstrumentedProcessMillis] */ data class UtConcreteExecutionData( val stateBefore: EnvironmentModels, val instrumentation: List, - val timeout: Long + val timeout: Long, + val isRerun: Boolean, ) fun UtConcreteExecutionData.mapModels(mapper: UtModelMapper) = copy( diff --git a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/spring/SpringUtExecutionInstrumentation.kt b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/spring/SpringUtExecutionInstrumentation.kt index c12c897ecb..2f760128c8 100644 --- a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/spring/SpringUtExecutionInstrumentation.kt +++ b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/spring/SpringUtExecutionInstrumentation.kt @@ -90,6 +90,9 @@ class SpringUtExecutionInstrumentation( throw IllegalArgumentException("Argument parameters must be of type UtConcreteExecutionData, but was: ${parameters?.javaClass}") } + if (parameters.isRerun) + springApi.resetContext() + // `RemovingConstructFailsUtExecutionInstrumentation` may detect that we fail to // construct `RequestBuilder` and use `requestBuilder = null`, leading to a nonsensical // test `mockMvc.perform((RequestBuilder) null)`, which we should discard diff --git a/utbot-spring-commons-api/src/main/kotlin/org/utbot/spring/api/SpringApi.kt b/utbot-spring-commons-api/src/main/kotlin/org/utbot/spring/api/SpringApi.kt index a866078eba..c856b00c92 100644 --- a/utbot-spring-commons-api/src/main/kotlin/org/utbot/spring/api/SpringApi.kt +++ b/utbot-spring-commons-api/src/main/kotlin/org/utbot/spring/api/SpringApi.kt @@ -38,6 +38,11 @@ interface SpringApi { * because transactions are bound to threads */ fun afterTestMethod() + + /** + * Time-consuming operation that fully resets Spring context + */ + fun resetContext() } data class RepositoryDescription( diff --git a/utbot-spring-commons/src/main/kotlin/org/utbot/spring/SpringApiImpl.kt b/utbot-spring-commons/src/main/kotlin/org/utbot/spring/SpringApiImpl.kt index 7e884f90b6..9bf806e7bb 100644 --- a/utbot-spring-commons/src/main/kotlin/org/utbot/spring/SpringApiImpl.kt +++ b/utbot-spring-commons/src/main/kotlin/org/utbot/spring/SpringApiImpl.kt @@ -162,6 +162,11 @@ class SpringApiImpl( isInsideTestMethod = false } + override fun resetContext() { + testContextManager.testContext.markApplicationContextDirty(null) + getOrLoadSpringApplicationContext() + } + private fun describesRepository(bean: Any): Boolean = try { bean is Repository<*, *> diff --git a/utbot-spring-framework/src/main/kotlin/org/utbot/framework/context/spring/SpringApplicationContextImpl.kt b/utbot-spring-framework/src/main/kotlin/org/utbot/framework/context/spring/SpringApplicationContextImpl.kt index f17bf1086a..9bddff2a52 100644 --- a/utbot-spring-framework/src/main/kotlin/org/utbot/framework/context/spring/SpringApplicationContextImpl.kt +++ b/utbot-spring-framework/src/main/kotlin/org/utbot/framework/context/spring/SpringApplicationContextImpl.kt @@ -11,6 +11,7 @@ import org.utbot.framework.context.ConcreteExecutionContext import org.utbot.framework.context.NonNullSpeculator import org.utbot.framework.context.TypeReplacer import org.utbot.framework.context.custom.CoverageFilteringConcreteExecutionContext +import org.utbot.framework.context.custom.RerunningConcreteExecutionContext import org.utbot.framework.context.custom.mockAllTypesWithoutSpecificValueProvider import org.utbot.framework.context.utils.transformJavaFuzzingContext import org.utbot.framework.context.utils.withValueProvider @@ -72,11 +73,14 @@ class SpringApplicationContextImpl( ) .mockAllTypesWithoutSpecificValueProvider() } - SpringTestType.INTEGRATION_TEST -> SpringIntegrationTestConcreteExecutionContext( - delegateConcreteExecutionContext, - classpathWithoutDependencies, - this - ) + SpringTestType.INTEGRATION_TEST -> + RerunningConcreteExecutionContext( + SpringIntegrationTestConcreteExecutionContext( + delegateConcreteExecutionContext, + classpathWithoutDependencies, + springApplicationContext = this + ) + ) } } diff --git a/utbot-spring-framework/src/main/kotlin/org/utbot/framework/context/spring/SpringIntegrationTestConcreteExecutionContext.kt b/utbot-spring-framework/src/main/kotlin/org/utbot/framework/context/spring/SpringIntegrationTestConcreteExecutionContext.kt index 2d4c5b8169..787f21403b 100644 --- a/utbot-spring-framework/src/main/kotlin/org/utbot/framework/context/spring/SpringIntegrationTestConcreteExecutionContext.kt +++ b/utbot-spring-framework/src/main/kotlin/org/utbot/framework/context/spring/SpringIntegrationTestConcreteExecutionContext.kt @@ -6,7 +6,6 @@ import org.utbot.framework.context.JavaFuzzingContext import org.utbot.framework.plugin.api.ClassId import org.utbot.framework.plugin.api.ConcreteContextLoadingResult import org.utbot.framework.plugin.api.SpringSettings -import org.utbot.framework.plugin.api.UtExecution import org.utbot.fuzzer.IdentityPreservingIdGenerator import org.utbot.instrumentation.ConcreteExecutor import org.utbot.instrumentation.getRelevantSpringRepositories @@ -21,7 +20,7 @@ class SpringIntegrationTestConcreteExecutionContext( private val delegateContext: ConcreteExecutionContext, classpathWithoutDependencies: String, private val springApplicationContext: SpringApplicationContext, -) : ConcreteExecutionContext { +) : ConcreteExecutionContext by delegateContext { private val springSettings = (springApplicationContext.springSettings as? SpringSettings.PresentSpringSettings) ?: error("Integration tests cannot be generated without Spring configuration") @@ -50,11 +49,6 @@ class SpringIntegrationTestConcreteExecutionContext( } } - override fun transformExecutionsBeforeMinimization( - executions: List, - classUnderTestId: ClassId - ): List = delegateContext.transformExecutionsBeforeMinimization(executions, classUnderTestId) - override fun tryCreateFuzzingContext( concreteExecutor: ConcreteExecutor, classUnderTest: ClassId, diff --git a/utbot-testing/src/main/kotlin/org/utbot/testing/TestSpecificTestCaseGenerator.kt b/utbot-testing/src/main/kotlin/org/utbot/testing/TestSpecificTestCaseGenerator.kt index 4f2af2dccd..5580b593ee 100644 --- a/utbot-testing/src/main/kotlin/org/utbot/testing/TestSpecificTestCaseGenerator.kt +++ b/utbot-testing/src/main/kotlin/org/utbot/testing/TestSpecificTestCaseGenerator.kt @@ -22,6 +22,7 @@ import org.utbot.framework.plugin.api.util.id import org.utbot.framework.plugin.services.JdkInfoDefaultProvider import org.utbot.framework.util.Conflict import org.utbot.framework.util.jimpleBody +import org.utbot.instrumentation.ConcreteExecutor import org.utbot.taint.TaintConfigurationProvider import java.nio.file.Path @@ -103,7 +104,11 @@ class TestSpecificTestCaseGenerator( forceMockListener.detach(this, forceMockListener) forceStaticMockListener.detach(this, forceStaticMockListener) - val minimizedExecutions = super.minimizeExecutions(method.classId, executions) + val minimizedExecutions = super.minimizeExecutions( + method, + executions, + rerunExecutor = ConcreteExecutor(concreteExecutionContext.instrumentationFactory, classpathForEngine) + ) return UtMethodTestSet(method, minimizedExecutions, jimpleBody(method), errors) } } \ No newline at end of file From 472a97141367887c5b94d5919cba5c59cd26ca60 Mon Sep 17 00:00:00 2001 From: IlyaMuravjov Date: Fri, 1 Sep 2023 12:53:10 +0300 Subject: [PATCH 2/2] Construct shallow result models during non-rerun runs --- .../execution/constructors/UtModelConstructor.kt | 2 +- .../execution/phases/ModelConstructionPhase.kt | 9 ++++++++- .../spring/SpringUtExecutionInstrumentation.kt | 3 +++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/constructors/UtModelConstructor.kt b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/constructors/UtModelConstructor.kt index dd371a6e25..14f4cb16bf 100644 --- a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/constructors/UtModelConstructor.kt +++ b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/constructors/UtModelConstructor.kt @@ -41,7 +41,7 @@ class UtModelConstructor( private val constructedObjects = IdentityHashMap() companion object { - private const val DEFAULT_MAX_DEPTH = 7L + const val DEFAULT_MAX_DEPTH = 7L fun createOnlyUserClassesConstructor( pathsToUserClasses: Set, diff --git a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/phases/ModelConstructionPhase.kt b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/phases/ModelConstructionPhase.kt index 188915ff3b..0620fd67dc 100644 --- a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/phases/ModelConstructionPhase.kt +++ b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/phases/ModelConstructionPhase.kt @@ -40,21 +40,28 @@ class ModelConstructionPhase( } } + private val constructorConfiguration = ConstructorConfiguration() private lateinit var constructor: UtModelConstructor class ConstructorConfiguration { lateinit var cache: IdentityHashMap lateinit var strategy: UtCompositeModelStrategy + var maxDepth: Long = UtModelConstructor.DEFAULT_MAX_DEPTH + } + + fun preconfigureConstructor(block: ConstructorConfiguration.() -> Unit) { + constructorConfiguration.block() } fun configureConstructor(block: ConstructorConfiguration.() -> Unit) { - ConstructorConfiguration().run { + constructorConfiguration.run { block() constructor = UtModelConstructor( objectToModelCache = cache, utModelWithCompositeOriginConstructorFinder = utModelWithCompositeOriginConstructorFinder, compositeModelStrategy = strategy, idGenerator = idGenerator, + maxDepth = maxDepth, ) } } diff --git a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/spring/SpringUtExecutionInstrumentation.kt b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/spring/SpringUtExecutionInstrumentation.kt index 2f760128c8..ea61498c73 100644 --- a/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/spring/SpringUtExecutionInstrumentation.kt +++ b/utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/spring/SpringUtExecutionInstrumentation.kt @@ -107,6 +107,9 @@ class SpringUtExecutionInstrumentation( return try { delegateInstrumentation.invoke(clazz, methodSignature, arguments, parameters) { invokeBasePhases -> + if (!parameters.isRerun) + modelConstructionPhase.preconfigureConstructor { maxDepth = 0 } + phasesWrapper { // NB! beforeTestMethod() and afterTestMethod() are intentionally called inside phases, // so they are executed in one thread with method under test