Rerun Spring integration tests after minimization with full context reset between tests

utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt

@@ -782,19 +782,21 @@ private fun ResolvedModels.constructStateForMethod(methodUnderTest: ExecutableId
     return EnvironmentModels(thisInstanceBefore, paramsBefore, statics, methodUnderTest)
 }
 
-private suspend fun ConcreteExecutor<UtConcreteExecutionResult, Instrumentation<UtConcreteExecutionResult>>.executeConcretely(
+suspend fun ConcreteExecutor<UtConcreteExecutionResult, Instrumentation<UtConcreteExecutionResult>>.executeConcretely(
     methodUnderTest: ExecutableId,
     stateBefore: EnvironmentModels,
     instrumentation: List<UtInstrumentation>,
-    timeoutInMillis: Long
+    timeoutInMillis: Long,
+    isRerun: Boolean = false,
 ): UtConcreteExecutionResult = executeAsync(
     methodUnderTest.classId.name,
     methodUnderTest.signature,
     arrayOf(),
     parameters = UtConcreteExecutionData(
         stateBefore,
         instrumentation,
-        timeoutInMillis
+        timeoutInMillis,
+        isRerun,
     )
 ).convertToAssemble(methodUnderTest.classId.packageName)
 

utbot-framework/src/main/kotlin/org/utbot/external/api/UtBotJavaApi.kt

@@ -237,7 +237,8 @@ object UtBotJavaApi {
                 parameters = UtConcreteExecutionData(
                     testInfo.initialState,
                     instrumentation = emptyList(),
-                    UtSettings.concreteExecutionDefaultTimeoutInInstrumentedProcessMillis
+                    UtSettings.concreteExecutionDefaultTimeoutInInstrumentedProcessMillis,
+                    isRerun = false,
                 )
             ).result
         } else {

utbot-framework/src/main/kotlin/org/utbot/framework/context/ConcreteExecutionContext.kt

@@ -2,6 +2,7 @@ package org.utbot.framework.context
 
 import org.utbot.framework.plugin.api.ClassId
 import org.utbot.framework.plugin.api.ConcreteContextLoadingResult
+import org.utbot.framework.plugin.api.ExecutableId
 import org.utbot.framework.plugin.api.UtExecution
 import org.utbot.fuzzer.IdentityPreservingIdGenerator
 import org.utbot.instrumentation.ConcreteExecutor
@@ -17,7 +18,13 @@ interface ConcreteExecutionContext {
 
     fun transformExecutionsBeforeMinimization(
         executions: List<UtExecution>,
-        classUnderTestId: ClassId
+        methodUnderTest: ExecutableId,
+    ): List<UtExecution>
+
+    fun transformExecutionsAfterMinimization(
+        executions: List<UtExecution>,
+        methodUnderTest: ExecutableId,
+        rerunExecutor: ConcreteExecutor<UtConcreteExecutionResult, UtExecutionInstrumentation>,
     ): List<UtExecution>
 
     fun tryCreateFuzzingContext(

utbot-framework/src/main/kotlin/org/utbot/framework/context/custom/CoverageFilteringConcreteExecutionContext.kt

@@ -5,6 +5,7 @@ import org.utbot.common.hasOnClasspath
 import org.utbot.common.tryLoadClass
 import org.utbot.framework.context.ConcreteExecutionContext
 import org.utbot.framework.plugin.api.ClassId
+import org.utbot.framework.plugin.api.ExecutableId
 import org.utbot.framework.plugin.api.UtExecution
 import org.utbot.framework.plugin.api.util.utContext
 import java.io.File
@@ -39,8 +40,13 @@ class CoverageFilteringConcreteExecutionContext(
 
     override fun transformExecutionsBeforeMinimization(
         executions: List<UtExecution>,
-        classUnderTestId: ClassId
+        methodUnderTest: ExecutableId,
     ): List<UtExecution> {
+        @Suppress("NAME_SHADOWING")
+        val executions = delegateContext.transformExecutionsBeforeMinimization(executions, methodUnderTest)
+
+        val classUnderTestId = methodUnderTest.classId
+
         val annotationsToIgnoreCoverage =
             annotationsToIgnoreCoverage.mapNotNull { utContext.classLoader.tryLoadClass(it.name) }
 

utbot-framework/src/main/kotlin/org/utbot/framework/context/custom/RerunningConcreteExecutionContext.kt

@@ -0,0 +1,53 @@
+package org.utbot.framework.context.custom
+
+import kotlinx.coroutines.runBlocking
+import mu.KotlinLogging
+import org.utbot.engine.executeConcretely
+import org.utbot.framework.UtSettings
+import org.utbot.framework.context.ConcreteExecutionContext
+import org.utbot.framework.plugin.api.ExecutableId
+import org.utbot.framework.plugin.api.UtExecution
+import org.utbot.instrumentation.ConcreteExecutor
+import org.utbot.instrumentation.instrumentation.execution.UtConcreteExecutionResult
+import org.utbot.instrumentation.instrumentation.execution.UtExecutionInstrumentation
+
+class RerunningConcreteExecutionContext(
+    private val delegateContext: ConcreteExecutionContext,
+    private val rerunTimeoutInMillis: Long = 10L * UtSettings.concreteExecutionDefaultTimeoutInInstrumentedProcessMillis,
+) : ConcreteExecutionContext by delegateContext {
+    companion object {
+        private val logger = KotlinLogging.logger {}
+    }
+
+    override fun transformExecutionsAfterMinimization(
+        executions: List<UtExecution>,
+        methodUnderTest: ExecutableId,
+        rerunExecutor: ConcreteExecutor<UtConcreteExecutionResult, UtExecutionInstrumentation>,
+    ): List<UtExecution> = delegateContext.transformExecutionsAfterMinimization(
+        executions,
+        methodUnderTest,
+        rerunExecutor
+    ).map { execution ->
+        runBlocking {
+            val result = try {
+                rerunExecutor.executeConcretely(
+                    methodUnderTest = methodUnderTest,
+                    stateBefore = execution.stateBefore,
+                    instrumentation = emptyList(),
+                    timeoutInMillis = rerunTimeoutInMillis,
+                    isRerun = true,
+                )
+            } catch (e: Throwable) {
+                // we can't update execution result if we don't have a result
+                logger.warn(e) { "Rerun failed, keeping original result for execution [$execution]" }
+                return@runBlocking execution
+            }
+            execution.copy(
+                stateBefore = result.stateBefore,
+                stateAfter = result.stateAfter,
+                result = result.result,
+                coverage = result.coverage,
+            )
+        }
+    }
+}

utbot-framework/src/main/kotlin/org/utbot/framework/context/simple/SimpleConcreteExecutionContext.kt

@@ -4,6 +4,7 @@ import org.utbot.framework.context.ConcreteExecutionContext
 import org.utbot.framework.context.JavaFuzzingContext
 import org.utbot.framework.plugin.api.ClassId
 import org.utbot.framework.plugin.api.ConcreteContextLoadingResult
+import org.utbot.framework.plugin.api.ExecutableId
 import org.utbot.framework.plugin.api.UtExecution
 import org.utbot.fuzzer.IdentityPreservingIdGenerator
 import org.utbot.instrumentation.ConcreteExecutor
@@ -22,7 +23,13 @@ class SimpleConcreteExecutionContext(fullClassPath: String) : ConcreteExecutionC
 
     override fun transformExecutionsBeforeMinimization(
         executions: List<UtExecution>,
-        classUnderTestId: ClassId
+        methodUnderTest: ExecutableId,
+    ): List<UtExecution> = executions
+
+    override fun transformExecutionsAfterMinimization(
+        executions: List<UtExecution>,
+        methodUnderTest: ExecutableId,
+        rerunExecutor: ConcreteExecutor<UtConcreteExecutionResult, UtExecutionInstrumentation>,
     ): List<UtExecution> = executions
 
     override fun tryCreateFuzzingContext(

utbot-framework/src/main/kotlin/org/utbot/framework/plugin/api/TestCaseGenerator.kt

@@ -39,6 +39,8 @@ import org.utbot.framework.util.SootUtils
 import org.utbot.framework.util.jimpleBody
 import org.utbot.framework.util.toModel
 import org.utbot.instrumentation.ConcreteExecutor
+import org.utbot.instrumentation.instrumentation.execution.UtConcreteExecutionResult
+import org.utbot.instrumentation.instrumentation.execution.UtExecutionInstrumentation
 import org.utbot.instrumentation.warmup
 import org.utbot.taint.TaintConfigurationProvider
 import java.io.File
@@ -73,12 +75,12 @@ open class TestCaseGenerator(
 ) {
     private val logger: KLogger = KotlinLogging.logger {}
     private val timeoutLogger: KLogger = KotlinLogging.logger(logger.name + ".timeout")
-    private val concreteExecutionContext = applicationContext.createConcreteExecutionContext(
+    protected val concreteExecutionContext = applicationContext.createConcreteExecutionContext(
         fullClasspath = classpathForEngine,
         classpathWithoutDependencies = buildDirs.joinToString(File.pathSeparator)
     )
 
-    private val classpathForEngine: String
+    protected val classpathForEngine: String
         get() = (buildDirs + listOfNotNull(classpath)).joinToString(File.pathSeparator)
 
     init {
@@ -110,16 +112,24 @@ open class TestCaseGenerator(
         }
     }
 
-    fun minimizeExecutions(classUnderTestId: ClassId, executions: List<UtExecution>): List<UtExecution> =
+    fun minimizeExecutions(
+        methodUnderTest: ExecutableId,
+        executions: List<UtExecution>,
+        rerunExecutor: ConcreteExecutor<UtConcreteExecutionResult, UtExecutionInstrumentation>,
+    ): List<UtExecution> =
         when (UtSettings.testMinimizationStrategyType) {
             TestSelectionStrategyType.DO_NOT_MINIMIZE_STRATEGY -> executions
             TestSelectionStrategyType.COVERAGE_STRATEGY ->
-                minimizeTestCase(
-                    concreteExecutionContext.transformExecutionsBeforeMinimization(
-                        executions,
-                        classUnderTestId
+                concreteExecutionContext.transformExecutionsAfterMinimization(
+                    minimizeTestCase(
+                        concreteExecutionContext.transformExecutionsBeforeMinimization(
+                            executions,
+                            methodUnderTest,
+                        ),
+                        executionToTestSuite = { it.result::class.java }
                     ),
-                    executionToTestSuite = { it.result::class.java }
+                    methodUnderTest,
+                    rerunExecutor = rerunExecutor,
                 )
         }
 
@@ -283,7 +293,11 @@ open class TestCaseGenerator(
         return@use methods.map { method ->
             UtMethodTestSet(
                 method,
-                minimizeExecutions(method.classId, method2executions.getValue(method)),
+                minimizeExecutions(
+                    method,
+                    method2executions.getValue(method),
+                    rerunExecutor = ConcreteExecutor(concreteExecutionContext.instrumentationFactory, classpathForEngine)
+                ),
                 jimpleBody(method),
                 method2errors.getValue(method)
             )

utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/UtExecutionInstrumentation.kt

@@ -16,12 +16,16 @@ import org.utbot.instrumentation.instrumentation.execution.phases.PhasesControll
  * @property [stateBefore] is necessary for construction of parameters of a concrete call.
  * @property [instrumentation] is necessary for mocking static methods and new instances.
  * @property [timeout] is timeout for specific concrete execution (in milliseconds).
+ * @property [isRerun] reruns can be used to obtain more reproducible results (e.g. on clean Spring application context),
+ * rerun can take more time due to context reinitialisation.
+ *
  * By default is initialized from [UtSettings.concreteExecutionDefaultTimeoutInInstrumentedProcessMillis]
  */
 data class UtConcreteExecutionData(
     val stateBefore: EnvironmentModels,
     val instrumentation: List<UtInstrumentation>,
-    val timeout: Long
+    val timeout: Long,
+    val isRerun: Boolean,
 )
 
 fun UtConcreteExecutionData.mapModels(mapper: UtModelMapper) = copy(

utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/constructors/UtModelConstructor.kt

@@ -41,7 +41,7 @@ class UtModelConstructor(
     private val constructedObjects = IdentityHashMap<Any, UtModel>()
 
     companion object {
-        private const val DEFAULT_MAX_DEPTH = 7L
+        const val DEFAULT_MAX_DEPTH = 7L
 
         fun createOnlyUserClassesConstructor(
             pathsToUserClasses: Set<String>,

utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/phases/ModelConstructionPhase.kt

@@ -40,21 +40,28 @@ class ModelConstructionPhase(
         }
     }
 
+    private val constructorConfiguration = ConstructorConfiguration()
     private lateinit var constructor: UtModelConstructor
 
     class ConstructorConfiguration {
         lateinit var cache: IdentityHashMap<Any, UtModel>
         lateinit var strategy: UtCompositeModelStrategy
+        var maxDepth: Long = UtModelConstructor.DEFAULT_MAX_DEPTH
+    }
+
+    fun preconfigureConstructor(block: ConstructorConfiguration.() -> Unit) {
+        constructorConfiguration.block()
     }
 
     fun configureConstructor(block: ConstructorConfiguration.() -> Unit) {
-        ConstructorConfiguration().run {
+        constructorConfiguration.run {
             block()
             constructor = UtModelConstructor(
                 objectToModelCache = cache,
                 utModelWithCompositeOriginConstructorFinder = utModelWithCompositeOriginConstructorFinder,
                 compositeModelStrategy = strategy,
                 idGenerator = idGenerator,
+                maxDepth = maxDepth,
             )
         }
     }

utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/spring/SpringUtExecutionInstrumentation.kt

@@ -90,6 +90,9 @@ class SpringUtExecutionInstrumentation(
             throw IllegalArgumentException("Argument parameters must be of type UtConcreteExecutionData, but was: ${parameters?.javaClass}")
         }
 
+        if (parameters.isRerun)
+            springApi.resetContext()
+
         // `RemovingConstructFailsUtExecutionInstrumentation` may detect that we fail to
         // construct `RequestBuilder` and use `requestBuilder = null`, leading to a nonsensical
         // test `mockMvc.perform((RequestBuilder) null)`, which we should discard
@@ -104,6 +107,9 @@ class SpringUtExecutionInstrumentation(
 
         return try {
             delegateInstrumentation.invoke(clazz, methodSignature, arguments, parameters) { invokeBasePhases ->
+                if (!parameters.isRerun)
+                    modelConstructionPhase.preconfigureConstructor { maxDepth = 0 }
+
                 phasesWrapper {
                     // NB! beforeTestMethod() and afterTestMethod() are intentionally called inside phases,
                     //     so they are executed in one thread with method under test

utbot-spring-commons-api/src/main/kotlin/org/utbot/spring/api/SpringApi.kt

@@ -38,6 +38,11 @@ interface SpringApi {
      * because transactions are bound to threads
      */
     fun afterTestMethod()
+
+    /**
+     * Time-consuming operation that fully resets Spring context
+     */
+    fun resetContext()
 }
 
 data class RepositoryDescription(

utbot-spring-commons/src/main/kotlin/org/utbot/spring/SpringApiImpl.kt

@@ -162,6 +162,11 @@ class SpringApiImpl(
         isInsideTestMethod = false
     }
 
+    override fun resetContext() {
+        testContextManager.testContext.markApplicationContextDirty(null)
+        getOrLoadSpringApplicationContext()
+    }
+
     private fun describesRepository(bean: Any): Boolean =
         try {
             bean is Repository<*, *>

utbot-spring-framework/src/main/kotlin/org/utbot/framework/context/spring/SpringApplicationContextImpl.kt

@@ -11,6 +11,7 @@ import org.utbot.framework.context.ConcreteExecutionContext
 import org.utbot.framework.context.NonNullSpeculator
 import org.utbot.framework.context.TypeReplacer
 import org.utbot.framework.context.custom.CoverageFilteringConcreteExecutionContext
+import org.utbot.framework.context.custom.RerunningConcreteExecutionContext
 import org.utbot.framework.context.custom.mockAllTypesWithoutSpecificValueProvider
 import org.utbot.framework.context.utils.transformJavaFuzzingContext
 import org.utbot.framework.context.utils.withValueProvider
@@ -72,11 +73,14 @@ class SpringApplicationContextImpl(
                     )
                     .mockAllTypesWithoutSpecificValueProvider()
             }
-            SpringTestType.INTEGRATION_TEST -> SpringIntegrationTestConcreteExecutionContext(
-                delegateConcreteExecutionContext,
-                classpathWithoutDependencies,
-                this
-            )
+            SpringTestType.INTEGRATION_TEST ->
+                RerunningConcreteExecutionContext(
+                    SpringIntegrationTestConcreteExecutionContext(
+                        delegateConcreteExecutionContext,
+                        classpathWithoutDependencies,
+                        springApplicationContext = this
+                    )
+                )
         }
     }
 

utbot-spring-framework/src/main/kotlin/org/utbot/framework/context/spring/SpringIntegrationTestConcreteExecutionContext.kt

@@ -6,7 +6,6 @@ import org.utbot.framework.context.JavaFuzzingContext
 import org.utbot.framework.plugin.api.ClassId
 import org.utbot.framework.plugin.api.ConcreteContextLoadingResult
 import org.utbot.framework.plugin.api.SpringSettings
-import org.utbot.framework.plugin.api.UtExecution
 import org.utbot.fuzzer.IdentityPreservingIdGenerator
 import org.utbot.instrumentation.ConcreteExecutor
 import org.utbot.instrumentation.getRelevantSpringRepositories
@@ -21,7 +20,7 @@ class SpringIntegrationTestConcreteExecutionContext(
     private val delegateContext: ConcreteExecutionContext,
     classpathWithoutDependencies: String,
     private val springApplicationContext: SpringApplicationContext,
-) : ConcreteExecutionContext {
+) : ConcreteExecutionContext by delegateContext {
     private val springSettings = (springApplicationContext.springSettings as? SpringSettings.PresentSpringSettings) ?:
         error("Integration tests cannot be generated without Spring configuration")
 
@@ -50,11 +49,6 @@ class SpringIntegrationTestConcreteExecutionContext(
             }
         }
 
-    override fun transformExecutionsBeforeMinimization(
-        executions: List<UtExecution>,
-        classUnderTestId: ClassId
-    ): List<UtExecution> = delegateContext.transformExecutionsBeforeMinimization(executions, classUnderTestId)
-
     override fun tryCreateFuzzingContext(
         concreteExecutor: ConcreteExecutor<UtConcreteExecutionResult, UtExecutionInstrumentation>,
         classUnderTest: ClassId,