Skip to content

Commit 45320ae

Browse files
IlyaMuravjovIlyaMuravjov
authored
Rerun Spring integration tests after minimization with full context reset between tests (#2574)
* Rerun Spring integration tests on clean context after minimization * Construct shallow result models during non-rerun runs
1 parent 1a605eb commit 45320ae

File tree

16 files changed

+152
-32
lines changed

16 files changed

+152
-32
lines changed

‎utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -782,19 +782,21 @@ private fun ResolvedModels.constructStateForMethod(methodUnderTest: ExecutableId
782782
return EnvironmentModels(thisInstanceBefore, paramsBefore, statics, methodUnderTest)
783783
}
784784

785-
private suspend fun ConcreteExecutor<UtConcreteExecutionResult, Instrumentation<UtConcreteExecutionResult>>.executeConcretely(
785+
suspend fun ConcreteExecutor<UtConcreteExecutionResult, Instrumentation<UtConcreteExecutionResult>>.executeConcretely(
786786
methodUnderTest: ExecutableId,
787787
stateBefore: EnvironmentModels,
788788
instrumentation: List<UtInstrumentation>,
789-
timeoutInMillis: Long
789+
timeoutInMillis: Long,
790+
isRerun: Boolean = false,
790791
): UtConcreteExecutionResult = executeAsync(
791792
methodUnderTest.classId.name,
792793
methodUnderTest.signature,
793794
arrayOf(),
794795
parameters = UtConcreteExecutionData(
795796
stateBefore,
796797
instrumentation,
797-
timeoutInMillis
798+
timeoutInMillis,
799+
isRerun,
798800
)
799801
).convertToAssemble(methodUnderTest.classId.packageName)
800802

‎utbot-framework/src/main/kotlin/org/utbot/external/api/UtBotJavaApi.kt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -237,7 +237,8 @@ object UtBotJavaApi {
237237
parameters = UtConcreteExecutionData(
238238
testInfo.initialState,
239239
instrumentation = emptyList(),
240-
UtSettings.concreteExecutionDefaultTimeoutInInstrumentedProcessMillis
240+
UtSettings.concreteExecutionDefaultTimeoutInInstrumentedProcessMillis,
241+
isRerun = false,
241242
)
242243
).result
243244
} else {

‎utbot-framework/src/main/kotlin/org/utbot/framework/context/ConcreteExecutionContext.kt

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package org.utbot.framework.context
22

33
import org.utbot.framework.plugin.api.ClassId
44
import org.utbot.framework.plugin.api.ConcreteContextLoadingResult
5+
import org.utbot.framework.plugin.api.ExecutableId
56
import org.utbot.framework.plugin.api.UtExecution
67
import org.utbot.fuzzer.IdentityPreservingIdGenerator
78
import org.utbot.instrumentation.ConcreteExecutor
@@ -17,7 +18,13 @@ interface ConcreteExecutionContext {
1718

1819
fun transformExecutionsBeforeMinimization(
1920
executions: List<UtExecution>,
20-
classUnderTestId: ClassId
21+
methodUnderTest: ExecutableId,
22+
): List<UtExecution>
23+
24+
fun transformExecutionsAfterMinimization(
25+
executions: List<UtExecution>,
26+
methodUnderTest: ExecutableId,
27+
rerunExecutor: ConcreteExecutor<UtConcreteExecutionResult, UtExecutionInstrumentation>,
2128
): List<UtExecution>
2229

2330
fun tryCreateFuzzingContext(

‎utbot-framework/src/main/kotlin/org/utbot/framework/context/custom/CoverageFilteringConcreteExecutionContext.kt

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import org.utbot.common.hasOnClasspath
55
import org.utbot.common.tryLoadClass
66
import org.utbot.framework.context.ConcreteExecutionContext
77
import org.utbot.framework.plugin.api.ClassId
8+
import org.utbot.framework.plugin.api.ExecutableId
89
import org.utbot.framework.plugin.api.UtExecution
910
import org.utbot.framework.plugin.api.util.utContext
1011
import java.io.File
@@ -39,8 +40,13 @@ class CoverageFilteringConcreteExecutionContext(
3940

4041
override fun transformExecutionsBeforeMinimization(
4142
executions: List<UtExecution>,
42-
classUnderTestId: ClassId
43+
methodUnderTest: ExecutableId,
4344
): List<UtExecution> {
45+
@Suppress("NAME_SHADOWING")
46+
val executions = delegateContext.transformExecutionsBeforeMinimization(executions, methodUnderTest)
47+
48+
val classUnderTestId = methodUnderTest.classId
49+
4450
val annotationsToIgnoreCoverage =
4551
annotationsToIgnoreCoverage.mapNotNull { utContext.classLoader.tryLoadClass(it.name) }
4652

‎utbot-framework/src/main/kotlin/org/utbot/framework/context/custom/RerunningConcreteExecutionContext.kt

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
package org.utbot.framework.context.custom
2+
3+
import kotlinx.coroutines.runBlocking
4+
import mu.KotlinLogging
5+
import org.utbot.engine.executeConcretely
6+
import org.utbot.framework.UtSettings
7+
import org.utbot.framework.context.ConcreteExecutionContext
8+
import org.utbot.framework.plugin.api.ExecutableId
9+
import org.utbot.framework.plugin.api.UtExecution
10+
import org.utbot.instrumentation.ConcreteExecutor
11+
import org.utbot.instrumentation.instrumentation.execution.UtConcreteExecutionResult
12+
import org.utbot.instrumentation.instrumentation.execution.UtExecutionInstrumentation
13+
14+
class RerunningConcreteExecutionContext(
15+
private val delegateContext: ConcreteExecutionContext,
16+
private val rerunTimeoutInMillis: Long = 10L * UtSettings.concreteExecutionDefaultTimeoutInInstrumentedProcessMillis,
17+
) : ConcreteExecutionContext by delegateContext {
18+
companion object {
19+
private val logger = KotlinLogging.logger {}
20+
}
21+
22+
override fun transformExecutionsAfterMinimization(
23+
executions: List<UtExecution>,
24+
methodUnderTest: ExecutableId,
25+
rerunExecutor: ConcreteExecutor<UtConcreteExecutionResult, UtExecutionInstrumentation>,
26+
): List<UtExecution> = delegateContext.transformExecutionsAfterMinimization(
27+
executions,
28+
methodUnderTest,
29+
rerunExecutor
30+
).map { execution ->
31+
runBlocking {
32+
val result = try {
33+
rerunExecutor.executeConcretely(
34+
methodUnderTest = methodUnderTest,
35+
stateBefore = execution.stateBefore,
36+
instrumentation = emptyList(),
37+
timeoutInMillis = rerunTimeoutInMillis,
38+
isRerun = true,
39+
)
40+
} catch (e: Throwable) {
41+
// we can't update execution result if we don't have a result
42+
logger.warn(e) { "Rerun failed, keeping original result for execution [$execution]" }
43+
return@runBlocking execution
44+
}
45+
execution.copy(
46+
stateBefore = result.stateBefore,
47+
stateAfter = result.stateAfter,
48+
result = result.result,
49+
coverage = result.coverage,
50+
)
51+
}
52+
}
53+
}

‎utbot-framework/src/main/kotlin/org/utbot/framework/context/simple/SimpleConcreteExecutionContext.kt

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import org.utbot.framework.context.ConcreteExecutionContext
44
import org.utbot.framework.context.JavaFuzzingContext
55
import org.utbot.framework.plugin.api.ClassId
66
import org.utbot.framework.plugin.api.ConcreteContextLoadingResult
7+
import org.utbot.framework.plugin.api.ExecutableId
78
import org.utbot.framework.plugin.api.UtExecution
89
import org.utbot.fuzzer.IdentityPreservingIdGenerator
910
import org.utbot.instrumentation.ConcreteExecutor
@@ -22,7 +23,13 @@ class SimpleConcreteExecutionContext(fullClassPath: String) : ConcreteExecutionC
2223

2324
override fun transformExecutionsBeforeMinimization(
2425
executions: List<UtExecution>,
25-
classUnderTestId: ClassId
26+
methodUnderTest: ExecutableId,
27+
): List<UtExecution> = executions
28+
29+
override fun transformExecutionsAfterMinimization(
30+
executions: List<UtExecution>,
31+
methodUnderTest: ExecutableId,
32+
rerunExecutor: ConcreteExecutor<UtConcreteExecutionResult, UtExecutionInstrumentation>,
2633
): List<UtExecution> = executions
2734

2835
override fun tryCreateFuzzingContext(

‎utbot-framework/src/main/kotlin/org/utbot/framework/plugin/api/TestCaseGenerator.kt

Lines changed: 23 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ import org.utbot.framework.util.SootUtils
3939
import org.utbot.framework.util.jimpleBody
4040
import org.utbot.framework.util.toModel
4141
import org.utbot.instrumentation.ConcreteExecutor
42+
import org.utbot.instrumentation.instrumentation.execution.UtConcreteExecutionResult
43+
import org.utbot.instrumentation.instrumentation.execution.UtExecutionInstrumentation
4244
import org.utbot.instrumentation.warmup
4345
import org.utbot.taint.TaintConfigurationProvider
4446
import java.io.File
@@ -73,12 +75,12 @@ open class TestCaseGenerator(
7375
) {
7476
private val logger: KLogger = KotlinLogging.logger {}
7577
private val timeoutLogger: KLogger = KotlinLogging.logger(logger.name + ".timeout")
76-
private val concreteExecutionContext = applicationContext.createConcreteExecutionContext(
78+
protected val concreteExecutionContext = applicationContext.createConcreteExecutionContext(
7779
fullClasspath = classpathForEngine,
7880
classpathWithoutDependencies = buildDirs.joinToString(File.pathSeparator)
7981
)
8082

81-
private val classpathForEngine: String
83+
protected val classpathForEngine: String
8284
get() = (buildDirs + listOfNotNull(classpath)).joinToString(File.pathSeparator)
8385

8486
init {
@@ -110,16 +112,24 @@ open class TestCaseGenerator(
110112
}
111113
}
112114

113-
fun minimizeExecutions(classUnderTestId: ClassId, executions: List<UtExecution>): List<UtExecution> =
115+
fun minimizeExecutions(
116+
methodUnderTest: ExecutableId,
117+
executions: List<UtExecution>,
118+
rerunExecutor: ConcreteExecutor<UtConcreteExecutionResult, UtExecutionInstrumentation>,
119+
): List<UtExecution> =
114120
when (UtSettings.testMinimizationStrategyType) {
115121
TestSelectionStrategyType.DO_NOT_MINIMIZE_STRATEGY -> executions
116122
TestSelectionStrategyType.COVERAGE_STRATEGY ->
117-
minimizeTestCase(
118-
concreteExecutionContext.transformExecutionsBeforeMinimization(
119-
executions,
120-
classUnderTestId
123+
concreteExecutionContext.transformExecutionsAfterMinimization(
124+
minimizeTestCase(
125+
concreteExecutionContext.transformExecutionsBeforeMinimization(
126+
executions,
127+
methodUnderTest,
128+
),
129+
executionToTestSuite = { it.result::class.java }
121130
),
122-
executionToTestSuite = { it.result::class.java }
131+
methodUnderTest,
132+
rerunExecutor = rerunExecutor,
123133
)
124134
}
125135

@@ -283,7 +293,11 @@ open class TestCaseGenerator(
283293
return@use methods.map { method ->
284294
UtMethodTestSet(
285295
method,
286-
minimizeExecutions(method.classId, method2executions.getValue(method)),
296+
minimizeExecutions(
297+
method,
298+
method2executions.getValue(method),
299+
rerunExecutor = ConcreteExecutor(concreteExecutionContext.instrumentationFactory, classpathForEngine)
300+
),
287301
jimpleBody(method),
288302
method2errors.getValue(method)
289303
)

‎utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/UtExecutionInstrumentation.kt

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,16 @@ import org.utbot.instrumentation.instrumentation.execution.phases.PhasesControll
1616
* @property [stateBefore] is necessary for construction of parameters of a concrete call.
1717
* @property [instrumentation] is necessary for mocking static methods and new instances.
1818
* @property [timeout] is timeout for specific concrete execution (in milliseconds).
19+
* @property [isRerun] reruns can be used to obtain more reproducible results (e.g. on clean Spring application context),
20+
* rerun can take more time due to context reinitialisation.
21+
*
1922
* By default is initialized from [UtSettings.concreteExecutionDefaultTimeoutInInstrumentedProcessMillis]
2023
*/
2124
data class UtConcreteExecutionData(
2225
val stateBefore: EnvironmentModels,
2326
val instrumentation: List<UtInstrumentation>,
24-
val timeout: Long
27+
val timeout: Long,
28+
val isRerun: Boolean,
2529
)
2630

2731
fun UtConcreteExecutionData.mapModels(mapper: UtModelMapper) = copy(

‎utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/constructors/UtModelConstructor.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ class UtModelConstructor(
4141
private val constructedObjects = IdentityHashMap<Any, UtModel>()
4242

4343
companion object {
44-
private const val DEFAULT_MAX_DEPTH = 7L
44+
const val DEFAULT_MAX_DEPTH = 7L
4545

4646
fun createOnlyUserClassesConstructor(
4747
pathsToUserClasses: Set<String>,

‎utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/phases/ModelConstructionPhase.kt

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,21 +40,28 @@ class ModelConstructionPhase(
4040
}
4141
}
4242

43+
private val constructorConfiguration = ConstructorConfiguration()
4344
private lateinit var constructor: UtModelConstructor
4445

4546
class ConstructorConfiguration {
4647
lateinit var cache: IdentityHashMap<Any, UtModel>
4748
lateinit var strategy: UtCompositeModelStrategy
49+
var maxDepth: Long = UtModelConstructor.DEFAULT_MAX_DEPTH
50+
}
51+
52+
fun preconfigureConstructor(block: ConstructorConfiguration.() -> Unit) {
53+
constructorConfiguration.block()
4854
}
4955

5056
fun configureConstructor(block: ConstructorConfiguration.() -> Unit) {
51-
ConstructorConfiguration().run {
57+
constructorConfiguration.run {
5258
block()
5359
constructor = UtModelConstructor(
5460
objectToModelCache = cache,
5561
utModelWithCompositeOriginConstructorFinder = utModelWithCompositeOriginConstructorFinder,
5662
compositeModelStrategy = strategy,
5763
idGenerator = idGenerator,
64+
maxDepth = maxDepth,
5865
)
5966
}
6067
}

‎utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/spring/SpringUtExecutionInstrumentation.kt

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,9 @@ class SpringUtExecutionInstrumentation(
9090
throw IllegalArgumentException("Argument parameters must be of type UtConcreteExecutionData, but was: ${parameters?.javaClass}")
9191
}
9292

93+
if (parameters.isRerun)
94+
springApi.resetContext()
95+
9396
// `RemovingConstructFailsUtExecutionInstrumentation` may detect that we fail to
9497
// construct `RequestBuilder` and use `requestBuilder = null`, leading to a nonsensical
9598
// test `mockMvc.perform((RequestBuilder) null)`, which we should discard
@@ -104,6 +107,9 @@ class SpringUtExecutionInstrumentation(
104107

105108
return try {
106109
delegateInstrumentation.invoke(clazz, methodSignature, arguments, parameters) { invokeBasePhases ->
110+
if (!parameters.isRerun)
111+
modelConstructionPhase.preconfigureConstructor { maxDepth = 0 }
112+
107113
phasesWrapper {
108114
// NB! beforeTestMethod() and afterTestMethod() are intentionally called inside phases,
109115
// so they are executed in one thread with method under test

‎utbot-spring-commons-api/src/main/kotlin/org/utbot/spring/api/SpringApi.kt

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,11 @@ interface SpringApi {
3838
* because transactions are bound to threads
3939
*/
4040
fun afterTestMethod()
41+
42+
/**
43+
* Time-consuming operation that fully resets Spring context
44+
*/
45+
fun resetContext()
4146
}
4247

4348
data class RepositoryDescription(

‎utbot-spring-commons/src/main/kotlin/org/utbot/spring/SpringApiImpl.kt

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,11 @@ class SpringApiImpl(
162162
isInsideTestMethod = false
163163
}
164164

165+
override fun resetContext() {
166+
testContextManager.testContext.markApplicationContextDirty(null)
167+
getOrLoadSpringApplicationContext()
168+
}
169+
165170
private fun describesRepository(bean: Any): Boolean =
166171
try {
167172
bean is Repository<*, *>

‎utbot-spring-framework/src/main/kotlin/org/utbot/framework/context/spring/SpringApplicationContextImpl.kt

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import org.utbot.framework.context.ConcreteExecutionContext
1111
import org.utbot.framework.context.NonNullSpeculator
1212
import org.utbot.framework.context.TypeReplacer
1313
import org.utbot.framework.context.custom.CoverageFilteringConcreteExecutionContext
14+
import org.utbot.framework.context.custom.RerunningConcreteExecutionContext
1415
import org.utbot.framework.context.custom.mockAllTypesWithoutSpecificValueProvider
1516
import org.utbot.framework.context.utils.transformJavaFuzzingContext
1617
import org.utbot.framework.context.utils.withValueProvider
@@ -72,11 +73,14 @@ class SpringApplicationContextImpl(
7273
)
7374
.mockAllTypesWithoutSpecificValueProvider()
7475
}
75-
SpringTestType.INTEGRATION_TEST -> SpringIntegrationTestConcreteExecutionContext(
76-
delegateConcreteExecutionContext,
77-
classpathWithoutDependencies,
78-
this
79-
)
76+
SpringTestType.INTEGRATION_TEST ->
77+
RerunningConcreteExecutionContext(
78+
SpringIntegrationTestConcreteExecutionContext(
79+
delegateConcreteExecutionContext,
80+
classpathWithoutDependencies,
81+
springApplicationContext = this
82+
)
83+
)
8084
}
8185
}
8286

‎utbot-spring-framework/src/main/kotlin/org/utbot/framework/context/spring/SpringIntegrationTestConcreteExecutionContext.kt

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ import org.utbot.framework.context.JavaFuzzingContext
66
import org.utbot.framework.plugin.api.ClassId
77
import org.utbot.framework.plugin.api.ConcreteContextLoadingResult
88
import org.utbot.framework.plugin.api.SpringSettings
9-
import org.utbot.framework.plugin.api.UtExecution
109
import org.utbot.fuzzer.IdentityPreservingIdGenerator
1110
import org.utbot.instrumentation.ConcreteExecutor
1211
import org.utbot.instrumentation.getRelevantSpringRepositories
@@ -21,7 +20,7 @@ class SpringIntegrationTestConcreteExecutionContext(
2120
private val delegateContext: ConcreteExecutionContext,
2221
classpathWithoutDependencies: String,
2322
private val springApplicationContext: SpringApplicationContext,
24-
) : ConcreteExecutionContext {
23+
) : ConcreteExecutionContext by delegateContext {
2524
private val springSettings = (springApplicationContext.springSettings as? SpringSettings.PresentSpringSettings) ?:
2625
error("Integration tests cannot be generated without Spring configuration")
2726

@@ -50,11 +49,6 @@ class SpringIntegrationTestConcreteExecutionContext(
5049
}
5150
}
5251

53-
override fun transformExecutionsBeforeMinimization(
54-
executions: List<UtExecution>,
55-
classUnderTestId: ClassId
56-
): List<UtExecution> = delegateContext.transformExecutionsBeforeMinimization(executions, classUnderTestId)
57-
5852
override fun tryCreateFuzzingContext(
5953
concreteExecutor: ConcreteExecutor<UtConcreteExecutionResult, UtExecutionInstrumentation>,
6054
classUnderTest: ClassId,

0 commit comments

Comments
 (0)