Skip to content

Commit acc2473

Browse files
authored
Initialize JacoDB and stat executor before starting contest, stop caching machine (#2708)
* Count "ensure JacoDB is initialized and executor is started" in force termination timeout * Initialize JacoDB and stat executor before starting contest, stop caching machine
1 parent 693c54f commit acc2473

File tree

3 files changed

+95
-84
lines changed

3 files changed

+95
-84
lines changed

utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ import kotlinx.coroutines.launch
4747
import kotlinx.coroutines.newSingleThreadContext
4848
import kotlinx.coroutines.runBlocking
4949
import kotlinx.coroutines.withTimeoutOrNull
50+
import org.utbot.contest.usvm.createJcContainer
5051
import org.utbot.contest.usvm.jc.JcContainer
5152
import org.utbot.contest.usvm.runUsvmGeneration
5253
import org.utbot.framework.SummariesGenerationType
@@ -128,6 +129,15 @@ fun main(args: Array<String>) {
128129
// TODO usvm-sbft-merge: Soot is not not used in usvm
129130
// TestCaseGenerator(listOf(classfileDir), classpathString, dependencyPath, JdkInfoService.provide())
130131

132+
// Initialize the JacoDB and start executor before contest is started.
133+
// This saves the time budget for real work instead of initialization.
134+
runBlocking {
135+
createJcContainer(
136+
tmpDir = tmpDir,
137+
classpathFiles = classpathString.split(File.pathSeparator).map { File(it) }
138+
).runner.ensureRunnerAlive()
139+
}
140+
131141
// TODO usvm-sbft-merge: utbot instrumentation not used in usvm
132142
// logger.info().measureTime({ "warmup: kotlin reflection :: init" }) {
133143
// prepareClass(ConcreteExecutorPool::class.java, "")

utbot-junit-contest/src/main/kotlin/org/utbot/contest/usvm/ContestUsvm.kt

Lines changed: 81 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -88,27 +88,7 @@ fun runUsvmGeneration(
8888

8989
val classpathFiles = classpathString.split(File.pathSeparator).map { File(it) }
9090

91-
val jcContainer by lazy {
92-
JcContainer(
93-
usePersistence = false,
94-
persistenceDir = tmpDir,
95-
classpath = classpathFiles,
96-
javaHome = JdkInfoService.provide().path.toFile(),
97-
machineOptions = UMachineOptions(
98-
// TODO usvm-sbft: if we have less than CONTEST_TEST_EXECUTION_TIMEOUT time left, we should try execute
99-
// with smaller timeout, but instrumentation currently doesn't allow to change timeout for individual runs
100-
timeout = generationTimeoutMillisWithoutCodegen.milliseconds - CONTEST_TEST_EXECUTION_TIMEOUT,
101-
pathSelectionStrategies = listOf(PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM),
102-
pathSelectorFairnessStrategy = PathSelectorFairnessStrategy.COMPLETELY_FAIR,
103-
solverType = SolverType.Z3, // TODO: usvm-ksmt: Yices doesn't work on old linux
104-
)
105-
) {
106-
// TODO usvm-sbft: we may want to tune these JcSettings for contest
107-
// TODO: require usePersistence=false for ClassScorer
108-
installFeatures(InMemoryHierarchy, Approximations, ClassScorer(TypeScorer, ::scoreClassNode))
109-
loadByteCode(classpathFiles)
110-
}
111-
}
91+
val jcContainer by lazy { createJcContainer(tmpDir, classpathFiles) }
11292

11393
val executor by lazy { JcTestExecutor(jcContainer.cp, jcContainer.runner) }
11494

@@ -127,6 +107,7 @@ fun runUsvmGeneration(
127107
}
128108

129109
logger.info { "STARTED COUNTING BUDGET FOR ${cut.classId.name}" }
110+
val budgetStartTimeMillis = System.currentTimeMillis()
130111

131112
if (cut.classLoader.javaClass != URLClassLoader::class.java) {
132113
logger.error("Seems like classloader for cut not valid (maybe it was backported to system): ${cut.classLoader}")
@@ -147,10 +128,10 @@ fun runUsvmGeneration(
147128
)
148129
)
149130

150-
logger.info().measureTime({ "Contest preparation: ensure JacoDB is initialized (NOT counted in time budget)" }) {
131+
logger.info().measureTime({ "Contest preparation: ensure JacoDB is initialized (counted in time budget)" }) {
151132
jcContainer // force init lazy property
152133
}
153-
logger.info().measureTime({ "Contest preparation: ensure executor is started (NOT counted in time budget)" }) {
134+
logger.info().measureTime({ "Contest preparation: ensure executor is started (counted in time budget)" }) {
154135
jcContainer.runner.ensureRunnerAlive()
155136
}
156137

@@ -178,60 +159,73 @@ fun runUsvmGeneration(
178159

179160
val timeStats = mutableMapOf<String, Long>()
180161

181-
jcContainer.machine.analyzeAsync(
182-
forceTerminationTimeout = (generationTimeoutMillisWithoutCodegen + timeBudgetMs) / 2,
183-
methods = jcMethods,
184-
targets = emptyList()
185-
) { state ->
186-
val jcExecution = accumulateMeasureTime("executor.execute(${cut.classId.name})", timeStats, state.entrypoint) {
187-
runCatching {
188-
executor.execute(
189-
method = state.entrypoint.typedMethod,
190-
state = state,
191-
stringConstants = jcContainer.machine.stringConstants,
192-
classConstants = jcContainer.machine.classConstants
193-
) ?: return@analyzeAsync
194-
}.getOrElse { e ->
195-
logger.error(e) { "executor.execute(${state.entrypoint}) failed" }
196-
return@analyzeAsync
162+
val alreadySpentBudgetMillis = System.currentTimeMillis() - budgetStartTimeMillis
163+
JcMachine(
164+
cp = jcContainer.cp,
165+
options = UMachineOptions(
166+
// TODO usvm-sbft: if we have less than CONTEST_TEST_EXECUTION_TIMEOUT time left, we should try execute
167+
// with smaller timeout, but instrumentation currently doesn't allow to change timeout for individual runs
168+
timeout = generationTimeoutMillisWithoutCodegen.milliseconds - alreadySpentBudgetMillis.milliseconds - CONTEST_TEST_EXECUTION_TIMEOUT,
169+
pathSelectionStrategies = listOf(PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM),
170+
pathSelectorFairnessStrategy = PathSelectorFairnessStrategy.COMPLETELY_FAIR,
171+
solverType = SolverType.Z3, // TODO: usvm-ksmt: Yices doesn't work on old linux
172+
)
173+
).use { jcMachine ->
174+
jcMachine.analyzeAsync(
175+
forceTerminationTimeout = (generationTimeoutMillisWithoutCodegen + timeBudgetMs) / 2 - alreadySpentBudgetMillis,
176+
methods = jcMethods,
177+
targets = emptyList()
178+
) { state ->
179+
val jcExecution = accumulateMeasureTime("executor.execute(${cut.classId.name})", timeStats, state.entrypoint) {
180+
runCatching {
181+
executor.execute(
182+
method = state.entrypoint.typedMethod,
183+
state = state,
184+
stringConstants = jcMachine.stringConstants,
185+
classConstants = jcMachine.classConstants
186+
) ?: return@analyzeAsync
187+
}.getOrElse { e ->
188+
logger.error(e) { "executor.execute(${state.entrypoint}) failed" }
189+
return@analyzeAsync
190+
}
191+
}
192+
val methodId = jcExecution.method.method.toExecutableId(jcContainer.cp)
193+
val utExecution = accumulateMeasureTime("JcToUtExecutionConverter.convert(${cut.classId.name})", timeStats, jcExecution.method.method) {
194+
runCatching {
195+
JcToUtExecutionConverter(
196+
jcExecution = jcExecution,
197+
jcClasspath = jcContainer.cp,
198+
idGenerator = utModelIdGenerator,
199+
instructionIdProvider = instructionIdGenerator,
200+
utilMethodProvider = codeGenerator.context.utilMethodProvider
201+
).convert()
202+
// for some JcExecutions like RD faults we don't construct UtExecutions, converter logs such cases
203+
?: return@analyzeAsync
204+
}.getOrElse { e ->
205+
logger.error(e) { "JcToUtExecutionConverter.convert(${jcExecution.method.method}) failed" }
206+
return@analyzeAsync
207+
}
197208
}
198-
}
199-
val methodId = jcExecution.method.method.toExecutableId(jcContainer.cp)
200-
val utExecution = accumulateMeasureTime("JcToUtExecutionConverter.convert(${cut.classId.name})", timeStats, jcExecution.method.method) {
201209
runCatching {
202-
JcToUtExecutionConverter(
203-
jcExecution = jcExecution,
204-
jcClasspath = jcContainer.cp,
205-
idGenerator = utModelIdGenerator,
206-
instructionIdProvider = instructionIdGenerator,
207-
utilMethodProvider = codeGenerator.context.utilMethodProvider
208-
).convert()
209-
// for some JcExecutions like RD faults we don't construct UtExecutions, converter logs such cases
210-
?: return@analyzeAsync
210+
val className = Type.getInternalName(methodId.classId.jClass)
211+
val statsForMethod = methodToStats.getValue(methodId)
212+
statsForMethod.testsGeneratedCount++
213+
utExecution.result.exceptionOrNull()?.let { exception ->
214+
statsForMethod.detectedExceptionFqns += exception::class.java.name
215+
}
216+
utExecution.coverage?.let {
217+
statsForClass.updateCoverage(
218+
newCoverage = it,
219+
isNewClass = !statsForClass.testedClassNames.contains(className),
220+
fromFuzzing = utExecution is UtFuzzedExecution
221+
)
222+
}
223+
statsForClass.testedClassNames.add(className)
224+
testsByMethod.getOrPut(methodId) { mutableListOf() } += utExecution
211225
}.getOrElse { e ->
212-
logger.error(e) { "JcToUtExecutionConverter.convert(${jcExecution.method.method}) failed" }
213-
return@analyzeAsync
226+
logger.error(e) { "Test generation failed during stats update for $methodId" }
214227
}
215228
}
216-
runCatching {
217-
val className = Type.getInternalName(methodId.classId.jClass)
218-
val statsForMethod = methodToStats.getValue(methodId)
219-
statsForMethod.testsGeneratedCount++
220-
utExecution.result.exceptionOrNull()?.let { exception ->
221-
statsForMethod.detectedExceptionFqns += exception::class.java.name
222-
}
223-
utExecution.coverage?.let {
224-
statsForClass.updateCoverage(
225-
newCoverage = it,
226-
isNewClass = !statsForClass.testedClassNames.contains(className),
227-
fromFuzzing = utExecution is UtFuzzedExecution
228-
)
229-
}
230-
statsForClass.testedClassNames.add(className)
231-
testsByMethod.getOrPut(methodId) { mutableListOf() } += utExecution
232-
}.getOrElse { e ->
233-
logger.error(e) { "Test generation failed during stats update for $methodId" }
234-
}
235229
}
236230

237231
timeStats.forEach { (regionName, timeMillis) ->
@@ -254,6 +248,21 @@ fun runUsvmGeneration(
254248
statsForClass
255249
}
256250

251+
fun createJcContainer(
252+
tmpDir: File,
253+
classpathFiles: List<File>
254+
) = JcContainer(
255+
usePersistence = false,
256+
persistenceDir = tmpDir,
257+
classpath = classpathFiles,
258+
javaHome = JdkInfoService.provide().path.toFile(),
259+
) {
260+
// TODO usvm-sbft: we may want to tune these JcSettings for contest
261+
// TODO: require usePersistence=false for ClassScorer
262+
installFeatures(InMemoryHierarchy, Approximations, ClassScorer(TypeScorer, ::scoreClassNode))
263+
loadByteCode(classpathFiles)
264+
}
265+
257266
fun JcClasspath.findMethodOrNull(method: ExecutableId): JcMethod? =
258267
findClass(method.classId.name).declaredMethods.firstOrNull {
259268
it.name == method.name && it.jcdbSignature == method.jcdbSignature

utbot-junit-contest/src/main/kotlin/org/utbot/contest/usvm/jc/JcContainer.kt

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,8 @@ import org.jacodb.api.JcDatabase
77
import org.jacodb.impl.JcSettings
88
import org.jacodb.impl.features.classpaths.UnknownClasses
99
import org.jacodb.impl.jacodb
10-
import org.usvm.UMachineOptions
1110
import org.usvm.instrumentation.executor.UTestConcreteExecutor
1211
import org.usvm.instrumentation.instrumentation.JcRuntimeTraceInstrumenterFactory
13-
import org.usvm.machine.JcMachine
1412
import org.usvm.util.classpathWithApproximations
1513
import java.io.File
1614
import kotlin.time.Duration.Companion.seconds
@@ -23,12 +21,10 @@ class JcContainer private constructor(
2321
persistenceDir: File,
2422
classpath: List<File>,
2523
javaHome: File,
26-
machineOptions: UMachineOptions,
2724
builder: JcSettings.() -> Unit,
2825
) : AutoCloseable {
2926
val db: JcDatabase
3027
val cp: JcClasspath
31-
val machine: JcMachine
3228
val runner: UTestConcreteExecutor
3329

3430
init {
@@ -60,7 +56,6 @@ class JcContainer private constructor(
6056
}
6157
this.db = db
6258
this.cp = cp
63-
this.machine = JcMachine(cp, machineOptions)
6459
this.runner = UTestConcreteExecutor(
6560
JcRuntimeTraceInstrumenterFactory::class,
6661
cpPath,
@@ -77,30 +72,27 @@ class JcContainer private constructor(
7772
override fun close() {
7873
cp.close()
7974
db.close()
80-
machine.close()
8175
runner.close()
8276
}
8377

8478
companion object : AutoCloseable {
8579
val CONTEST_TEST_EXECUTION_TIMEOUT = 1.seconds
8680

87-
private val cache = HashMap<Pair<List<File>, UMachineOptions>, JcContainer>()
81+
private val cache = HashMap<List<File>, JcContainer>()
8882

8983
operator fun invoke(
9084
usePersistence: Boolean,
9185
persistenceDir: File,
9286
classpath: List<File>,
9387
javaHome: File,
94-
machineOptions: UMachineOptions,
9588
builder: JcSettings.() -> Unit,
9689
): JcContainer {
97-
val cacheKey = classpath to machineOptions
98-
return cache[cacheKey] ?: run {
90+
return cache[classpath] ?: run {
9991
// TODO usvm-sbft: right now max cache size is 1, do we need to increase it?
10092
logger.info { "JcContainer cache miss" }
10193
close()
102-
JcContainer(usePersistence, persistenceDir, classpath, javaHome, machineOptions, builder)
103-
.also { cache[cacheKey] = it }
94+
JcContainer(usePersistence, persistenceDir, classpath, javaHome, builder)
95+
.also { cache[classpath] = it }
10496
}
10597
}
10698

0 commit comments

Comments
 (0)