From 203f2d62fb82ecd2cb11c9bcbdbfd573c663ace2 Mon Sep 17 00:00:00 2001 From: IlyaMuravjov Date: Fri, 1 Dec 2023 13:52:58 +0300 Subject: [PATCH 1/2] Count "ensure JacoDB is initialized and executor is started" in force termination timeout --- .../src/main/kotlin/org/utbot/contest/usvm/ContestUsvm.kt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/usvm/ContestUsvm.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/usvm/ContestUsvm.kt index aea77bf1e5..9534dadaf4 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/usvm/ContestUsvm.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/usvm/ContestUsvm.kt @@ -127,6 +127,7 @@ fun runUsvmGeneration( } logger.info { "STARTED COUNTING BUDGET FOR ${cut.classId.name}" } + val budgetStartTimeMillis = System.currentTimeMillis() if (cut.classLoader.javaClass != URLClassLoader::class.java) { logger.error("Seems like classloader for cut not valid (maybe it was backported to system): ${cut.classLoader}") @@ -147,10 +148,10 @@ fun runUsvmGeneration( ) ) - logger.info().measureTime({ "Contest preparation: ensure JacoDB is initialized (NOT counted in time budget)" }) { + logger.info().measureTime({ "Contest preparation: ensure JacoDB is initialized (counted in time budget)" }) { jcContainer // force init lazy property } - logger.info().measureTime({ "Contest preparation: ensure executor is started (NOT counted in time budget)" }) { + logger.info().measureTime({ "Contest preparation: ensure executor is started (counted in time budget)" }) { jcContainer.runner.ensureRunnerAlive() } @@ -178,8 +179,9 @@ fun runUsvmGeneration( val timeStats = mutableMapOf() + val alreadySpentBudgetMillis = System.currentTimeMillis() - budgetStartTimeMillis jcContainer.machine.analyzeAsync( - forceTerminationTimeout = (generationTimeoutMillisWithoutCodegen + timeBudgetMs) / 2, + forceTerminationTimeout = (generationTimeoutMillisWithoutCodegen + timeBudgetMs) / 2 - alreadySpentBudgetMillis, methods = jcMethods, targets = emptyList() ) { state -> From 6a6aca5b67e32b62fecd135c70d13061c2e75ce9 Mon Sep 17 00:00:00 2001 From: IlyaMuravjov Date: Fri, 1 Dec 2023 14:47:50 +0300 Subject: [PATCH 2/2] Initialize JacoDB and stat executor before starting contest, stop caching machine --- .../main/kotlin/org/utbot/contest/Contest.kt | 10 ++ .../org/utbot/contest/usvm/ContestUsvm.kt | 147 +++++++++--------- .../org/utbot/contest/usvm/jc/JcContainer.kt | 16 +- 3 files changed, 91 insertions(+), 82 deletions(-) diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt index 0c0f20b7f9..047a7de05b 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt @@ -47,6 +47,7 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.newSingleThreadContext import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withTimeoutOrNull +import org.utbot.contest.usvm.createJcContainer import org.utbot.contest.usvm.jc.JcContainer import org.utbot.contest.usvm.runUsvmGeneration import org.utbot.framework.SummariesGenerationType @@ -128,6 +129,15 @@ fun main(args: Array) { // TODO usvm-sbft-merge: Soot is not not used in usvm // TestCaseGenerator(listOf(classfileDir), classpathString, dependencyPath, JdkInfoService.provide()) + // Initialize the JacoDB and start executor before contest is started. + // This saves the time budget for real work instead of initialization. + runBlocking { + createJcContainer( + tmpDir = tmpDir, + classpathFiles = classpathString.split(File.pathSeparator).map { File(it) } + ).runner.ensureRunnerAlive() + } + // TODO usvm-sbft-merge: utbot instrumentation not used in usvm // logger.info().measureTime({ "warmup: kotlin reflection :: init" }) { // prepareClass(ConcreteExecutorPool::class.java, "") diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/usvm/ContestUsvm.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/usvm/ContestUsvm.kt index 9534dadaf4..5c3cfbb669 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/usvm/ContestUsvm.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/usvm/ContestUsvm.kt @@ -88,27 +88,7 @@ fun runUsvmGeneration( val classpathFiles = classpathString.split(File.pathSeparator).map { File(it) } - val jcContainer by lazy { - JcContainer( - usePersistence = false, - persistenceDir = tmpDir, - classpath = classpathFiles, - javaHome = JdkInfoService.provide().path.toFile(), - machineOptions = UMachineOptions( - // TODO usvm-sbft: if we have less than CONTEST_TEST_EXECUTION_TIMEOUT time left, we should try execute - // with smaller timeout, but instrumentation currently doesn't allow to change timeout for individual runs - timeout = generationTimeoutMillisWithoutCodegen.milliseconds - CONTEST_TEST_EXECUTION_TIMEOUT, - pathSelectionStrategies = listOf(PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM), - pathSelectorFairnessStrategy = PathSelectorFairnessStrategy.COMPLETELY_FAIR, - solverType = SolverType.Z3, // TODO: usvm-ksmt: Yices doesn't work on old linux - ) - ) { - // TODO usvm-sbft: we may want to tune these JcSettings for contest - // TODO: require usePersistence=false for ClassScorer - installFeatures(InMemoryHierarchy, Approximations, ClassScorer(TypeScorer, ::scoreClassNode)) - loadByteCode(classpathFiles) - } - } + val jcContainer by lazy { createJcContainer(tmpDir, classpathFiles) } val executor by lazy { JcTestExecutor(jcContainer.cp, jcContainer.runner) } @@ -180,60 +160,72 @@ fun runUsvmGeneration( val timeStats = mutableMapOf() val alreadySpentBudgetMillis = System.currentTimeMillis() - budgetStartTimeMillis - jcContainer.machine.analyzeAsync( - forceTerminationTimeout = (generationTimeoutMillisWithoutCodegen + timeBudgetMs) / 2 - alreadySpentBudgetMillis, - methods = jcMethods, - targets = emptyList() - ) { state -> - val jcExecution = accumulateMeasureTime("executor.execute(${cut.classId.name})", timeStats, state.entrypoint) { - runCatching { - executor.execute( - method = state.entrypoint.typedMethod, - state = state, - stringConstants = jcContainer.machine.stringConstants, - classConstants = jcContainer.machine.classConstants - ) ?: return@analyzeAsync - }.getOrElse { e -> - logger.error(e) { "executor.execute(${state.entrypoint}) failed" } - return@analyzeAsync + JcMachine( + cp = jcContainer.cp, + options = UMachineOptions( + // TODO usvm-sbft: if we have less than CONTEST_TEST_EXECUTION_TIMEOUT time left, we should try execute + // with smaller timeout, but instrumentation currently doesn't allow to change timeout for individual runs + timeout = generationTimeoutMillisWithoutCodegen.milliseconds - alreadySpentBudgetMillis.milliseconds - CONTEST_TEST_EXECUTION_TIMEOUT, + pathSelectionStrategies = listOf(PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM), + pathSelectorFairnessStrategy = PathSelectorFairnessStrategy.COMPLETELY_FAIR, + solverType = SolverType.Z3, // TODO: usvm-ksmt: Yices doesn't work on old linux + ) + ).use { jcMachine -> + jcMachine.analyzeAsync( + forceTerminationTimeout = (generationTimeoutMillisWithoutCodegen + timeBudgetMs) / 2 - alreadySpentBudgetMillis, + methods = jcMethods, + targets = emptyList() + ) { state -> + val jcExecution = accumulateMeasureTime("executor.execute(${cut.classId.name})", timeStats, state.entrypoint) { + runCatching { + executor.execute( + method = state.entrypoint.typedMethod, + state = state, + stringConstants = jcMachine.stringConstants, + classConstants = jcMachine.classConstants + ) ?: return@analyzeAsync + }.getOrElse { e -> + logger.error(e) { "executor.execute(${state.entrypoint}) failed" } + return@analyzeAsync + } + } + val methodId = jcExecution.method.method.toExecutableId(jcContainer.cp) + val utExecution = accumulateMeasureTime("JcToUtExecutionConverter.convert(${cut.classId.name})", timeStats, jcExecution.method.method) { + runCatching { + JcToUtExecutionConverter( + jcExecution = jcExecution, + jcClasspath = jcContainer.cp, + idGenerator = utModelIdGenerator, + instructionIdProvider = instructionIdGenerator, + utilMethodProvider = codeGenerator.context.utilMethodProvider + ).convert() + // for some JcExecutions like RD faults we don't construct UtExecutions, converter logs such cases + ?: return@analyzeAsync + }.getOrElse { e -> + logger.error(e) { "JcToUtExecutionConverter.convert(${jcExecution.method.method}) failed" } + return@analyzeAsync + } } - } - val methodId = jcExecution.method.method.toExecutableId(jcContainer.cp) - val utExecution = accumulateMeasureTime("JcToUtExecutionConverter.convert(${cut.classId.name})", timeStats, jcExecution.method.method) { runCatching { - JcToUtExecutionConverter( - jcExecution = jcExecution, - jcClasspath = jcContainer.cp, - idGenerator = utModelIdGenerator, - instructionIdProvider = instructionIdGenerator, - utilMethodProvider = codeGenerator.context.utilMethodProvider - ).convert() - // for some JcExecutions like RD faults we don't construct UtExecutions, converter logs such cases - ?: return@analyzeAsync + val className = Type.getInternalName(methodId.classId.jClass) + val statsForMethod = methodToStats.getValue(methodId) + statsForMethod.testsGeneratedCount++ + utExecution.result.exceptionOrNull()?.let { exception -> + statsForMethod.detectedExceptionFqns += exception::class.java.name + } + utExecution.coverage?.let { + statsForClass.updateCoverage( + newCoverage = it, + isNewClass = !statsForClass.testedClassNames.contains(className), + fromFuzzing = utExecution is UtFuzzedExecution + ) + } + statsForClass.testedClassNames.add(className) + testsByMethod.getOrPut(methodId) { mutableListOf() } += utExecution }.getOrElse { e -> - logger.error(e) { "JcToUtExecutionConverter.convert(${jcExecution.method.method}) failed" } - return@analyzeAsync + logger.error(e) { "Test generation failed during stats update for $methodId" } } } - runCatching { - val className = Type.getInternalName(methodId.classId.jClass) - val statsForMethod = methodToStats.getValue(methodId) - statsForMethod.testsGeneratedCount++ - utExecution.result.exceptionOrNull()?.let { exception -> - statsForMethod.detectedExceptionFqns += exception::class.java.name - } - utExecution.coverage?.let { - statsForClass.updateCoverage( - newCoverage = it, - isNewClass = !statsForClass.testedClassNames.contains(className), - fromFuzzing = utExecution is UtFuzzedExecution - ) - } - statsForClass.testedClassNames.add(className) - testsByMethod.getOrPut(methodId) { mutableListOf() } += utExecution - }.getOrElse { e -> - logger.error(e) { "Test generation failed during stats update for $methodId" } - } } timeStats.forEach { (regionName, timeMillis) -> @@ -256,6 +248,21 @@ fun runUsvmGeneration( statsForClass } +fun createJcContainer( + tmpDir: File, + classpathFiles: List +) = JcContainer( + usePersistence = false, + persistenceDir = tmpDir, + classpath = classpathFiles, + javaHome = JdkInfoService.provide().path.toFile(), +) { + // TODO usvm-sbft: we may want to tune these JcSettings for contest + // TODO: require usePersistence=false for ClassScorer + installFeatures(InMemoryHierarchy, Approximations, ClassScorer(TypeScorer, ::scoreClassNode)) + loadByteCode(classpathFiles) +} + fun JcClasspath.findMethodOrNull(method: ExecutableId): JcMethod? = findClass(method.classId.name).declaredMethods.firstOrNull { it.name == method.name && it.jcdbSignature == method.jcdbSignature diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/usvm/jc/JcContainer.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/usvm/jc/JcContainer.kt index e3ace55213..6ef671fcfe 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/usvm/jc/JcContainer.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/usvm/jc/JcContainer.kt @@ -7,10 +7,8 @@ import org.jacodb.api.JcDatabase import org.jacodb.impl.JcSettings import org.jacodb.impl.features.classpaths.UnknownClasses import org.jacodb.impl.jacodb -import org.usvm.UMachineOptions import org.usvm.instrumentation.executor.UTestConcreteExecutor import org.usvm.instrumentation.instrumentation.JcRuntimeTraceInstrumenterFactory -import org.usvm.machine.JcMachine import org.usvm.util.classpathWithApproximations import java.io.File import kotlin.time.Duration.Companion.seconds @@ -23,12 +21,10 @@ class JcContainer private constructor( persistenceDir: File, classpath: List, javaHome: File, - machineOptions: UMachineOptions, builder: JcSettings.() -> Unit, ) : AutoCloseable { val db: JcDatabase val cp: JcClasspath - val machine: JcMachine val runner: UTestConcreteExecutor init { @@ -60,7 +56,6 @@ class JcContainer private constructor( } this.db = db this.cp = cp - this.machine = JcMachine(cp, machineOptions) this.runner = UTestConcreteExecutor( JcRuntimeTraceInstrumenterFactory::class, cpPath, @@ -77,30 +72,27 @@ class JcContainer private constructor( override fun close() { cp.close() db.close() - machine.close() runner.close() } companion object : AutoCloseable { val CONTEST_TEST_EXECUTION_TIMEOUT = 1.seconds - private val cache = HashMap, UMachineOptions>, JcContainer>() + private val cache = HashMap, JcContainer>() operator fun invoke( usePersistence: Boolean, persistenceDir: File, classpath: List, javaHome: File, - machineOptions: UMachineOptions, builder: JcSettings.() -> Unit, ): JcContainer { - val cacheKey = classpath to machineOptions - return cache[cacheKey] ?: run { + return cache[classpath] ?: run { // TODO usvm-sbft: right now max cache size is 1, do we need to increase it? logger.info { "JcContainer cache miss" } close() - JcContainer(usePersistence, persistenceDir, classpath, javaHome, machineOptions, builder) - .also { cache[cacheKey] = it } + JcContainer(usePersistence, persistenceDir, classpath, javaHome, builder) + .also { cache[classpath] = it } } }