Skip to content

Commit 70d099a

Browse files
committed
[utbot-rider]
1. New timeouts system for contest estimator. 2. Concrete execution phases redesign Fix #1726 Possible fix #1718
1 parent 03cd979 commit 70d099a

File tree

27 files changed

+1268
-1276
lines changed

27 files changed

+1268
-1276
lines changed

utbot-core/src/main/kotlin/org/utbot/common/StopWatch.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ class StopWatch {
5252
}
5353
}
5454

55-
fun get(unit: TimeUnit) = lock.withLockInterruptibly {
55+
fun get(unit: TimeUnit = TimeUnit.MILLISECONDS) = lock.withLockInterruptibly {
5656
unsafeUpdate()
5757
unit.convert(elapsedMillis, TimeUnit.MILLISECONDS)
5858
}

utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -277,7 +277,7 @@ object UtSettings : AbstractSettings(logger, defaultKeyForSettingsPath, defaultS
277277
/**
278278
* Timeout for specific concrete execution (in milliseconds).
279279
*/
280-
var concreteExecutionTimeoutInInstrumentedProcess: Long by getLongProperty(
280+
var concreteExecutionDefaultTimeoutInInstrumentedProcessMillis: Long by getLongProperty(
281281
DEFAULT_EXECUTION_TIMEOUT_IN_INSTRUMENTED_PROCESS_MS
282282
)
283283

Lines changed: 108 additions & 108 deletions
Original file line numberDiff line numberDiff line change
@@ -1,108 +1,108 @@
1-
package org.utbot.framework.plugin.api
2-
3-
import org.utbot.framework.plugin.api.visible.UtStreamConsumingException
4-
import java.io.File
5-
import java.util.LinkedList
6-
7-
sealed class UtExecutionResult
8-
9-
data class UtExecutionSuccess(val model: UtModel) : UtExecutionResult() {
10-
override fun toString() = "$model"
11-
}
12-
13-
sealed class UtExecutionFailure : UtExecutionResult() {
14-
abstract val exception: Throwable
15-
16-
/**
17-
* Represents the most inner exception in the failure.
18-
* Often equals to [exception], but is wrapped exception in [UtStreamConsumingException].
19-
*/
20-
open val rootCauseException: Throwable
21-
get() = exception
22-
}
23-
24-
data class UtOverflowFailure(
25-
override val exception: Throwable,
26-
) : UtExecutionFailure()
27-
28-
data class UtSandboxFailure(
29-
override val exception: Throwable
30-
) : UtExecutionFailure()
31-
32-
data class UtStreamConsumingFailure(
33-
override val exception: UtStreamConsumingException,
34-
) : UtExecutionFailure() {
35-
override val rootCauseException: Throwable
36-
get() = exception.innerExceptionOrAny
37-
}
38-
39-
/**
40-
* unexpectedFail (when exceptions such as NPE, IOBE, etc. appear, but not thrown by a user, applies both for function under test and nested calls )
41-
* expectedCheckedThrow (when function under test or nested call explicitly says that checked exception could be thrown and throws it)
42-
* expectedUncheckedThrow (when there is a throw statement for unchecked exception inside of function under test)
43-
* unexpectedUncheckedThrow (in case when there is unchecked exception thrown from nested call)
44-
*/
45-
data class UtExplicitlyThrownException(
46-
override val exception: Throwable,
47-
val fromNestedMethod: Boolean
48-
) : UtExecutionFailure()
49-
50-
data class UtImplicitlyThrownException(
51-
override val exception: Throwable,
52-
val fromNestedMethod: Boolean
53-
) : UtExecutionFailure()
54-
55-
class TimeoutException(s: String) : Exception(s)
56-
57-
data class UtTimeoutException(override val exception: TimeoutException) : UtExecutionFailure()
58-
59-
/**
60-
* Indicates failure in concrete execution.
61-
* For now it is explicitly throwing by ConcreteExecutor in case instrumented process death.
62-
*/
63-
class ConcreteExecutionFailureException(cause: Throwable, errorFile: File, val processStdout: List<String>) :
64-
Exception(
65-
buildString {
66-
appendLine()
67-
appendLine("----------------------------------------")
68-
appendLine("The instrumented process is dead")
69-
appendLine("Cause:\n${cause.message}")
70-
appendLine("Last 1000 lines of the error log ${errorFile.absolutePath}:")
71-
appendLine("----------------------------------------")
72-
errorFile.useLines { lines ->
73-
val lastLines = LinkedList<String>()
74-
for (line in lines) {
75-
lastLines.add(line)
76-
if (lastLines.size > 1000) {
77-
lastLines.removeFirst()
78-
}
79-
}
80-
lastLines.forEach { appendLine(it) }
81-
}
82-
appendLine("----------------------------------------")
83-
},
84-
cause
85-
)
86-
87-
data class UtConcreteExecutionFailure(override val exception: ConcreteExecutionFailureException) : UtExecutionFailure()
88-
89-
val UtExecutionResult.isSuccess: Boolean
90-
get() = this is UtExecutionSuccess
91-
92-
val UtExecutionResult.isFailure: Boolean
93-
get() = this is UtExecutionFailure
94-
95-
inline fun UtExecutionResult.onSuccess(action: (model: UtModel) -> Unit): UtExecutionResult {
96-
if (this is UtExecutionSuccess) action(model)
97-
return this
98-
}
99-
100-
inline fun UtExecutionResult.onFailure(action: (exception: Throwable) -> Unit): UtExecutionResult {
101-
if (this is UtExecutionFailure) action(rootCauseException)
102-
return this
103-
}
104-
105-
fun UtExecutionResult.exceptionOrNull(): Throwable? = when (this) {
106-
is UtExecutionFailure -> rootCauseException
107-
is UtExecutionSuccess -> null
108-
}
1+
package org.utbot.framework.plugin.api
2+
3+
import org.utbot.framework.plugin.api.visible.UtStreamConsumingException
4+
import java.io.File
5+
import java.util.LinkedList
6+
7+
sealed class UtExecutionResult
8+
9+
data class UtExecutionSuccess(val model: UtModel) : UtExecutionResult() {
10+
override fun toString() = "$model"
11+
}
12+
13+
sealed class UtExecutionFailure : UtExecutionResult() {
14+
abstract val exception: Throwable
15+
16+
/**
17+
* Represents the most inner exception in the failure.
18+
* Often equals to [exception], but is wrapped exception in [UtStreamConsumingException].
19+
*/
20+
open val rootCauseException: Throwable
21+
get() = exception
22+
}
23+
24+
data class UtOverflowFailure(
25+
override val exception: Throwable,
26+
) : UtExecutionFailure()
27+
28+
data class UtSandboxFailure(
29+
override val exception: Throwable
30+
) : UtExecutionFailure()
31+
32+
data class UtStreamConsumingFailure(
33+
override val exception: UtStreamConsumingException,
34+
) : UtExecutionFailure() {
35+
override val rootCauseException: Throwable
36+
get() = exception.innerExceptionOrAny
37+
}
38+
39+
/**
40+
* unexpectedFail (when exceptions such as NPE, IOBE, etc. appear, but not thrown by a user, applies both for function under test and nested calls )
41+
* expectedCheckedThrow (when function under test or nested call explicitly says that checked exception could be thrown and throws it)
42+
* expectedUncheckedThrow (when there is a throw statement for unchecked exception inside of function under test)
43+
* unexpectedUncheckedThrow (in case when there is unchecked exception thrown from nested call)
44+
*/
45+
data class UtExplicitlyThrownException(
46+
override val exception: Throwable,
47+
val fromNestedMethod: Boolean
48+
) : UtExecutionFailure()
49+
50+
data class UtImplicitlyThrownException(
51+
override val exception: Throwable,
52+
val fromNestedMethod: Boolean
53+
) : UtExecutionFailure()
54+
55+
class TimeoutException(s: String) : Exception(s)
56+
57+
data class UtTimeoutException(override val exception: TimeoutException) : UtExecutionFailure()
58+
59+
/**
60+
* Indicates failure in concrete execution.
61+
* For now it is explicitly throwing by ConcreteExecutor in case instrumented process death.
62+
*/
63+
class InstrumentedProcessDeathException(cause: Throwable, errorFile: File, val processStdout: List<String>) :
64+
Exception(
65+
buildString {
66+
appendLine()
67+
appendLine("----------------------------------------")
68+
appendLine("The instrumented process is dead")
69+
appendLine("Cause:\n${cause.message}")
70+
appendLine("Last 1000 lines of the error log ${errorFile.absolutePath}:")
71+
appendLine("----------------------------------------")
72+
errorFile.useLines { lines ->
73+
val lastLines = LinkedList<String>()
74+
for (line in lines) {
75+
lastLines.add(line)
76+
if (lastLines.size > 1000) {
77+
lastLines.removeFirst()
78+
}
79+
}
80+
lastLines.forEach { appendLine(it) }
81+
}
82+
appendLine("----------------------------------------")
83+
},
84+
cause
85+
)
86+
87+
data class UtConcreteExecutionFailure(override val exception: InstrumentedProcessDeathException) : UtExecutionFailure()
88+
89+
val UtExecutionResult.isSuccess: Boolean
90+
get() = this is UtExecutionSuccess
91+
92+
val UtExecutionResult.isFailure: Boolean
93+
get() = this is UtExecutionFailure
94+
95+
inline fun UtExecutionResult.onSuccess(action: (model: UtModel) -> Unit): UtExecutionResult {
96+
if (this is UtExecutionSuccess) action(model)
97+
return this
98+
}
99+
100+
inline fun UtExecutionResult.onFailure(action: (exception: Throwable) -> Unit): UtExecutionResult {
101+
if (this is UtExecutionFailure) action(rootCauseException)
102+
return this
103+
}
104+
105+
fun UtExecutionResult.exceptionOrNull(): Throwable? = when (this) {
106+
is UtExecutionFailure -> rootCauseException
107+
is UtExecutionSuccess -> null
108+
}

utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt

Lines changed: 26 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ import org.utbot.instrumentation.instrumentation.execution.UtExecutionInstrument
4848
import soot.jimple.Stmt
4949
import soot.tagkit.ParamNamesTag
5050
import java.lang.reflect.Method
51+
import kotlin.math.min
5152
import kotlin.system.measureTimeMillis
5253

5354
val logger = KotlinLogging.logger {}
@@ -247,7 +248,7 @@ class UtBotSymbolicEngine(
247248

248249
try {
249250
val concreteExecutionResult =
250-
concreteExecutor.executeConcretely(methodUnderTest, stateBefore, instrumentation)
251+
concreteExecutor.executeConcretely(methodUnderTest, stateBefore, instrumentation, UtSettings.concreteExecutionDefaultTimeoutInInstrumentedProcessMillis)
251252

252253
if (concreteExecutionResult.violatesUtMockAssumption()) {
253254
logger.debug { "Generated test case violates the UtMock assumption: $concreteExecutionResult" }
@@ -268,7 +269,7 @@ class UtBotSymbolicEngine(
268269
logger.debug { "concolicStrategy<${methodUnderTest}>: returned $concreteUtExecution" }
269270
} catch (e: CancellationException) {
270271
logger.debug(e) { "Cancellation happened" }
271-
} catch (e: ConcreteExecutionFailureException) {
272+
} catch (e: InstrumentedProcessDeathException) {
272273
emitFailedConcreteExecutionResult(stateBefore, e)
273274
} catch (e: Throwable) {
274275
emit(UtError("Concrete execution failed", e))
@@ -353,7 +354,14 @@ class UtBotSymbolicEngine(
353354
names,
354355
listOf(transform(ValueProvider.of(defaultValueProviders(defaultIdGenerator))))
355356
) { thisInstance, descr, values ->
356-
if (controller.job?.isActive == false || System.currentTimeMillis() >= until) {
357+
val diff = until - System.currentTimeMillis()
358+
val thresholdMillisForFuzzingOperation = 0 // may be better use 10-20 millis as it might not be possible
359+
// to concretely execute that values because request to instrumentation process involves
360+
// 1. serializing/deserializing it with kryo
361+
// 2. sending over rd
362+
// 3. concrete execution itself
363+
// 4. analyzing concrete result
364+
if (controller.job?.isActive == false || diff <= thresholdMillisForFuzzingOperation) {
357365
logger.info { "Fuzzing overtime: $methodUnderTest" }
358366
logger.info { "Test created by fuzzer: $testEmittedByFuzzer" }
359367
return@runJavaFuzzing BaseFeedback(result = Trie.emptyNode(), control = Control.STOP)
@@ -362,10 +370,11 @@ class UtBotSymbolicEngine(
362370
val initialEnvironmentModels = EnvironmentModels(thisInstance?.model, values.map { it.model }, mapOf())
363371

364372
val concreteExecutionResult: UtConcreteExecutionResult? = try {
365-
concreteExecutor.executeConcretely(methodUnderTest, initialEnvironmentModels, listOf())
373+
val timeoutMillis = min(UtSettings.concreteExecutionDefaultTimeoutInInstrumentedProcessMillis, diff)
374+
concreteExecutor.executeConcretely(methodUnderTest, initialEnvironmentModels, listOf(), timeoutMillis)
366375
} catch (e: CancellationException) {
367376
logger.debug { "Cancelled by timeout" }; null
368-
} catch (e: ConcreteExecutionFailureException) {
377+
} catch (e: InstrumentedProcessDeathException) {
369378
emitFailedConcreteExecutionResult(initialEnvironmentModels, e); null
370379
} catch (e: Throwable) {
371380
emit(UtError("Default concrete execution failed", e)); null
@@ -418,7 +427,7 @@ class UtBotSymbolicEngine(
418427

419428
private suspend fun FlowCollector<UtResult>.emitFailedConcreteExecutionResult(
420429
stateBefore: EnvironmentModels,
421-
e: ConcreteExecutionFailureException
430+
e: InstrumentedProcessDeathException
422431
) {
423432
val failedConcreteExecution = UtFailedExecution(
424433
stateBefore = stateBefore,
@@ -508,7 +517,8 @@ class UtBotSymbolicEngine(
508517
val concreteExecutionResult = concreteExecutor.executeConcretely(
509518
methodUnderTest,
510519
stateBefore,
511-
instrumentation
520+
instrumentation,
521+
UtSettings.concreteExecutionDefaultTimeoutInInstrumentedProcessMillis
512522
)
513523

514524
if (concreteExecutionResult.violatesUtMockAssumption()) {
@@ -525,8 +535,12 @@ class UtBotSymbolicEngine(
525535
emit(concolicUtExecution)
526536
logger.debug { "processResult<${methodUnderTest}>: returned $concolicUtExecution" }
527537
}
528-
} catch (e: ConcreteExecutionFailureException) {
538+
} catch (e: InstrumentedProcessDeathException) {
529539
emitFailedConcreteExecutionResult(stateBefore, e)
540+
} catch (e: CancellationException) {
541+
logger.debug(e) { "Cancellation happened" }
542+
} catch (e: Throwable) {
543+
emit(UtError("Default concrete execution failed", e));
530544
}
531545
}
532546

@@ -559,14 +573,16 @@ private fun ResolvedModels.constructStateForMethod(methodUnderTest: ExecutableId
559573
private suspend fun ConcreteExecutor<UtConcreteExecutionResult, UtExecutionInstrumentation>.executeConcretely(
560574
methodUnderTest: ExecutableId,
561575
stateBefore: EnvironmentModels,
562-
instrumentation: List<UtInstrumentation>
576+
instrumentation: List<UtInstrumentation>,
577+
timeoutInMillis: Long
563578
): UtConcreteExecutionResult = executeAsync(
564579
methodUnderTest.classId.name,
565580
methodUnderTest.signature,
566581
arrayOf(),
567582
parameters = UtConcreteExecutionData(
568583
stateBefore,
569-
instrumentation
584+
instrumentation,
585+
timeoutInMillis
570586
)
571587
).convertToAssemble(methodUnderTest.classId.packageName)
572588

utbot-framework/src/main/kotlin/org/utbot/external/api/UtBotJavaApi.kt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -221,7 +221,8 @@ object UtBotJavaApi {
221221
arrayOf(),
222222
parameters = UtConcreteExecutionData(
223223
testInfo.initialState,
224-
instrumentation = emptyList()
224+
instrumentation = emptyList(),
225+
UtSettings.concreteExecutionDefaultTimeoutInInstrumentedProcessMillis
225226
)
226227
).result
227228
} else {

utbot-framework/src/main/kotlin/org/utbot/framework/codegen/tree/CgMethodConstructor.kt

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ import org.utbot.framework.plugin.api.BuiltinClassId
6767
import org.utbot.framework.plugin.api.BuiltinMethodId
6868
import org.utbot.framework.plugin.api.ClassId
6969
import org.utbot.framework.plugin.api.CodegenLanguage
70-
import org.utbot.framework.plugin.api.ConcreteExecutionFailureException
70+
import org.utbot.framework.plugin.api.InstrumentedProcessDeathException
7171
import org.utbot.framework.plugin.api.ConstructorId
7272
import org.utbot.framework.plugin.api.ExecutableId
7373
import org.utbot.framework.plugin.api.FieldId
@@ -343,7 +343,7 @@ open class CgMethodConstructor(val context: CgContext) : CgContextOwner by conte
343343
}
344344
}
345345
CRASH -> when (expectedException) {
346-
is ConcreteExecutionFailureException -> {
346+
is InstrumentedProcessDeathException -> {
347347
writeWarningAboutCrash()
348348
methodInvocationBlock()
349349
}
@@ -425,7 +425,7 @@ open class CgMethodConstructor(val context: CgContext) : CgContextOwner by conte
425425
protected fun shouldTestPassWithException(execution: UtExecution, exception: Throwable): Boolean {
426426
if (exception is AccessControlException) return false
427427
// tests with timeout or crash should be processed differently
428-
if (exception is TimeoutException || exception is ConcreteExecutionFailureException) return false
428+
if (exception is TimeoutException || exception is InstrumentedProcessDeathException) return false
429429
if (exception is ArtificialError) return false
430430
if (UtSettings.treatAssertAsErrorSuite && exception is AssertionError) return false
431431

@@ -1844,7 +1844,7 @@ open class CgMethodConstructor(val context: CgContext) : CgContextOwner by conte
18441844
shouldTestPassWithTimeoutException(currentExecution, exception) -> TIMEOUT
18451845
else -> when (exception) {
18461846
is ArtificialError -> ARTIFICIAL
1847-
is ConcreteExecutionFailureException -> CRASH
1847+
is InstrumentedProcessDeathException -> CRASH
18481848
is AccessControlException -> CRASH // exception from sandbox
18491849
else -> FAILING
18501850
}

0 commit comments

Comments
 (0)