Skip to content

Commit 4713374

Browse files
DamtevDamtev
committed
Removed field state assertions for failing tests
1 parent 2834e39 commit 4713374

File tree

4 files changed

+95
-53
lines changed

4 files changed

+95
-53
lines changed

‎utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/CgMethodConstructor.kt

Lines changed: 76 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -53,11 +53,7 @@ import org.utbot.framework.codegen.model.tree.CgStatement
5353
import org.utbot.framework.codegen.model.tree.CgStaticFieldAccess
5454
import org.utbot.framework.codegen.model.tree.CgTestMethod
5555
import org.utbot.framework.codegen.model.tree.CgTestMethodType
56-
import org.utbot.framework.codegen.model.tree.CgTestMethodType.CRASH
57-
import org.utbot.framework.codegen.model.tree.CgTestMethodType.FAILING
58-
import org.utbot.framework.codegen.model.tree.CgTestMethodType.PARAMETRIZED
59-
import org.utbot.framework.codegen.model.tree.CgTestMethodType.SUCCESSFUL
60-
import org.utbot.framework.codegen.model.tree.CgTestMethodType.TIMEOUT
56+
import org.utbot.framework.codegen.model.tree.CgTestMethodType.*
6157
import org.utbot.framework.codegen.model.tree.CgTryCatch
6258
import org.utbot.framework.codegen.model.tree.CgTypeCast
6359
import org.utbot.framework.codegen.model.tree.CgValue
@@ -310,15 +306,13 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
310306
assertEquality(expected, actual)
311307
}
312308
}
313-
.onFailure { exception ->
314-
processExecutionFailure(currentExecution, exception)
315-
}
309+
.onFailure { exception -> processExecutionFailure(exception) }
316310
}
317311
else -> {} // TODO: check this specific case
318312
}
319313
}
320314

321-
private fun processExecutionFailure(execution: UtExecution, exception: Throwable) {
315+
private fun processExecutionFailure(exception: Throwable) {
322316
val methodInvocationBlock = {
323317
with(currentExecutable) {
324318
when (this) {
@@ -329,42 +323,36 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
329323
}
330324
}
331325

332-
if (shouldTestPassWithException(execution, exception)) {
333-
testFrameworkManager.expectException(exception::class.id) {
334-
methodInvocationBlock()
335-
}
336-
methodType = SUCCESSFUL
337-
338-
return
339-
}
340-
341-
if (shouldTestPassWithTimeoutException(execution, exception)) {
342-
writeWarningAboutTimeoutExceeding()
343-
testFrameworkManager.expectTimeout(hangingTestsTimeout.timeoutMs) {
344-
methodInvocationBlock()
326+
when (methodType) {
327+
SUCCESSFUL -> error("Unexpected successful without exception method type for execution with exception $exception")
328+
PASSED_EXCEPTION -> {
329+
testFrameworkManager.expectException(exception::class.id) {
330+
methodInvocationBlock()
331+
}
345332
}
346-
methodType = TIMEOUT
347-
348-
return
349-
}
350-
351-
when (exception) {
352-
is ConcreteExecutionFailureException -> {
353-
methodType = CRASH
354-
writeWarningAboutCrash()
333+
TIMEOUT -> {
334+
writeWarningAboutTimeoutExceeding()
335+
testFrameworkManager.expectTimeout(hangingTestsTimeout.timeoutMs) {
336+
methodInvocationBlock()
337+
}
355338
}
356-
is AccessControlException -> {
357-
methodType = CRASH
358-
writeWarningAboutFailureTest(exception)
359-
return
339+
CRASH -> when (exception) {
340+
is ConcreteExecutionFailureException -> {
341+
writeWarningAboutCrash()
342+
methodInvocationBlock()
343+
}
344+
is AccessControlException -> {
345+
// exception from sandbox
346+
writeWarningAboutFailureTest(exception)
347+
}
348+
else -> error("Unexpected crash suite for failing execution with $exception exception")
360349
}
361-
else -> {
362-
methodType = FAILING
350+
FAILING -> {
363351
writeWarningAboutFailureTest(exception)
352+
methodInvocationBlock()
364353
}
354+
PARAMETRIZED -> error("Unexpected $PARAMETRIZED method type for failing execution with $exception exception")
365355
}
366-
367-
methodInvocationBlock()
368356
}
369357

370358
private fun shouldTestPassWithException(execution: UtExecution, exception: Throwable): Boolean {
@@ -1193,9 +1181,7 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
11931181
constructorCall(*methodArguments.toTypedArray())
11941182
}
11951183
}
1196-
.onFailure { exception ->
1197-
processExecutionFailure(currentExecution, exception)
1198-
}
1184+
.onFailure { exception -> processExecutionFailure(exception) }
11991185
}
12001186

12011187
/**
@@ -1261,6 +1247,12 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
12611247
) {
12621248
thisInstance[executable](*methodArguments.toTypedArray())
12631249
}
1250+
1251+
// We need to clear scope of created variables
1252+
// because after invocation of MUT some expressions could be changed.
1253+
// For example, fields of object that were null could be assigned to some new objects
1254+
// and previously created variables do not reflect these changes.
1255+
createdFromExpressionVariables.clear()
12641256
}
12651257
else -> {} // TODO: check this specific case
12661258
}
@@ -1290,11 +1282,22 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
12901282
val name = paramNames[executableId]?.get(index)
12911283
methodArguments += variableConstructor.getOrCreateVariable(param, name)
12921284
}
1293-
rememberInitialEnvironmentState(modificationInfo)
1285+
1286+
if (requiresFieldStateAssertions()) {
1287+
// we should generate field assertions only for successful tests
1288+
// that does not break the current test execution after invocation of method under test
1289+
rememberInitialEnvironmentState(modificationInfo)
1290+
}
1291+
12941292
recordActualResult()
12951293
generateResultAssertions()
1296-
rememberFinalEnvironmentState(modificationInfo)
1297-
generateFieldStateAssertions()
1294+
1295+
if (requiresFieldStateAssertions()) {
1296+
// we should generate field assertions only for successful tests
1297+
// that does not break the current test execution after invocation of method under test
1298+
rememberFinalEnvironmentState(modificationInfo)
1299+
generateFieldStateAssertions()
1300+
}
12981301
}
12991302

13001303
if (statics.isNotEmpty()) {
@@ -1343,6 +1346,10 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
13431346
}
13441347
}
13451348

1349+
private fun requiresFieldStateAssertions(): Boolean =
1350+
!methodType.isThrowing ||
1351+
(methodType == PASSED_EXCEPTION && !testFrameworkManager.isExpectedExceptionExecutionBreaking)
1352+
13461353
private val expectedResultVarName = "expectedResult"
13471354
private val expectedErrorVarName = "expectedError"
13481355

@@ -1373,7 +1380,8 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
13731380
substituteStaticFields(statics, isParametrized = true)
13741381

13751382
// build this instance
1376-
thisInstance = genericExecution.stateBefore.thisInstance?.let { currentMethodParameters[CgParameterKind.ThisInstance] }
1383+
thisInstance =
1384+
genericExecution.stateBefore.thisInstance?.let { currentMethodParameters[CgParameterKind.ThisInstance] }
13771385

13781386
// build arguments for method under test and parameterized test
13791387
for (index in genericExecution.stateBefore.parameters.indices) {
@@ -1597,6 +1605,7 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
15971605
private fun <R> withTestMethodScope(execution: UtExecution, block: () -> R): R {
15981606
clearTestMethodScope()
15991607
currentExecution = execution
1608+
determineExecutionType()
16001609
statesCache = EnvironmentFieldStateCache.emptyCacheFor(execution)
16011610
return try {
16021611
block()
@@ -1664,6 +1673,27 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
16641673
testSet.executions.any { it.result is UtExecutionFailure }
16651674

16661675

1676+
/**
1677+
* Determines [CgTestMethodType] for current execution according to its success or failure.
1678+
*/
1679+
private fun determineExecutionType() {
1680+
val currentExecution = currentExecution!!
1681+
1682+
currentExecution.result
1683+
.onSuccess { methodType = SUCCESSFUL }
1684+
.onFailure { exception ->
1685+
methodType = when {
1686+
shouldTestPassWithException(currentExecution, exception) -> PASSED_EXCEPTION
1687+
shouldTestPassWithTimeoutException(currentExecution, exception) -> TIMEOUT
1688+
else -> when (exception) {
1689+
is ConcreteExecutionFailureException -> CRASH
1690+
is AccessControlException -> CRASH // exception from sandbox
1691+
else -> FAILING
1692+
}
1693+
}
1694+
}
1695+
}
1696+
16671697
private fun testMethod(
16681698
methodName: String,
16691699
displayName: String?,

‎utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/CgTestClassConstructor.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -302,7 +302,7 @@ data class TestsGenerationReport(
302302

303303
testMethods.forEach {
304304
when (it.type) {
305-
SUCCESSFUL -> updateExecutions(it, successfulExecutions)
305+
SUCCESSFUL, PASSED_EXCEPTION -> updateExecutions(it, successfulExecutions)
306306
FAILING -> updateExecutions(it, failedExecutions)
307307
TIMEOUT -> updateExecutions(it, timeoutExecutions)
308308
CRASH -> updateExecutions(it, crashExecutions)

‎utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/TestFrameworkManager.kt

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,11 @@ internal abstract class TestFrameworkManager(val context: CgContext)
8484

8585
abstract val annotationForOuterClasses: CgAnnotation?
8686

87+
/**
88+
* Determines whether appearance of expected exception in test method breaks current test execution or not.
89+
*/
90+
abstract val isExpectedExceptionExecutionBreaking: Boolean
91+
8792
protected open val timeoutArgumentName: String = "timeout"
8893

8994
open fun assertEquals(expected: CgValue, actual: CgValue) {
@@ -244,6 +249,8 @@ internal class TestNgManager(context: CgContext) : TestFrameworkManager(context)
244249
override val annotationForOuterClasses: CgAnnotation?
245250
get() = null
246251

252+
override val isExpectedExceptionExecutionBreaking: Boolean = false
253+
247254
override val timeoutArgumentName: String = "timeOut"
248255

249256
private val assertThrows: BuiltinMethodId
@@ -401,6 +408,8 @@ internal class Junit4Manager(context: CgContext) : TestFrameworkManager(context)
401408
)
402409
}
403410

411+
override val isExpectedExceptionExecutionBreaking: Boolean = true
412+
404413
override fun expectException(exception: ClassId, block: () -> Unit) {
405414
require(testFramework is Junit4) { "According to settings, JUnit4 was expected, but got: $testFramework" }
406415

@@ -464,6 +473,8 @@ internal class Junit5Manager(context: CgContext) : TestFrameworkManager(context)
464473
override val annotationForOuterClasses: CgAnnotation?
465474
get() = null
466475

476+
override val isExpectedExceptionExecutionBreaking: Boolean = false
477+
467478
private val assertThrows: BuiltinMethodId
468479
get() {
469480
require(testFramework is Junit5) { "According to settings, JUnit5 was expected, but got: $testFramework" }

‎utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/tree/CgElement.kt

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -314,12 +314,13 @@ class CgParameterizedTestDataProviderMethod(
314314
override val requiredFields: List<CgParameterDeclaration> = emptyList()
315315
}
316316

317-
enum class CgTestMethodType(val displayName: String) {
318-
SUCCESSFUL("Successful tests"),
319-
FAILING("Failing tests (with exceptions)"),
320-
TIMEOUT("Failing tests (with timeout)"),
321-
CRASH("Possibly crashing tests"),
322-
PARAMETRIZED("Parametrized tests");
317+
enum class CgTestMethodType(val displayName: String, val isThrowing: Boolean) {
318+
SUCCESSFUL(displayName = "Successful tests without exceptions", isThrowing = false),
319+
PASSED_EXCEPTION(displayName = "Thrown exceptions marked as passed", isThrowing = true),
320+
FAILING(displayName = "Failing tests (with exceptions)", isThrowing = true),
321+
TIMEOUT(displayName = "Failing tests (with timeout)", isThrowing = true),
322+
CRASH(displayName = "Possibly crashing tests", isThrowing = true),
323+
PARAMETRIZED(displayName = "Parametrized tests", isThrowing = false);
323324

324325
override fun toString(): String = displayName
325326
}

0 commit comments

Comments
 (0)