Skip to content

Commit 84eb506

Browse files
DamtevDamtev
committed
Removed field state assertions for failing tests
1 parent 3794310 commit 84eb506

File tree

3 files changed

+94
-52
lines changed

3 files changed

+94
-52
lines changed

‎utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/CgMethodConstructor.kt

Lines changed: 76 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -58,11 +58,7 @@ import org.utbot.framework.codegen.model.tree.CgStatement
5858
import org.utbot.framework.codegen.model.tree.CgStaticFieldAccess
5959
import org.utbot.framework.codegen.model.tree.CgTestMethod
6060
import org.utbot.framework.codegen.model.tree.CgTestMethodType
61-
import org.utbot.framework.codegen.model.tree.CgTestMethodType.CRASH
62-
import org.utbot.framework.codegen.model.tree.CgTestMethodType.FAILING
63-
import org.utbot.framework.codegen.model.tree.CgTestMethodType.PARAMETRIZED
64-
import org.utbot.framework.codegen.model.tree.CgTestMethodType.SUCCESSFUL
65-
import org.utbot.framework.codegen.model.tree.CgTestMethodType.TIMEOUT
61+
import org.utbot.framework.codegen.model.tree.CgTestMethodType.*
6662
import org.utbot.framework.codegen.model.tree.CgTryCatch
6763
import org.utbot.framework.codegen.model.tree.CgTypeCast
6864
import org.utbot.framework.codegen.model.tree.CgValue
@@ -305,15 +301,13 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
305301
assertEquality(expected, actual)
306302
}
307303
}
308-
.onFailure { exception ->
309-
processExecutionFailure(currentExecution, exception)
310-
}
304+
.onFailure { exception -> processExecutionFailure(exception) }
311305
}
312306
else -> {} // TODO: check this specific case
313307
}
314308
}
315309

316-
private fun processExecutionFailure(execution: UtExecution, exception: Throwable) {
310+
private fun processExecutionFailure(exception: Throwable) {
317311
val methodInvocationBlock = {
318312
with(currentExecutable) {
319313
when (this) {
@@ -324,42 +318,36 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
324318
}
325319
}
326320

327-
if (shouldTestPassWithException(execution, exception)) {
328-
testFrameworkManager.expectException(exception::class.id) {
329-
methodInvocationBlock()
330-
}
331-
methodType = SUCCESSFUL
332-
333-
return
334-
}
335-
336-
if (shouldTestPassWithTimeoutException(execution, exception)) {
337-
writeWarningAboutTimeoutExceeding()
338-
testFrameworkManager.expectTimeout(hangingTestsTimeout.timeoutMs) {
339-
methodInvocationBlock()
321+
when (methodType) {
322+
SUCCESSFUL -> error("Unexpected successful without exception method type for execution with exception $exception")
323+
PASSED_EXCEPTION -> {
324+
testFrameworkManager.expectException(exception::class.id) {
325+
methodInvocationBlock()
326+
}
340327
}
341-
methodType = TIMEOUT
342-
343-
return
344-
}
345-
346-
when (exception) {
347-
is ConcreteExecutionFailureException -> {
348-
methodType = CRASH
349-
writeWarningAboutCrash()
328+
TIMEOUT -> {
329+
writeWarningAboutTimeoutExceeding()
330+
testFrameworkManager.expectTimeout(hangingTestsTimeout.timeoutMs) {
331+
methodInvocationBlock()
332+
}
350333
}
351-
is AccessControlException -> {
352-
methodType = CRASH
353-
writeWarningAboutFailureTest(exception)
354-
return
334+
CRASH -> when (exception) {
335+
is ConcreteExecutionFailureException -> {
336+
writeWarningAboutCrash()
337+
methodInvocationBlock()
338+
}
339+
is AccessControlException -> {
340+
// exception from sandbox
341+
writeWarningAboutFailureTest(exception)
342+
}
343+
else -> error("Unexpected crash suite for failing execution with $exception exception")
355344
}
356-
else -> {
357-
methodType = FAILING
345+
FAILING -> {
358346
writeWarningAboutFailureTest(exception)
347+
methodInvocationBlock()
359348
}
349+
PARAMETRIZED -> error("Unexpected $PARAMETRIZED method type for failing execution with $exception exception")
360350
}
361-
362-
methodInvocationBlock()
363351
}
364352

365353
private fun shouldTestPassWithException(execution: UtExecution, exception: Throwable): Boolean {
@@ -1187,9 +1175,7 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
11871175
constructorCall(*methodArguments.toTypedArray())
11881176
}
11891177
}
1190-
.onFailure { exception ->
1191-
processExecutionFailure(currentExecution, exception)
1192-
}
1178+
.onFailure { exception -> processExecutionFailure(exception) }
11931179
}
11941180

11951181
/**
@@ -1258,6 +1244,12 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
12581244
) {
12591245
thisInstance[executable](*methodArguments.toTypedArray())
12601246
}
1247+
1248+
// We need to clear scope of created variables
1249+
// because after invocation of MUT some expressions could be changed.
1250+
// For example, fields of object that were null could be assigned to some new objects
1251+
// and previously created variables do not reflect these changes.
1252+
createdFromExpressionVariables.clear()
12611253
}
12621254
else -> {} // TODO: check this specific case
12631255
}
@@ -1288,11 +1280,22 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
12881280
val name = paramNames[executableId]?.get(index)
12891281
methodArguments += variableConstructor.getOrCreateVariable(param, name)
12901282
}
1291-
fieldStateManager.rememberInitialEnvironmentState(modificationInfo)
1283+
1284+
if (requiresFieldStateAssertions()) {
1285+
// we should generate field assertions only for successful tests
1286+
// that does not break the current test execution after invocation of method under test
1287+
fieldStateManager.rememberInitialEnvironmentState(modificationInfo)
1288+
}
1289+
12921290
recordActualResult()
12931291
generateResultAssertions()
1294-
fieldStateManager.rememberFinalEnvironmentState(modificationInfo)
1295-
generateFieldStateAssertions()
1292+
1293+
if (requiresFieldStateAssertions()) {
1294+
// we should generate field assertions only for successful tests
1295+
// that does not break the current test execution after invocation of method under test
1296+
fieldStateManager.rememberFinalEnvironmentState(modificationInfo)
1297+
generateFieldStateAssertions()
1298+
}
12961299
}
12971300

12981301
if (statics.isNotEmpty()) {
@@ -1340,6 +1343,10 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
13401343
}
13411344
}
13421345

1346+
private fun requiresFieldStateAssertions(): Boolean =
1347+
!methodType.isThrowing ||
1348+
(methodType == PASSED_EXCEPTION && !testFrameworkManager.isExpectedExceptionExecutionBreaking)
1349+
13431350
private val expectedResultVarName = "expectedResult"
13441351
private val expectedErrorVarName = "expectedError"
13451352

@@ -1367,7 +1374,8 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
13671374
substituteStaticFields(statics, isParametrized = true)
13681375

13691376
// build this instance
1370-
thisInstance = genericExecution.stateBefore.thisInstance?.let {
1377+
thisInstance =
1378+
genericExecution.stateBefore.thisInstance?.let {
13711379
variableConstructor.getOrCreateVariable(it)
13721380
}
13731381

@@ -1583,6 +1591,7 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
15831591
private fun <R> withTestMethodScope(execution: UtExecution, block: () -> R): R {
15841592
clearTestMethodScope()
15851593
currentExecution = execution
1594+
determineExecutionType()
15861595
statesCache = EnvironmentFieldStateCache.emptyCacheFor(execution)
15871596
return try {
15881597
block()
@@ -1649,6 +1658,27 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
16491658
testSet.executions.any { it.result is UtExecutionFailure }
16501659

16511660

1661+
/**
1662+
* Determines [CgTestMethodType] for current execution according to its success or failure.
1663+
*/
1664+
private fun determineExecutionType() {
1665+
val currentExecution = currentExecution!!
1666+
1667+
currentExecution.result
1668+
.onSuccess { methodType = SUCCESSFUL }
1669+
.onFailure { exception ->
1670+
methodType = when {
1671+
shouldTestPassWithException(currentExecution, exception) -> PASSED_EXCEPTION
1672+
shouldTestPassWithTimeoutException(currentExecution, exception) -> TIMEOUT
1673+
else -> when (exception) {
1674+
is ConcreteExecutionFailureException -> CRASH
1675+
is AccessControlException -> CRASH // exception from sandbox
1676+
else -> FAILING
1677+
}
1678+
}
1679+
}
1680+
}
1681+
16521682
private fun testMethod(
16531683
methodName: String,
16541684
displayName: String?,

‎utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/TestFrameworkManager.kt

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,11 @@ internal abstract class TestFrameworkManager(val context: CgContext)
8686

8787
abstract val annotationForOuterClasses: CgAnnotation?
8888

89+
/**
90+
* Determines whether appearance of expected exception in test method breaks current test execution or not.
91+
*/
92+
abstract val isExpectedExceptionExecutionBreaking: Boolean
93+
8994
protected open val timeoutArgumentName: String = "timeout"
9095

9196
open fun assertEquals(expected: CgValue, actual: CgValue) {
@@ -246,6 +251,8 @@ internal class TestNgManager(context: CgContext) : TestFrameworkManager(context)
246251
override val annotationForOuterClasses: CgAnnotation?
247252
get() = null
248253

254+
override val isExpectedExceptionExecutionBreaking: Boolean = false
255+
249256
override val timeoutArgumentName: String = "timeOut"
250257

251258
private val assertThrows: BuiltinMethodId
@@ -403,6 +410,8 @@ internal class Junit4Manager(context: CgContext) : TestFrameworkManager(context)
403410
)
404411
}
405412

413+
override val isExpectedExceptionExecutionBreaking: Boolean = true
414+
406415
override fun expectException(exception: ClassId, block: () -> Unit) {
407416
require(testFramework is Junit4) { "According to settings, JUnit4 was expected, but got: $testFramework" }
408417

@@ -466,6 +475,8 @@ internal class Junit5Manager(context: CgContext) : TestFrameworkManager(context)
466475
override val annotationForOuterClasses: CgAnnotation?
467476
get() = null
468477

478+
override val isExpectedExceptionExecutionBreaking: Boolean = false
479+
469480
private val assertThrows: BuiltinMethodId
470481
get() {
471482
require(testFramework is Junit5) { "According to settings, JUnit5 was expected, but got: $testFramework" }

‎utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/tree/CgElement.kt

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -335,12 +335,13 @@ class CgParameterizedTestDataProviderMethod(
335335
override val requiredFields: List<CgParameterDeclaration> = emptyList()
336336
}
337337

338-
enum class CgTestMethodType(val displayName: String) {
339-
SUCCESSFUL("Successful tests"),
340-
FAILING("Failing tests (with exceptions)"),
341-
TIMEOUT("Failing tests (with timeout)"),
342-
CRASH("Possibly crashing tests"),
343-
PARAMETRIZED("Parametrized tests");
338+
enum class CgTestMethodType(val displayName: String, val isThrowing: Boolean) {
339+
SUCCESSFUL(displayName = "Successful tests without exceptions", isThrowing = false),
340+
PASSED_EXCEPTION(displayName = "Thrown exceptions marked as passed", isThrowing = true),
341+
FAILING(displayName = "Failing tests (with exceptions)", isThrowing = true),
342+
TIMEOUT(displayName = "Failing tests (with timeout)", isThrowing = true),
343+
CRASH(displayName = "Possibly crashing tests", isThrowing = true),
344+
PARAMETRIZED(displayName = "Parametrized tests", isThrowing = false);
344345

345346
override fun toString(): String = displayName
346347
}

0 commit comments

Comments
 (0)