Skip to content

Commit e6e39d3

Browse files
tamarinvs19tamarinvs19
authored
Python codegen improvement: timeouted tests and exceptions (#2101)
* Add skipping of tests with too long execution * Add asserts for exception * Add timeouted tests limit and fix setting fail/pass exceptions * Fix handleTimeoutResult * Fix pytest skip decorator * Fix mistake in pytest error message
1 parent f26d423 commit e6e39d3

File tree

17 files changed

+227
-44
lines changed

17 files changed

+227
-44
lines changed

‎utbot-framework/src/main/kotlin/org/utbot/framework/codegen/tree/CgMethodConstructor.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -418,7 +418,7 @@ open class CgMethodConstructor(val context: CgContext) : CgContextOwner by conte
418418
return { +actual[streamConsumingMethodId]() }
419419
}
420420

421-
protected fun shouldTestPassWithException(execution: UtExecution, exception: Throwable): Boolean {
421+
protected open fun shouldTestPassWithException(execution: UtExecution, exception: Throwable): Boolean {
422422
if (exception is AccessControlException) return false
423423
// tests with timeout or crash should be processed differently
424424
if (exception is TimeoutException || exception is InstrumentedProcessDeathException) return false

‎utbot-intellij-python/src/main/kotlin/org/utbot/intellij/plugin/language/python/PythonDialogProcessor.kt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,7 @@ object PythonDialogProcessor {
205205
val message = it.fold(StringBuilder()) { acc, line -> acc.appendHtmlLine(line) }
206206
WarningTestsReportNotifier.notify(message.toString())
207207
},
208+
runtimeExceptionTestsBehaviour = model.runtimeExceptionTestsBehaviour,
208209
startedCleaningAction = { indicator.text = "Cleaning up..." }
209210
)
210211
} finally {

‎utbot-intellij-python/src/main/kotlin/org/utbot/intellij/plugin/language/python/PythonDialogWindow.kt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,7 @@ class PythonDialogWindow(val model: PythonTestsModel) : DialogWrapper(model.proj
177177
val settings = model.project.service<Settings>()
178178
with(settings) {
179179
model.timeoutForRun = hangingTestsTimeout.timeoutMs
180+
model.runtimeExceptionTestsBehaviour = runtimeExceptionTestsBehaviour
180181
}
181182

182183
super.doOKAction()

‎utbot-intellij-python/src/main/kotlin/org/utbot/intellij/plugin/language/python/PythonTestsModel.kt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import com.intellij.openapi.project.Project
55
import com.jetbrains.python.psi.PyClass
66
import com.jetbrains.python.psi.PyFile
77
import com.jetbrains.python.psi.PyFunction
8+
import org.utbot.framework.codegen.domain.RuntimeExceptionTestsBehaviour
89
import org.utbot.framework.codegen.domain.TestFramework
910
import org.utbot.framework.codegen.services.language.CgLanguageAssistant
1011
import org.utbot.intellij.plugin.models.BaseTestsModel
@@ -31,4 +32,5 @@ class PythonTestsModel(
3132
lateinit var testSourceRootPath: String
3233
lateinit var testFramework: TestFramework
3334
lateinit var selectedFunctions: Set<PyFunction>
35+
lateinit var runtimeExceptionTestsBehaviour: RuntimeExceptionTestsBehaviour
3436
}

‎utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt

Lines changed: 36 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ class PythonEngine(
4747
// can be improved
4848
description.name
4949
}
50-
is UtExplicitlyThrownException -> "${description.name}_with_exception"
50+
is UtExecutionFailure -> "${description.name}_with_exception"
5151
else -> description.name
5252
}
5353
val testName = "test_$testSuffix"
@@ -81,6 +81,37 @@ class PythonEngine(
8181
return Pair(stateThisObject, modelList)
8282
}
8383

84+
private fun handleTimeoutResult(
85+
arguments: List<PythonFuzzedValue>,
86+
methodUnderTestDescription: PythonMethodDescription,
87+
): FuzzingExecutionFeedback {
88+
val summary = arguments
89+
.zip(methodUnderTest.arguments)
90+
.mapNotNull { it.first.summary?.replace("%var%", it.second.name) }
91+
val executionResult = UtTimeoutException(TimeoutException("Execution is too long"))
92+
val testMethodName = suggestExecutionName(methodUnderTestDescription, executionResult)
93+
94+
val hasThisObject = methodUnderTest.hasThisArgument
95+
val (beforeThisObjectTree, beforeModelListTree) = if (hasThisObject) {
96+
arguments.first() to arguments.drop(1)
97+
} else {
98+
null to arguments
99+
}
100+
val beforeThisObject = beforeThisObjectTree?.let { PythonTreeModel(it.tree) }
101+
val beforeModelList = beforeModelListTree.map { PythonTreeModel(it.tree) }
102+
103+
val utFuzzedExecution = PythonUtExecution(
104+
stateInit = EnvironmentModels(beforeThisObject, beforeModelList, emptyMap()),
105+
stateBefore = EnvironmentModels(beforeThisObject, beforeModelList, emptyMap()),
106+
stateAfter = EnvironmentModels(beforeThisObject, beforeModelList, emptyMap()),
107+
diffIds = emptyList(),
108+
result = executionResult,
109+
testMethodName = testMethodName.testName?.camelToSnakeCase(),
110+
displayName = testMethodName.displayName,
111+
summary = summary.map { DocRegularStmt(it) }
112+
)
113+
return ValidExecution(utFuzzedExecution)
114+
}
84115
private fun handleSuccessResult(
85116
arguments: List<PythonFuzzedValue>,
86117
types: List<Type>,
@@ -110,7 +141,7 @@ class PythonEngine(
110141

111142
val executionResult =
112143
if (evaluationResult.isException) {
113-
UtExplicitlyThrownException(Throwable(resultModel.type.toString()), false)
144+
UtImplicitlyThrownException(Throwable(resultModel.type.toString()), false)
114145
}
115146
else {
116147
UtExecutionSuccess(PythonTreeModel(resultModel))
@@ -157,8 +188,7 @@ class PythonEngine(
157188
serverSocket,
158189
pythonPath,
159190
until,
160-
{ constructEvaluationInput(it) },
161-
)
191+
) { constructEvaluationInput(it) }
162192
} catch (_: TimeoutException) {
163193
return@flow
164194
}
@@ -199,8 +229,8 @@ class PythonEngine(
199229
}
200230

201231
is PythonEvaluationTimeout -> {
202-
val utError = UtError(evaluationResult.message, Throwable())
203-
PythonExecutionResult(InvalidExecution(utError), PythonFeedback(control = Control.PASS))
232+
val utTimeoutException = handleTimeoutResult(arguments, description)
233+
PythonExecutionResult(utTimeoutException, PythonFeedback(control = Control.PASS))
204234
}
205235

206236
is PythonEvaluationSuccess -> {

‎utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import java.io.File
3131

3232
private val logger = KotlinLogging.logger {}
3333
private const val RANDOM_TYPE_FREQUENCY = 6
34+
private const val MAX_EMPTY_COVERAGE_TESTS = 5
3435

3536
class PythonTestCaseGenerator(
3637
private val withMinimization: Boolean = true,
@@ -262,12 +263,15 @@ class PythonTestCaseGenerator(
262263
}
263264

264265
logger.info("Collect all test executions for ${method.name}")
265-
val (successfulExecutions, failedExecutions) = executions.partition { it.result is UtExecutionSuccess }
266+
val (emptyCoverageExecutions, coverageExecutions) = executions.partition { it.coverage == null }
267+
val (successfulExecutions, failedExecutions) = coverageExecutions.partition { it.result is UtExecutionSuccess }
266268

267269
return PythonTestSet(
268270
method,
269271
if (withMinimization)
270-
minimizeExecutions(successfulExecutions) + minimizeExecutions(failedExecutions)
272+
minimizeExecutions(successfulExecutions) +
273+
minimizeExecutions(failedExecutions) +
274+
emptyCoverageExecutions.take(MAX_EMPTY_COVERAGE_TESTS)
271275
else
272276
executions,
273277
errors,

‎utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package org.utbot.python
33
import com.squareup.moshi.Moshi
44
import com.squareup.moshi.kotlin.reflect.KotlinJsonAdapterFactory
55
import org.parsers.python.PythonParser
6+
import org.utbot.framework.codegen.domain.RuntimeExceptionTestsBehaviour
67
import org.utbot.python.framework.codegen.model.PythonSysPathImport
78
import org.utbot.python.framework.codegen.model.PythonSystemImport
89
import org.utbot.python.framework.codegen.model.PythonUserImport
@@ -50,6 +51,7 @@ object PythonTestGenerationProcessor {
5051
pythonRunRoot: Path,
5152
doNotCheckRequirements: Boolean = false,
5253
withMinimization: Boolean = true,
54+
runtimeExceptionTestsBehaviour: RuntimeExceptionTestsBehaviour = RuntimeExceptionTestsBehaviour.FAIL,
5355
isCanceled: () -> Boolean = { false },
5456
checkingRequirementsAction: () -> Unit = {},
5557
installingRequirementsAction: () -> Unit = {},
@@ -202,6 +204,7 @@ object PythonTestGenerationProcessor {
202204
paramNames = paramNames,
203205
testFramework = testFramework,
204206
testClassPackageName = "",
207+
runtimeExceptionTestsBehaviour = runtimeExceptionTestsBehaviour,
205208
)
206209
val testCode = codegen.pythonGenerateAsStringWithTestReport(
207210
notEmptyTests.map { testSet ->

‎utbot-python/src/main/kotlin/org/utbot/python/evaluation/UtExecutorThread.kt

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,14 @@
11
package org.utbot.python.evaluation
22

3+
import java.net.SocketException
4+
35
class UtExecutorThread : Thread() {
46
override fun run() {
5-
response = pythonWorker?.receiveMessage()
7+
response = try {
8+
pythonWorker?.receiveMessage()
9+
} catch (ex: SocketException) {
10+
null
11+
}
612
}
713

814
enum class Status {

‎utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonApi.kt

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@ class PythonClassId(
3434
override val simpleName: String = typeName
3535
override val canonicalName = name
3636
override val packageName = moduleName
37+
val prettyName: String = if (rootModuleName == pythonBuiltinsModuleName)
38+
name.split(".", limit=2).last()
39+
else
40+
name
3741
}
3842

3943
open class RawPythonAnnotation(

‎utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonIdUtils.kt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,4 @@ val pythonDictClassId = PythonClassId("builtins.dict")
1919
val pythonSetClassId = PythonClassId("builtins.set")
2020
val pythonBytearrayClassId = PythonClassId("builtins.bytearray")
2121
val pythonBytesClassId = PythonClassId("builtins.bytes")
22+
val pythonExceptionClassId = PythonClassId("builtins.Exception")

‎utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/PythonDomain.kt

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@ object Pytest : TestFramework(displayName = "pytest", id = "pytest") {
4141
override val nestedClassesShouldBeStatic: Boolean = false
4242
override val argListClassId: ClassId = pythonAnyClassId
4343

44+
val skipDecoratorClassId = PythonClassId("pytest", "mark.skip")
45+
4446
@OptIn(ExperimentalStdlibApi::class)
4547
override fun getRunTestsCommand(
4648
executionInvoke: String,
@@ -89,6 +91,8 @@ object Unittest : TestFramework(displayName = "Unittest", id = "Unittest") {
8991
override val nestedClassesShouldBeStatic: Boolean = false
9092
override val argListClassId: ClassId = pythonAnyClassId
9193

94+
val skipDecoratorClassId = PythonClassId("unittest.skip")
95+
9296
override fun getRunTestsCommand(
9397
executionInvoke: String,
9498
classPath: String,

‎utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonCgMethodConstructor.kt

Lines changed: 86 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,24 @@
11
package org.utbot.python.framework.codegen.model.constructor.tree
22

3+
import org.utbot.framework.codegen.domain.RuntimeExceptionTestsBehaviour
34
import org.utbot.framework.codegen.domain.context.CgContext
5+
import org.utbot.framework.codegen.domain.models.CgDocumentationComment
46
import org.utbot.framework.codegen.domain.models.CgFieldAccess
57
import org.utbot.framework.codegen.domain.models.CgGetLength
68
import org.utbot.framework.codegen.domain.models.CgLiteral
9+
import org.utbot.framework.codegen.domain.models.CgMultilineComment
10+
import org.utbot.framework.codegen.domain.models.CgParameterDeclaration
711
import org.utbot.framework.codegen.domain.models.CgReferenceExpression
812
import org.utbot.framework.codegen.domain.models.CgTestMethod
13+
import org.utbot.framework.codegen.domain.models.CgTestMethodType
914
import org.utbot.framework.codegen.domain.models.CgValue
1015
import org.utbot.framework.codegen.domain.models.CgVariable
16+
import org.utbot.framework.codegen.domain.models.convertDocToCg
1117
import org.utbot.framework.codegen.tree.CgMethodConstructor
18+
import org.utbot.framework.codegen.tree.buildTestMethod
1219
import org.utbot.framework.plugin.api.*
1320
import org.utbot.python.framework.api.python.*
21+
import org.utbot.python.framework.api.python.util.pythonExceptionClassId
1422
import org.utbot.python.framework.api.python.util.pythonIntClassId
1523
import org.utbot.python.framework.api.python.util.pythonNoneClassId
1624
import org.utbot.python.framework.codegen.PythonCgLanguageAssistant
@@ -23,20 +31,6 @@ class PythonCgMethodConstructor(context: CgContext) : CgMethodConstructor(contex
2331
pythonDeepEquals(expected, actual)
2432
}
2533

26-
private fun generatePythonTestComments(execution: UtExecution) {
27-
when (execution.result) {
28-
is UtExplicitlyThrownException ->
29-
(execution.result as UtExplicitlyThrownException).exception.message?.let {
30-
emptyLineIfNeeded()
31-
comment("raises $it")
32-
}
33-
34-
else -> {
35-
// nothing
36-
}
37-
}
38-
}
39-
4034
override fun createTestMethod(executableId: ExecutableId, execution: UtExecution): CgTestMethod =
4135
withTestMethodScope(execution) {
4236
val constructorState = (execution as PythonUtExecution).stateInit
@@ -48,7 +42,7 @@ class PythonCgMethodConstructor(context: CgContext) : CgMethodConstructor(contex
4842
}
4943
// TODO: remove this line when SAT-1273 is completed
5044
execution.displayName = execution.displayName?.let { "${executableId.name}: $it" }
51-
testMethod(testMethodName, execution.displayName) {
45+
pythonTestMethod(testMethodName, execution.displayName) {
5246
val statics = currentExecution!!.stateBefore.statics
5347
rememberInitialStaticFields(statics)
5448

@@ -115,8 +109,6 @@ class PythonCgMethodConstructor(context: CgContext) : CgMethodConstructor(contex
115109
generateResultAssertions()
116110

117111
generateFieldStateAssertions(stateAssertions, assertThisObject, executableId)
118-
119-
generatePythonTestComments(execution)
120112
}
121113

122114
if (statics.isNotEmpty()) {
@@ -131,6 +123,44 @@ class PythonCgMethodConstructor(context: CgContext) : CgMethodConstructor(contex
131123
}
132124
}
133125

126+
override fun generateResultAssertions() {
127+
if (currentExecutable is MethodId) {
128+
val currentExecution = currentExecution!!
129+
val executionResult = currentExecution.result
130+
if (executionResult is UtExecutionFailure) {
131+
val exceptionId = executionResult.rootCauseException.message?.let {PythonClassId(it)} ?: pythonExceptionClassId
132+
val executionBlock = {
133+
with(currentExecutable) {
134+
when (this) {
135+
is MethodId -> thisInstance[this](*methodArguments.toTypedArray()).intercepted()
136+
else -> {}
137+
}
138+
}
139+
}
140+
when (methodType) {
141+
CgTestMethodType.PASSED_EXCEPTION -> {
142+
testFrameworkManager.expectException(exceptionId) {
143+
executionBlock()
144+
}
145+
return
146+
}
147+
CgTestMethodType.FAILING -> {
148+
val executable = currentExecutable!! as PythonMethodId
149+
val executableName = "${executable.moduleName}.${executable.name}"
150+
val warningLine =
151+
"This test fails because function [$executableName] produces [${exceptionId.prettyName}]"
152+
+CgMultilineComment(warningLine)
153+
emptyLineIfNeeded()
154+
executionBlock()
155+
return
156+
}
157+
else -> {}
158+
}
159+
}
160+
}
161+
super.generateResultAssertions()
162+
}
163+
134164
private fun generateFieldStateAssertions(
135165
stateAssertions: MutableMap<Int, Pair<CgVariable, UtModel>>,
136166
assertThisObject: MutableList<Pair<CgVariable, UtModel>>,
@@ -154,6 +184,45 @@ class PythonCgMethodConstructor(context: CgContext) : CgMethodConstructor(contex
154184
}
155185
}
156186

187+
override fun shouldTestPassWithException(execution: UtExecution, exception: Throwable): Boolean {
188+
if (exception is TimeoutException || exception is InstrumentedProcessDeathException) return false
189+
return runtimeExceptionTestsBehaviour == RuntimeExceptionTestsBehaviour.PASS
190+
}
191+
192+
private fun pythonTestMethod(
193+
methodName: String,
194+
displayName: String?,
195+
params: List<CgParameterDeclaration> = emptyList(),
196+
body: () -> Unit,
197+
): CgTestMethod {
198+
displayName?.let {
199+
testFrameworkManager.addTestDescription(displayName)
200+
}
201+
202+
val result = currentExecution!!.result
203+
if (result is UtTimeoutException) {
204+
testFrameworkManager.disableTestMethod(
205+
"Disabled due to the fact that the execution is longer then ${hangingTestsTimeout.timeoutMs} ms"
206+
)
207+
}
208+
209+
val testMethod = buildTestMethod {
210+
name = methodName
211+
parameters = params
212+
statements = block(body)
213+
// Exceptions and annotations assignment must run after the statements block is build,
214+
// because we collect info about exceptions and required annotations while building the statements
215+
exceptions += collectedExceptions
216+
annotations += collectedMethodAnnotations
217+
methodType = this@PythonCgMethodConstructor.methodType
218+
219+
val docComment = currentExecution?.summary?.map { convertDocToCg(it) }?.toMutableList() ?: mutableListOf()
220+
documentation = CgDocumentationComment(docComment)
221+
}
222+
testMethods += testMethod
223+
return testMethod
224+
}
225+
157226
private fun pythonDeepEquals(expected: CgValue, actual: CgVariable) {
158227
require(expected is CgPythonTree) {
159228
"Expected value have to be CgPythonTree but `${expected::class}` found"

0 commit comments

Comments
 (0)