Skip to content

Commit 9c898bb

Browse files
tamarinvs19tamarinvs19
authored
Update time management of python test generation (#1893)
1 parent 0a6417a commit 9c898bb

File tree

6 files changed

+235
-136
lines changed

6 files changed

+235
-136
lines changed

‎utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt

Lines changed: 128 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import org.utbot.python.newtyping.PythonTypeStorage
1818
import org.utbot.python.newtyping.general.Type
1919
import org.utbot.python.newtyping.pythonModules
2020
import org.utbot.python.newtyping.pythonTypeRepresentation
21+
import org.utbot.python.utils.TestGenerationLimitManager
2122
import org.utbot.python.utils.camelToSnakeCase
2223
import org.utbot.summary.fuzzer.names.TestSuggestedInfo
2324
import java.net.ServerSocket
@@ -31,7 +32,6 @@ class PythonEngine(
3132
private val pythonPath: String,
3233
private val fuzzedConcreteValues: List<PythonFuzzedConcreteValue>,
3334
private val timeoutForRun: Long,
34-
private val initialCoveredLines: Set<Int>,
3535
private val pythonTypeStorage: PythonTypeStorage,
3636
) {
3737

@@ -81,17 +81,21 @@ class PythonEngine(
8181
}
8282

8383
private fun handleSuccessResult(
84+
arguments: List<PythonFuzzedValue>,
8485
types: List<Type>,
8586
evaluationResult: PythonEvaluationSuccess,
8687
methodUnderTestDescription: PythonMethodDescription,
87-
hasThisObject: Boolean,
88-
summary: List<String>,
8988
): FuzzingExecutionFeedback {
9089
val prohibitedExceptions = listOf(
9190
"builtins.AttributeError",
9291
"builtins.TypeError"
9392
)
9493

94+
val summary = arguments
95+
.zip(methodUnderTest.arguments)
96+
.mapNotNull { it.first.summary?.replace("%var%", it.second.name) }
97+
val hasThisObject = methodUnderTest.hasThisArgument
98+
9599
val resultModel = evaluationResult.stateAfter.getById(evaluationResult.resultId).toPythonTree(evaluationResult.stateAfter)
96100

97101
if (evaluationResult.isException && (resultModel.type.name in prohibitedExceptions)) { // wrong type (sometimes mypy fails)
@@ -139,27 +143,30 @@ class PythonEngine(
139143
)
140144
}
141145

142-
fun fuzzing(parameters: List<Type>, isCancelled: () -> Boolean, until: Long): Flow<FuzzingExecutionFeedback> = flow {
146+
fun fuzzing(parameters: List<Type>, isCancelled: () -> Boolean, limitManager: TestGenerationLimitManager): Flow<FuzzingExecutionFeedback> = flow {
143147
val additionalModules = parameters.flatMap { it.pythonModules() }
144-
val coveredLines = initialCoveredLines.toMutableSet()
145148

146149
ServerSocket(0).use { serverSocket ->
147-
logger.info { "Server port: ${serverSocket.localPort}" }
148-
val manager = PythonWorkerManager(
149-
serverSocket,
150-
pythonPath,
151-
until,
152-
{ constructEvaluationInput(it) },
153-
timeoutForRun.toInt()
154-
)
150+
logger.debug { "Server port: ${serverSocket.localPort}" }
151+
val manager = try {
152+
PythonWorkerManager(
153+
serverSocket,
154+
pythonPath,
155+
limitManager.until,
156+
{ constructEvaluationInput(it) },
157+
timeoutForRun.toInt()
158+
)
159+
} catch (_: TimeoutException) {
160+
logger.info { "Cannot connect to python executor" }
161+
return@flow
162+
}
155163
logger.info { "Executor manager was created successfully" }
156164

157-
fun fuzzingResultHandler(
158-
description: PythonMethodDescription,
159-
arguments: List<PythonFuzzedValue>
160-
): PythonExecutionResult? {
165+
fun runWithFuzzedValues(
166+
arguments: List<PythonFuzzedValue>,
167+
): PythonEvaluationResult? {
161168
val argumentValues = arguments.map { PythonTreeModel(it.tree, it.tree.type) }
162-
logger.debug(argumentValues.map { it.tree } .toString())
169+
logger.debug(argumentValues.map { it.tree }.toString())
163170
val argumentModules = argumentValues
164171
.flatMap { it.allContainingClassIds }
165172
.map { it.moduleName }
@@ -177,59 +184,83 @@ class PythonEngine(
177184
modelList,
178185
methodUnderTest.argumentsNames
179186
)
180-
try {
181-
return when (val evaluationResult = manager.run(functionArguments, localAdditionalModules)) {
182-
is PythonEvaluationError -> {
183-
val utError = UtError(
184-
"Error evaluation: ${evaluationResult.status}, ${evaluationResult.message}",
185-
Throwable(evaluationResult.stackTrace.joinToString("\n"))
186-
)
187-
logger.debug(evaluationResult.stackTrace.joinToString("\n"))
188-
PythonExecutionResult(InvalidExecution(utError), PythonFeedback(control = Control.PASS))
189-
}
187+
return try {
188+
manager.run(functionArguments, localAdditionalModules)
189+
} catch (_: TimeoutException) {
190+
logger.info { "Fuzzing process was interrupted by timeout" }
191+
null
192+
}
193+
}
190194

191-
is PythonEvaluationTimeout -> {
192-
val utError = UtError(evaluationResult.message, Throwable())
193-
PythonExecutionResult(InvalidExecution(utError), PythonFeedback(control = Control.PASS))
194-
}
195+
fun handleExecutionResult(
196+
result: PythonEvaluationResult,
197+
arguments: List<PythonFuzzedValue>,
198+
description: PythonMethodDescription,
199+
): Pair<PythonExecutionResult, Boolean> {
200+
val executionFeedback: FuzzingExecutionFeedback
201+
val fuzzingFeedback: PythonFeedback
195202

196-
is PythonEvaluationSuccess -> {
197-
val coveredInstructions = evaluationResult.coverage.coveredInstructions
198-
coveredInstructions.forEach { coveredLines.add(it.lineNumber) }
199-
200-
val summary = arguments
201-
.zip(methodUnderTest.arguments)
202-
.mapNotNull { it.first.summary?.replace("%var%", it.second.name) }
203-
204-
val hasThisObject = methodUnderTest.hasThisArgument
205-
206-
when (val result = handleSuccessResult(
207-
parameters,
208-
evaluationResult,
209-
description,
210-
hasThisObject,
211-
summary
212-
)) {
213-
is ValidExecution -> {
214-
val trieNode: Trie.Node<Instruction> = description.tracer.add(coveredInstructions)
215-
PythonExecutionResult(
216-
result,
217-
PythonFeedback(control = Control.CONTINUE, result = trieNode)
218-
)
219-
}
203+
when(result) {
204+
is PythonEvaluationError -> {
205+
val utError = UtError(
206+
"Error evaluation: ${result.status}, ${result.message}",
207+
Throwable(result.stackTrace.joinToString("\n"))
208+
)
209+
logger.debug(result.stackTrace.joinToString("\n"))
220210

221-
is ArgumentsTypeErrorFeedback, is TypeErrorFeedback -> {
222-
PythonExecutionResult(result, PythonFeedback(control = Control.PASS))
223-
}
211+
limitManager.addSuccessExecution()
212+
executionFeedback = InvalidExecution(utError)
213+
fuzzingFeedback = PythonFeedback(control = Control.PASS)
214+
return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), true)
215+
}
216+
217+
is PythonEvaluationTimeout -> {
218+
val utError = UtError(result.message, Throwable())
219+
limitManager.addInvalidExecution()
220+
executionFeedback = InvalidExecution(utError)
221+
fuzzingFeedback = PythonFeedback(control = Control.PASS)
222+
return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false)
223+
}
224+
225+
is PythonEvaluationSuccess -> {
226+
val coveredInstructions = result.coverage.coveredInstructions
227+
executionFeedback = handleSuccessResult(
228+
arguments,
229+
parameters,
230+
result,
231+
description,
232+
)
224233

225-
is InvalidExecution -> {
226-
PythonExecutionResult(result, PythonFeedback(control = Control.CONTINUE))
234+
val trieNode: Trie.Node<Instruction> = description.tracer.add(coveredInstructions)
235+
when (executionFeedback) {
236+
is ValidExecution -> {
237+
limitManager.addSuccessExecution()
238+
if (trieNode.count > 1) {
239+
fuzzingFeedback = PythonFeedback(control = Control.CONTINUE, result = trieNode)
240+
return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false)
227241
}
228242
}
243+
244+
is ArgumentsTypeErrorFeedback -> {
245+
fuzzingFeedback = PythonFeedback(control = Control.PASS)
246+
return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false)
247+
}
248+
249+
is TypeErrorFeedback -> {
250+
limitManager.addInvalidExecution()
251+
fuzzingFeedback = PythonFeedback(control = Control.PASS)
252+
return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false)
253+
}
254+
255+
is InvalidExecution -> {
256+
limitManager.addInvalidExecution()
257+
fuzzingFeedback = PythonFeedback(control = Control.CONTINUE)
258+
return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false)
259+
}
229260
}
261+
fuzzingFeedback = PythonFeedback(control = Control.CONTINUE, result = trieNode)
262+
return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), true)
230263
}
231-
} catch (_: TimeoutException) {
232-
return null
233264
}
234265
}
235266

@@ -242,43 +273,47 @@ class PythonEngine(
242273
)
243274

244275
if (parameters.isEmpty()) {
245-
val result = fuzzingResultHandler(pmd, emptyList())
276+
val result = runWithFuzzedValues(emptyList())
246277
result?.let {
247-
emit(it.fuzzingExecutionFeedback)
278+
val (executionResult, needToEmit) = handleExecutionResult(result, emptyList(), pmd)
279+
if (needToEmit) {
280+
emit(executionResult.fuzzingExecutionFeedback)
281+
}
248282
}
249283
manager.disconnect()
250284
} else {
251-
PythonFuzzing(pmd.pythonTypeStorage) { description, arguments ->
252-
if (isCancelled()) {
253-
logger.info { "Fuzzing process was interrupted" }
254-
manager.disconnect()
255-
return@PythonFuzzing PythonFeedback(control = Control.STOP)
256-
}
257-
if (System.currentTimeMillis() >= until) {
258-
logger.info { "Fuzzing process was interrupted by timeout" }
259-
manager.disconnect()
260-
return@PythonFuzzing PythonFeedback(control = Control.STOP)
261-
}
285+
try {
286+
PythonFuzzing(pmd.pythonTypeStorage) { description, arguments ->
287+
if (isCancelled()) {
288+
logger.info { "Fuzzing process was interrupted" }
289+
manager.disconnect()
290+
return@PythonFuzzing PythonFeedback(control = Control.STOP)
291+
}
262292

263-
val pair = Pair(description, arguments.map { PythonTreeWrapper(it.tree) })
264-
val mem = cache.get(pair)
265-
if (mem != null) {
266-
logger.debug("Repeat in fuzzing")
267-
emit(mem.fuzzingExecutionFeedback)
268-
return@PythonFuzzing mem.fuzzingPlatformFeedback
269-
}
270-
val result = fuzzingResultHandler(description, arguments)
271-
if (result == null) { // timeout
272-
logger.info { "Fuzzing process was interrupted by timeout" }
273-
manager.disconnect()
274-
return@PythonFuzzing PythonFeedback(control = Control.STOP)
275-
}
293+
val pair = Pair(description, arguments.map { PythonTreeWrapper(it.tree) })
294+
val mem = cache.get(pair)
295+
if (mem != null) {
296+
logger.debug("Repeat in fuzzing")
297+
return@PythonFuzzing mem.fuzzingPlatformFeedback
298+
}
276299

277-
cache.add(pair, result)
278-
emit(result.fuzzingExecutionFeedback)
279-
return@PythonFuzzing result.fuzzingPlatformFeedback
300+
val result = runWithFuzzedValues(arguments)
301+
if (result == null) { // timeout
302+
manager.disconnect()
303+
return@PythonFuzzing PythonFeedback(control = Control.STOP)
304+
}
280305

281-
}.fuzz(pmd)
306+
val (executionResult, needToEmit) = handleExecutionResult(result, arguments, description)
307+
cache.add(pair, executionResult)
308+
if (needToEmit) {
309+
emit(executionResult.fuzzingExecutionFeedback)
310+
}
311+
return@PythonFuzzing executionResult.fuzzingPlatformFeedback
312+
}.fuzz(pmd)
313+
} catch (ex: Exception) { // NoSeedValueException
314+
logger.info { "Cannot fuzz values for types: $parameters" }
315+
}
316+
manager.disconnect()
282317
}
283318
}
284319
}

0 commit comments

Comments
 (0)