Skip to content

Update time management of python test generation #1893

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Mar 9, 2023
Merged

Update time management of python test generation #1893

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
b002d99
Refactor time management
tamarinvs19 Mar 3, 2023
dd863d1
Add iteration counter to baseline algorithm
tamarinvs19 Mar 6, 2023
f56ffaf
Merge branch 'main' into tamarinvs19/timeout-generation
tamarinvs19 Mar 6, 2023
b0cbb92
Add initial annotation handling
tamarinvs19 Mar 6, 2023
310c71b
Update fuzzing process: add duplicate ignoring, fix feedback returning
tamarinvs19 Mar 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
221 changes: 128 additions & 93 deletions utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import org.utbot.python.newtyping.PythonTypeStorage
import org.utbot.python.newtyping.general.Type
import org.utbot.python.newtyping.pythonModules
import org.utbot.python.newtyping.pythonTypeRepresentation
import org.utbot.python.utils.TestGenerationLimitManager
import org.utbot.python.utils.camelToSnakeCase
import org.utbot.summary.fuzzer.names.TestSuggestedInfo
import java.net.ServerSocket
Expand All @@ -31,7 +32,6 @@ class PythonEngine(
private val pythonPath: String,
private val fuzzedConcreteValues: List<PythonFuzzedConcreteValue>,
private val timeoutForRun: Long,
private val initialCoveredLines: Set<Int>,
private val pythonTypeStorage: PythonTypeStorage,
) {

Expand Down Expand Up @@ -81,17 +81,21 @@ class PythonEngine(
}

private fun handleSuccessResult(
arguments: List<PythonFuzzedValue>,
types: List<Type>,
evaluationResult: PythonEvaluationSuccess,
methodUnderTestDescription: PythonMethodDescription,
hasThisObject: Boolean,
summary: List<String>,
): FuzzingExecutionFeedback {
val prohibitedExceptions = listOf(
"builtins.AttributeError",
"builtins.TypeError"
)

val summary = arguments
.zip(methodUnderTest.arguments)
.mapNotNull { it.first.summary?.replace("%var%", it.second.name) }
val hasThisObject = methodUnderTest.hasThisArgument

val resultModel = evaluationResult.stateAfter.getById(evaluationResult.resultId).toPythonTree(evaluationResult.stateAfter)

if (evaluationResult.isException && (resultModel.type.name in prohibitedExceptions)) { // wrong type (sometimes mypy fails)
Expand Down Expand Up @@ -139,27 +143,30 @@ class PythonEngine(
)
}

fun fuzzing(parameters: List<Type>, isCancelled: () -> Boolean, until: Long): Flow<FuzzingExecutionFeedback> = flow {
fun fuzzing(parameters: List<Type>, isCancelled: () -> Boolean, limitManager: TestGenerationLimitManager): Flow<FuzzingExecutionFeedback> = flow {
val additionalModules = parameters.flatMap { it.pythonModules() }
val coveredLines = initialCoveredLines.toMutableSet()

ServerSocket(0).use { serverSocket ->
logger.info { "Server port: ${serverSocket.localPort}" }
val manager = PythonWorkerManager(
serverSocket,
pythonPath,
until,
{ constructEvaluationInput(it) },
timeoutForRun.toInt()
)
logger.debug { "Server port: ${serverSocket.localPort}" }
val manager = try {
PythonWorkerManager(
serverSocket,
pythonPath,
limitManager.until,
{ constructEvaluationInput(it) },
timeoutForRun.toInt()
)
} catch (_: TimeoutException) {
logger.info { "Cannot connect to python executor" }
return@flow
}
logger.info { "Executor manager was created successfully" }

fun fuzzingResultHandler(
description: PythonMethodDescription,
arguments: List<PythonFuzzedValue>
): PythonExecutionResult? {
fun runWithFuzzedValues(
arguments: List<PythonFuzzedValue>,
): PythonEvaluationResult? {
val argumentValues = arguments.map { PythonTreeModel(it.tree, it.tree.type) }
logger.debug(argumentValues.map { it.tree } .toString())
logger.debug(argumentValues.map { it.tree }.toString())
val argumentModules = argumentValues
.flatMap { it.allContainingClassIds }
.map { it.moduleName }
Expand All @@ -177,59 +184,83 @@ class PythonEngine(
modelList,
methodUnderTest.argumentsNames
)
try {
return when (val evaluationResult = manager.run(functionArguments, localAdditionalModules)) {
is PythonEvaluationError -> {
val utError = UtError(
"Error evaluation: ${evaluationResult.status}, ${evaluationResult.message}",
Throwable(evaluationResult.stackTrace.joinToString("\n"))
)
logger.debug(evaluationResult.stackTrace.joinToString("\n"))
PythonExecutionResult(InvalidExecution(utError), PythonFeedback(control = Control.PASS))
}
return try {
manager.run(functionArguments, localAdditionalModules)
} catch (_: TimeoutException) {
logger.info { "Fuzzing process was interrupted by timeout" }
null
}
}

is PythonEvaluationTimeout -> {
val utError = UtError(evaluationResult.message, Throwable())
PythonExecutionResult(InvalidExecution(utError), PythonFeedback(control = Control.PASS))
}
fun handleExecutionResult(
result: PythonEvaluationResult,
arguments: List<PythonFuzzedValue>,
description: PythonMethodDescription,
): Pair<PythonExecutionResult, Boolean> {
val executionFeedback: FuzzingExecutionFeedback
val fuzzingFeedback: PythonFeedback

is PythonEvaluationSuccess -> {
val coveredInstructions = evaluationResult.coverage.coveredInstructions
coveredInstructions.forEach { coveredLines.add(it.lineNumber) }

val summary = arguments
.zip(methodUnderTest.arguments)
.mapNotNull { it.first.summary?.replace("%var%", it.second.name) }

val hasThisObject = methodUnderTest.hasThisArgument

when (val result = handleSuccessResult(
parameters,
evaluationResult,
description,
hasThisObject,
summary
)) {
is ValidExecution -> {
val trieNode: Trie.Node<Instruction> = description.tracer.add(coveredInstructions)
PythonExecutionResult(
result,
PythonFeedback(control = Control.CONTINUE, result = trieNode)
)
}
when(result) {
is PythonEvaluationError -> {
val utError = UtError(
"Error evaluation: ${result.status}, ${result.message}",
Throwable(result.stackTrace.joinToString("\n"))
)
logger.debug(result.stackTrace.joinToString("\n"))

is ArgumentsTypeErrorFeedback, is TypeErrorFeedback -> {
PythonExecutionResult(result, PythonFeedback(control = Control.PASS))
}
limitManager.addSuccessExecution()
executionFeedback = InvalidExecution(utError)
fuzzingFeedback = PythonFeedback(control = Control.PASS)
return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), true)
}

is PythonEvaluationTimeout -> {
val utError = UtError(result.message, Throwable())
limitManager.addInvalidExecution()
executionFeedback = InvalidExecution(utError)
fuzzingFeedback = PythonFeedback(control = Control.PASS)
return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false)
}

is PythonEvaluationSuccess -> {
val coveredInstructions = result.coverage.coveredInstructions
executionFeedback = handleSuccessResult(
arguments,
parameters,
result,
description,
)

is InvalidExecution -> {
PythonExecutionResult(result, PythonFeedback(control = Control.CONTINUE))
val trieNode: Trie.Node<Instruction> = description.tracer.add(coveredInstructions)
when (executionFeedback) {
is ValidExecution -> {
limitManager.addSuccessExecution()
if (trieNode.count > 1) {
fuzzingFeedback = PythonFeedback(control = Control.CONTINUE, result = trieNode)
return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false)
}
}

is ArgumentsTypeErrorFeedback -> {
fuzzingFeedback = PythonFeedback(control = Control.PASS)
return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false)
}

is TypeErrorFeedback -> {
limitManager.addInvalidExecution()
fuzzingFeedback = PythonFeedback(control = Control.PASS)
return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false)
}

is InvalidExecution -> {
limitManager.addInvalidExecution()
fuzzingFeedback = PythonFeedback(control = Control.CONTINUE)
return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false)
}
}
fuzzingFeedback = PythonFeedback(control = Control.CONTINUE, result = trieNode)
return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), true)
}
} catch (_: TimeoutException) {
return null
}
}

Expand All @@ -242,43 +273,47 @@ class PythonEngine(
)

if (parameters.isEmpty()) {
val result = fuzzingResultHandler(pmd, emptyList())
val result = runWithFuzzedValues(emptyList())
result?.let {
emit(it.fuzzingExecutionFeedback)
val (executionResult, needToEmit) = handleExecutionResult(result, emptyList(), pmd)
if (needToEmit) {
emit(executionResult.fuzzingExecutionFeedback)
}
}
manager.disconnect()
} else {
PythonFuzzing(pmd.pythonTypeStorage) { description, arguments ->
if (isCancelled()) {
logger.info { "Fuzzing process was interrupted" }
manager.disconnect()
return@PythonFuzzing PythonFeedback(control = Control.STOP)
}
if (System.currentTimeMillis() >= until) {
logger.info { "Fuzzing process was interrupted by timeout" }
manager.disconnect()
return@PythonFuzzing PythonFeedback(control = Control.STOP)
}
try {
PythonFuzzing(pmd.pythonTypeStorage) { description, arguments ->
if (isCancelled()) {
logger.info { "Fuzzing process was interrupted" }
manager.disconnect()
return@PythonFuzzing PythonFeedback(control = Control.STOP)
}

val pair = Pair(description, arguments.map { PythonTreeWrapper(it.tree) })
val mem = cache.get(pair)
if (mem != null) {
logger.debug("Repeat in fuzzing")
emit(mem.fuzzingExecutionFeedback)
return@PythonFuzzing mem.fuzzingPlatformFeedback
}
val result = fuzzingResultHandler(description, arguments)
if (result == null) { // timeout
logger.info { "Fuzzing process was interrupted by timeout" }
manager.disconnect()
return@PythonFuzzing PythonFeedback(control = Control.STOP)
}
val pair = Pair(description, arguments.map { PythonTreeWrapper(it.tree) })
val mem = cache.get(pair)
if (mem != null) {
logger.debug("Repeat in fuzzing")
return@PythonFuzzing mem.fuzzingPlatformFeedback
}

cache.add(pair, result)
emit(result.fuzzingExecutionFeedback)
return@PythonFuzzing result.fuzzingPlatformFeedback
val result = runWithFuzzedValues(arguments)
if (result == null) { // timeout
manager.disconnect()
return@PythonFuzzing PythonFeedback(control = Control.STOP)
}

}.fuzz(pmd)
val (executionResult, needToEmit) = handleExecutionResult(result, arguments, description)
cache.add(pair, executionResult)
if (needToEmit) {
emit(executionResult.fuzzingExecutionFeedback)
}
return@PythonFuzzing executionResult.fuzzingPlatformFeedback
}.fuzz(pmd)
} catch (ex: Exception) { // NoSeedValueException
logger.info { "Cannot fuzz values for types: $parameters" }
}
manager.disconnect()
}
}
}
Expand Down
Loading