From b002d99a3ae585a75fd938fe6c244858132654ae Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Fri, 3 Mar 2023 22:37:52 +0300 Subject: [PATCH 1/4] Refactor time management --- .../kotlin/org/utbot/python/PythonEngine.kt | 22 +++--- .../utbot/python/PythonTestCaseGenerator.kt | 53 +++++-------- .../python/evaluation/PythonWorkerManager.kt | 6 +- .../python/utils/GenerationLimitManager.kt | 75 +++++++++++++++++++ 4 files changed, 109 insertions(+), 47 deletions(-) create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/utils/GenerationLimitManager.kt diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index 211cda6387..e3305841d0 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -31,7 +31,6 @@ class PythonEngine( private val pythonPath: String, private val fuzzedConcreteValues: List, private val timeoutForRun: Long, - private val initialCoveredLines: Set, private val pythonTypeStorage: PythonTypeStorage, ) { @@ -141,17 +140,20 @@ class PythonEngine( fun fuzzing(parameters: List, isCancelled: () -> Boolean, until: Long): Flow = flow { val additionalModules = parameters.flatMap { it.pythonModules() } - val coveredLines = initialCoveredLines.toMutableSet() ServerSocket(0).use { serverSocket -> logger.info { "Server port: ${serverSocket.localPort}" } - val manager = PythonWorkerManager( - serverSocket, - pythonPath, - until, - { constructEvaluationInput(it) }, - timeoutForRun.toInt() - ) + val manager = try { + PythonWorkerManager( + serverSocket, + pythonPath, + until, + { constructEvaluationInput(it) }, + timeoutForRun.toInt() + ) + } catch (_: TimeoutException) { + return@flow + } logger.info { "Executor manager was created successfully" } fun fuzzingResultHandler( @@ -195,7 +197,6 @@ class PythonEngine( is PythonEvaluationSuccess -> { val coveredInstructions = evaluationResult.coverage.coveredInstructions - coveredInstructions.forEach { coveredLines.add(it.lineNumber) } val summary = arguments .zip(methodUnderTest.arguments) @@ -274,7 +275,6 @@ class PythonEngine( cache.add(pair, result) emit(result.fuzzingExecutionFeedback) return@PythonFuzzing result.fuzzingPlatformFeedback - }.fuzz(pmd) } } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt index 9215bff3c2..9e9ca5a5f1 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt @@ -22,15 +22,14 @@ import org.utbot.python.newtyping.mypy.MypyReportLine import org.utbot.python.newtyping.mypy.getErrorNumber import org.utbot.python.newtyping.utils.getOffsetLine import org.utbot.python.typing.MypyAnnotations +import org.utbot.python.utils.ExecutionWithTimeoutMode +import org.utbot.python.utils.GenerationLimitManager import org.utbot.python.utils.PriorityCartesianProduct +import org.utbot.python.utils.TimeoutMode import java.io.File private val logger = KotlinLogging.logger {} -private const val COVERAGE_LIMIT = 150 -private const val ADDITIONAL_LIMIT = 5 -private const val INVALID_EXECUTION_LIMIT = 10 - class PythonTestCaseGenerator( private val withMinimization: Boolean = true, private val directoriesForSysPath: Set, @@ -120,6 +119,10 @@ class PythonTestCaseGenerator( fun generate(method: PythonMethod, until: Long): PythonTestSet { storageForMypyMessages.clear() + val limitManager = GenerationLimitManager( + ExecutionWithTimeoutMode, + until, + ) val typeStorage = PythonTypeStorage.get(mypyStorage) @@ -135,10 +138,6 @@ class PythonTestCaseGenerator( val coveredLines = mutableSetOf() var generated = 0 - var additionalLimit = ADDITIONAL_LIMIT - val typeInferenceCancellation = - { isCancelled() || System.currentTimeMillis() >= until || additionalLimit <= 0 } - logger.info("Start test generation for ${method.name}") substituteTypeParameters(method, typeStorage).forEach { newMethod -> inferAnnotations( @@ -148,7 +147,7 @@ class PythonTestCaseGenerator( hintCollector, mypyReportLine, mypyConfigFile, - typeInferenceCancellation + limitManager, ) { functionType -> val args = (functionType as FunctionType).arguments @@ -161,22 +160,12 @@ class PythonTestCaseGenerator( pythonPath, constants, timeoutForRun, - coveredLines, PythonTypeStorage.get(mypyStorage) ) - var invalidExecutionLimit = INVALID_EXECUTION_LIMIT - var coverageLimit = COVERAGE_LIMIT - var coveredBefore = coveredLines.size - var feedback: InferredTypeFeedback = SuccessFeedback - val fuzzerCancellation = { - typeInferenceCancellation() - || coverageLimit == 0 - || additionalLimit == 0 - || invalidExecutionLimit == 0 - } + val fuzzerCancellation = { isCancelled() || limitManager.isCancelled() } val startTime = System.currentTimeMillis() engine.fuzzing(args, fuzzerCancellation, until).collect { @@ -186,30 +175,26 @@ class PythonTestCaseGenerator( executions += it.utFuzzedExecution missingLines = updateCoverage(it.utFuzzedExecution, coveredLines, missingLines) feedback = SuccessFeedback + limitManager.addSuccessExecution() } is InvalidExecution -> { errors += it.utError feedback = SuccessFeedback } is ArgumentsTypeErrorFeedback -> { - invalidExecutionLimit-- feedback = InvalidTypeFeedback + limitManager.addInvalidExecution() } is TypeErrorFeedback -> { - invalidExecutionLimit-- feedback = InvalidTypeFeedback + limitManager.addInvalidExecution() } } - if (missingLines?.size == 0) { - additionalLimit-- - } - val coveredAfter = coveredLines.size - if (coveredAfter == coveredBefore) { - coverageLimit-- - } + limitManager.missedLines = missingLines?.size + logger.info { "Time ${System.currentTimeMillis() - startTime}: $generated, $missingLines" } - coveredBefore = coveredAfter } + limitManager.restart() feedback } } @@ -250,7 +235,7 @@ class PythonTestCaseGenerator( hintCollector: HintCollector, report: List, mypyConfigFile: File, - isCancelled: () -> Boolean, + limitManager: GenerationLimitManager, annotationHandler: suspend (Type) -> InferredTypeFeedback, ) { val namesInModule = mypyStorage.names @@ -259,6 +244,7 @@ class PythonTestCaseGenerator( .filter { it.length < 4 || !it.startsWith("__") || !it.endsWith("__") } + val typeInferenceCancellation = { isCancelled() || limitManager.isCancelled() } val algo = BaselineAlgorithm( typeStorage, @@ -277,14 +263,15 @@ class PythonTestCaseGenerator( ) runBlocking breaking@{ - if (isCancelled()) { + if (typeInferenceCancellation()) { return@breaking } - algo.run(hintCollector.result, isCancelled, annotationHandler) + algo.run(hintCollector.result, typeInferenceCancellation, annotationHandler) val existsAnnotation = method.definition.type if (existsAnnotation.arguments.all { it.pythonTypeName() != "typing.Any" }) { + limitManager.mode = TimeoutMode annotationHandler(existsAnnotation) } } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt index fd98864bf8..31ca73db03 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt @@ -24,8 +24,8 @@ class PythonWorkerManager( var timeout: Long = 0 lateinit var process: Process - lateinit var workerSocket: Socket - lateinit var codeExecutor: PythonCodeExecutor + private lateinit var workerSocket: Socket + private lateinit var codeExecutor: PythonCodeExecutor init { connect() @@ -39,7 +39,7 @@ class PythonWorkerManager( "localhost", serverSocket.localPort.toString(), "--logfile", logfile.absolutePath, - //"--loglevel", "DEBUG", + "--loglevel", "INFO", // "DEBUG", "INFO", "ERROR" )) timeout = max(until - processStartTime, 0) workerSocket = try { diff --git a/utbot-python/src/main/kotlin/org/utbot/python/utils/GenerationLimitManager.kt b/utbot-python/src/main/kotlin/org/utbot/python/utils/GenerationLimitManager.kt new file mode 100644 index 0000000000..83166effc7 --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/utils/GenerationLimitManager.kt @@ -0,0 +1,75 @@ +package org.utbot.python.utils + +import kotlin.math.min + +class GenerationLimitManager( + // global settings + var mode: LimitManagerMode, + val until: Long, + + // local settings: one type inference iteration + var executions: Int = 150, + var invalidExecutions: Int = 10, + var additionalExecutions: Int = 5, + var missedLines: Int? = null, +) { + private val initExecution = executions + private val initInvalidExecutions = invalidExecutions + private val initAdditionalExecutions = additionalExecutions + private val initMissedLines = missedLines + + fun restart() { + executions = initExecution + invalidExecutions = initInvalidExecutions + additionalExecutions = initAdditionalExecutions + missedLines = initMissedLines + } + + fun addSuccessExecution() { + executions -= 1 + } + fun addInvalidExecution() { + invalidExecutions -= 1 + } + + fun isCancelled(): Boolean { + return mode.isCancelled(this) + } +} + +interface LimitManagerMode { + fun isCancelled(manager: GenerationLimitManager): Boolean +} + +object MaxCoverageMode : LimitManagerMode { + override fun isCancelled(manager: GenerationLimitManager): Boolean { + return manager.missedLines?.equals(0) == true + } +} + +object TimeoutMode : LimitManagerMode { + override fun isCancelled(manager: GenerationLimitManager): Boolean { + return System.currentTimeMillis() >= manager.until + } +} + +object ExecutionMode : LimitManagerMode { + override fun isCancelled(manager: GenerationLimitManager): Boolean { + if (manager.invalidExecutions <= 0 || manager.executions <= 0) { + return min(manager.invalidExecutions, 0) + min(manager.executions, 0) <= manager.additionalExecutions + } + return false + } +} + +object MaxCoverageWithTimeoutMode : LimitManagerMode { + override fun isCancelled(manager: GenerationLimitManager): Boolean { + return MaxCoverageMode.isCancelled(manager) || TimeoutMode.isCancelled(manager) + } +} + +object ExecutionWithTimeoutMode : LimitManagerMode { + override fun isCancelled(manager: GenerationLimitManager): Boolean { + return ExecutionMode.isCancelled(manager) || TimeoutMode.isCancelled(manager) + } +} From dd863d109769e73f36bcd2c454dd6cca592008da Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Mon, 6 Mar 2023 15:45:39 +0300 Subject: [PATCH 2/4] Add iteration counter to baseline algorithm --- .../main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt | 6 +++--- .../utbot/python/newtyping/inference/TypeInferenceAPI.kt | 2 +- .../newtyping/inference/baseline/BaselineAlgorithm.kt | 6 +++++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt index 9e9ca5a5f1..4304478cf6 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt @@ -267,11 +267,11 @@ class PythonTestCaseGenerator( return@breaking } - algo.run(hintCollector.result, typeInferenceCancellation, annotationHandler) + val iterationNumber = algo.run(hintCollector.result, typeInferenceCancellation, annotationHandler) - val existsAnnotation = method.definition.type - if (existsAnnotation.arguments.all { it.pythonTypeName() != "typing.Any" }) { + if (iterationNumber == 1) { limitManager.mode = TimeoutMode + val existsAnnotation = method.definition.type annotationHandler(existsAnnotation) } } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/TypeInferenceAPI.kt b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/TypeInferenceAPI.kt index 8bd05be12c..babcd9a49b 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/TypeInferenceAPI.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/TypeInferenceAPI.kt @@ -8,7 +8,7 @@ abstract class TypeInferenceAlgorithm { hintCollectorResult: HintCollectorResult, isCancelled: () -> Boolean, annotationHandler: suspend (Type) -> InferredTypeFeedback, - ) + ): Int } sealed interface InferredTypeFeedback diff --git a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt index 4466a9abb8..f980239f2a 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt @@ -39,17 +39,20 @@ class BaselineAlgorithm( hintCollectorResult: HintCollectorResult, isCancelled: () -> Boolean, annotationHandler: suspend (Type) -> InferredTypeFeedback, - ) { + ): Int { val generalRating = createGeneralTypeRating(hintCollectorResult, storage) val initialState = getInitialState(hintCollectorResult, generalRating) val states: MutableList = mutableListOf(initialState) val fileForMypyRuns = TemporaryFileManager.assignTemporaryFile(tag = "mypy.py") + var iterationCounter = 0 run breaking@ { while (states.isNotEmpty()) { if (isCancelled()) return@breaking logger.debug("State number: ${states.size}") + iterationCounter++ + val state = chooseState(states) val newState = expandState(state, storage) if (newState != null) { @@ -71,6 +74,7 @@ class BaselineAlgorithm( } } } + return iterationCounter } private fun checkSignature(signature: FunctionType, fileForMypyRuns: File, configFile: File): Boolean { From b0cbb9270d2c5c81a54948b51444e48804756d25 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Mon, 6 Mar 2023 18:37:45 +0300 Subject: [PATCH 3/4] Add initial annotation handling --- .../python/newtyping/inference/baseline/BaselineAlgorithm.kt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt index f980239f2a..48aa0cae02 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt @@ -56,6 +56,9 @@ class BaselineAlgorithm( val state = chooseState(states) val newState = expandState(state, storage) if (newState != null) { + if (iterationCounter == 1) { + annotationHandler(initialState.signature) + } logger.info("Checking ${newState.signature.pythonTypeRepresentation()}") if (checkSignature(newState.signature as FunctionType, fileForMypyRuns, configFile)) { logger.debug("Found new state!") From 310c71b1c26f1b9352c2c9e5db5a4b67f42ced30 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Wed, 8 Mar 2023 08:28:38 +0300 Subject: [PATCH 4/4] Update fuzzing process: add duplicate ignoring, fix feedback returning --- .../kotlin/org/utbot/python/PythonEngine.kt | 203 ++++++++++-------- .../utbot/python/PythonTestCaseGenerator.kt | 14 +- ...nager.kt => TestGenerationLimitManager.kt} | 15 +- 3 files changed, 131 insertions(+), 101 deletions(-) rename utbot-python/src/main/kotlin/org/utbot/python/utils/{GenerationLimitManager.kt => TestGenerationLimitManager.kt} (78%) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index 94e33438b1..a123d8a76a 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -18,6 +18,7 @@ import org.utbot.python.newtyping.PythonTypeStorage import org.utbot.python.newtyping.general.Type import org.utbot.python.newtyping.pythonModules import org.utbot.python.newtyping.pythonTypeRepresentation +import org.utbot.python.utils.TestGenerationLimitManager import org.utbot.python.utils.camelToSnakeCase import org.utbot.summary.fuzzer.names.TestSuggestedInfo import java.net.ServerSocket @@ -80,17 +81,21 @@ class PythonEngine( } private fun handleSuccessResult( + arguments: List, types: List, evaluationResult: PythonEvaluationSuccess, methodUnderTestDescription: PythonMethodDescription, - hasThisObject: Boolean, - summary: List, ): FuzzingExecutionFeedback { val prohibitedExceptions = listOf( "builtins.AttributeError", "builtins.TypeError" ) + val summary = arguments + .zip(methodUnderTest.arguments) + .mapNotNull { it.first.summary?.replace("%var%", it.second.name) } + val hasThisObject = methodUnderTest.hasThisArgument + val resultModel = evaluationResult.stateAfter.getById(evaluationResult.resultId).toPythonTree(evaluationResult.stateAfter) if (evaluationResult.isException && (resultModel.type.name in prohibitedExceptions)) { // wrong type (sometimes mypy fails) @@ -138,30 +143,30 @@ class PythonEngine( ) } - fun fuzzing(parameters: List, isCancelled: () -> Boolean, until: Long): Flow = flow { + fun fuzzing(parameters: List, isCancelled: () -> Boolean, limitManager: TestGenerationLimitManager): Flow = flow { val additionalModules = parameters.flatMap { it.pythonModules() } ServerSocket(0).use { serverSocket -> - logger.info { "Server port: ${serverSocket.localPort}" } + logger.debug { "Server port: ${serverSocket.localPort}" } val manager = try { PythonWorkerManager( serverSocket, pythonPath, - until, + limitManager.until, { constructEvaluationInput(it) }, timeoutForRun.toInt() ) } catch (_: TimeoutException) { + logger.info { "Cannot connect to python executor" } return@flow } logger.info { "Executor manager was created successfully" } - fun fuzzingResultHandler( - description: PythonMethodDescription, - arguments: List - ): PythonExecutionResult? { + fun runWithFuzzedValues( + arguments: List, + ): PythonEvaluationResult? { val argumentValues = arguments.map { PythonTreeModel(it.tree, it.tree.type) } - logger.debug(argumentValues.map { it.tree } .toString()) + logger.debug(argumentValues.map { it.tree }.toString()) val argumentModules = argumentValues .flatMap { it.allContainingClassIds } .map { it.moduleName } @@ -179,58 +184,83 @@ class PythonEngine( modelList, methodUnderTest.argumentsNames ) - try { - return when (val evaluationResult = manager.run(functionArguments, localAdditionalModules)) { - is PythonEvaluationError -> { - val utError = UtError( - "Error evaluation: ${evaluationResult.status}, ${evaluationResult.message}", - Throwable(evaluationResult.stackTrace.joinToString("\n")) - ) - logger.debug(evaluationResult.stackTrace.joinToString("\n")) - PythonExecutionResult(InvalidExecution(utError), PythonFeedback(control = Control.PASS)) - } + return try { + manager.run(functionArguments, localAdditionalModules) + } catch (_: TimeoutException) { + logger.info { "Fuzzing process was interrupted by timeout" } + null + } + } - is PythonEvaluationTimeout -> { - val utError = UtError(evaluationResult.message, Throwable()) - PythonExecutionResult(InvalidExecution(utError), PythonFeedback(control = Control.PASS)) - } + fun handleExecutionResult( + result: PythonEvaluationResult, + arguments: List, + description: PythonMethodDescription, + ): Pair { + val executionFeedback: FuzzingExecutionFeedback + val fuzzingFeedback: PythonFeedback - is PythonEvaluationSuccess -> { - val coveredInstructions = evaluationResult.coverage.coveredInstructions - - val summary = arguments - .zip(methodUnderTest.arguments) - .mapNotNull { it.first.summary?.replace("%var%", it.second.name) } - - val hasThisObject = methodUnderTest.hasThisArgument - - when (val result = handleSuccessResult( - parameters, - evaluationResult, - description, - hasThisObject, - summary - )) { - is ValidExecution -> { - val trieNode: Trie.Node = description.tracer.add(coveredInstructions) - PythonExecutionResult( - result, - PythonFeedback(control = Control.CONTINUE, result = trieNode) - ) - } + when(result) { + is PythonEvaluationError -> { + val utError = UtError( + "Error evaluation: ${result.status}, ${result.message}", + Throwable(result.stackTrace.joinToString("\n")) + ) + logger.debug(result.stackTrace.joinToString("\n")) - is ArgumentsTypeErrorFeedback, is TypeErrorFeedback -> { - PythonExecutionResult(result, PythonFeedback(control = Control.PASS)) - } + limitManager.addSuccessExecution() + executionFeedback = InvalidExecution(utError) + fuzzingFeedback = PythonFeedback(control = Control.PASS) + return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), true) + } - is InvalidExecution -> { - PythonExecutionResult(result, PythonFeedback(control = Control.CONTINUE)) + is PythonEvaluationTimeout -> { + val utError = UtError(result.message, Throwable()) + limitManager.addInvalidExecution() + executionFeedback = InvalidExecution(utError) + fuzzingFeedback = PythonFeedback(control = Control.PASS) + return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false) + } + + is PythonEvaluationSuccess -> { + val coveredInstructions = result.coverage.coveredInstructions + executionFeedback = handleSuccessResult( + arguments, + parameters, + result, + description, + ) + + val trieNode: Trie.Node = description.tracer.add(coveredInstructions) + when (executionFeedback) { + is ValidExecution -> { + limitManager.addSuccessExecution() + if (trieNode.count > 1) { + fuzzingFeedback = PythonFeedback(control = Control.CONTINUE, result = trieNode) + return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false) } } + + is ArgumentsTypeErrorFeedback -> { + fuzzingFeedback = PythonFeedback(control = Control.PASS) + return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false) + } + + is TypeErrorFeedback -> { + limitManager.addInvalidExecution() + fuzzingFeedback = PythonFeedback(control = Control.PASS) + return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false) + } + + is InvalidExecution -> { + limitManager.addInvalidExecution() + fuzzingFeedback = PythonFeedback(control = Control.CONTINUE) + return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false) + } } + fuzzingFeedback = PythonFeedback(control = Control.CONTINUE, result = trieNode) + return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), true) } - } catch (_: TimeoutException) { - return null } } @@ -243,42 +273,47 @@ class PythonEngine( ) if (parameters.isEmpty()) { - val result = fuzzingResultHandler(pmd, emptyList()) + val result = runWithFuzzedValues(emptyList()) result?.let { - emit(it.fuzzingExecutionFeedback) + val (executionResult, needToEmit) = handleExecutionResult(result, emptyList(), pmd) + if (needToEmit) { + emit(executionResult.fuzzingExecutionFeedback) + } } manager.disconnect() } else { - PythonFuzzing(pmd.pythonTypeStorage) { description, arguments -> - if (isCancelled()) { - logger.info { "Fuzzing process was interrupted" } - manager.disconnect() - return@PythonFuzzing PythonFeedback(control = Control.STOP) - } - if (System.currentTimeMillis() >= until) { - logger.info { "Fuzzing process was interrupted by timeout" } - manager.disconnect() - return@PythonFuzzing PythonFeedback(control = Control.STOP) - } + try { + PythonFuzzing(pmd.pythonTypeStorage) { description, arguments -> + if (isCancelled()) { + logger.info { "Fuzzing process was interrupted" } + manager.disconnect() + return@PythonFuzzing PythonFeedback(control = Control.STOP) + } - val pair = Pair(description, arguments.map { PythonTreeWrapper(it.tree) }) - val mem = cache.get(pair) - if (mem != null) { - logger.debug("Repeat in fuzzing") - emit(mem.fuzzingExecutionFeedback) - return@PythonFuzzing mem.fuzzingPlatformFeedback - } - val result = fuzzingResultHandler(description, arguments) - if (result == null) { // timeout - logger.info { "Fuzzing process was interrupted by timeout" } - manager.disconnect() - return@PythonFuzzing PythonFeedback(control = Control.STOP) - } + val pair = Pair(description, arguments.map { PythonTreeWrapper(it.tree) }) + val mem = cache.get(pair) + if (mem != null) { + logger.debug("Repeat in fuzzing") + return@PythonFuzzing mem.fuzzingPlatformFeedback + } - cache.add(pair, result) - emit(result.fuzzingExecutionFeedback) - return@PythonFuzzing result.fuzzingPlatformFeedback - }.fuzz(pmd) + val result = runWithFuzzedValues(arguments) + if (result == null) { // timeout + manager.disconnect() + return@PythonFuzzing PythonFeedback(control = Control.STOP) + } + + val (executionResult, needToEmit) = handleExecutionResult(result, arguments, description) + cache.add(pair, executionResult) + if (needToEmit) { + emit(executionResult.fuzzingExecutionFeedback) + } + return@PythonFuzzing executionResult.fuzzingPlatformFeedback + }.fuzz(pmd) + } catch (ex: Exception) { // NoSeedValueException + logger.info { "Cannot fuzz values for types: $parameters" } + } + manager.disconnect() } } } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt index 4304478cf6..33a419ecc2 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt @@ -23,7 +23,7 @@ import org.utbot.python.newtyping.mypy.getErrorNumber import org.utbot.python.newtyping.utils.getOffsetLine import org.utbot.python.typing.MypyAnnotations import org.utbot.python.utils.ExecutionWithTimeoutMode -import org.utbot.python.utils.GenerationLimitManager +import org.utbot.python.utils.TestGenerationLimitManager import org.utbot.python.utils.PriorityCartesianProduct import org.utbot.python.utils.TimeoutMode import java.io.File @@ -119,7 +119,7 @@ class PythonTestCaseGenerator( fun generate(method: PythonMethod, until: Long): PythonTestSet { storageForMypyMessages.clear() - val limitManager = GenerationLimitManager( + val limitManager = TestGenerationLimitManager( ExecutionWithTimeoutMode, until, ) @@ -166,16 +166,14 @@ class PythonTestCaseGenerator( var feedback: InferredTypeFeedback = SuccessFeedback val fuzzerCancellation = { isCancelled() || limitManager.isCancelled() } - val startTime = System.currentTimeMillis() - engine.fuzzing(args, fuzzerCancellation, until).collect { + engine.fuzzing(args, fuzzerCancellation, limitManager).collect { generated += 1 when (it) { is ValidExecution -> { executions += it.utFuzzedExecution missingLines = updateCoverage(it.utFuzzedExecution, coveredLines, missingLines) feedback = SuccessFeedback - limitManager.addSuccessExecution() } is InvalidExecution -> { errors += it.utError @@ -183,16 +181,12 @@ class PythonTestCaseGenerator( } is ArgumentsTypeErrorFeedback -> { feedback = InvalidTypeFeedback - limitManager.addInvalidExecution() } is TypeErrorFeedback -> { feedback = InvalidTypeFeedback - limitManager.addInvalidExecution() } } limitManager.missedLines = missingLines?.size - - logger.info { "Time ${System.currentTimeMillis() - startTime}: $generated, $missingLines" } } limitManager.restart() feedback @@ -235,7 +229,7 @@ class PythonTestCaseGenerator( hintCollector: HintCollector, report: List, mypyConfigFile: File, - limitManager: GenerationLimitManager, + limitManager: TestGenerationLimitManager, annotationHandler: suspend (Type) -> InferredTypeFeedback, ) { val namesInModule = mypyStorage.names diff --git a/utbot-python/src/main/kotlin/org/utbot/python/utils/GenerationLimitManager.kt b/utbot-python/src/main/kotlin/org/utbot/python/utils/TestGenerationLimitManager.kt similarity index 78% rename from utbot-python/src/main/kotlin/org/utbot/python/utils/GenerationLimitManager.kt rename to utbot-python/src/main/kotlin/org/utbot/python/utils/TestGenerationLimitManager.kt index 83166effc7..fe2fd5a80d 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/utils/GenerationLimitManager.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/utils/TestGenerationLimitManager.kt @@ -2,7 +2,7 @@ package org.utbot.python.utils import kotlin.math.min -class GenerationLimitManager( +class TestGenerationLimitManager( // global settings var mode: LimitManagerMode, val until: Long, @@ -28,6 +28,7 @@ class GenerationLimitManager( fun addSuccessExecution() { executions -= 1 } + fun addInvalidExecution() { invalidExecutions -= 1 } @@ -38,23 +39,23 @@ class GenerationLimitManager( } interface LimitManagerMode { - fun isCancelled(manager: GenerationLimitManager): Boolean + fun isCancelled(manager: TestGenerationLimitManager): Boolean } object MaxCoverageMode : LimitManagerMode { - override fun isCancelled(manager: GenerationLimitManager): Boolean { + override fun isCancelled(manager: TestGenerationLimitManager): Boolean { return manager.missedLines?.equals(0) == true } } object TimeoutMode : LimitManagerMode { - override fun isCancelled(manager: GenerationLimitManager): Boolean { + override fun isCancelled(manager: TestGenerationLimitManager): Boolean { return System.currentTimeMillis() >= manager.until } } object ExecutionMode : LimitManagerMode { - override fun isCancelled(manager: GenerationLimitManager): Boolean { + override fun isCancelled(manager: TestGenerationLimitManager): Boolean { if (manager.invalidExecutions <= 0 || manager.executions <= 0) { return min(manager.invalidExecutions, 0) + min(manager.executions, 0) <= manager.additionalExecutions } @@ -63,13 +64,13 @@ object ExecutionMode : LimitManagerMode { } object MaxCoverageWithTimeoutMode : LimitManagerMode { - override fun isCancelled(manager: GenerationLimitManager): Boolean { + override fun isCancelled(manager: TestGenerationLimitManager): Boolean { return MaxCoverageMode.isCancelled(manager) || TimeoutMode.isCancelled(manager) } } object ExecutionWithTimeoutMode : LimitManagerMode { - override fun isCancelled(manager: GenerationLimitManager): Boolean { + override fun isCancelled(manager: TestGenerationLimitManager): Boolean { return ExecutionMode.isCancelled(manager) || TimeoutMode.isCancelled(manager) } }