diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index b344b42163..a123d8a76a 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -18,6 +18,7 @@ import org.utbot.python.newtyping.PythonTypeStorage import org.utbot.python.newtyping.general.Type import org.utbot.python.newtyping.pythonModules import org.utbot.python.newtyping.pythonTypeRepresentation +import org.utbot.python.utils.TestGenerationLimitManager import org.utbot.python.utils.camelToSnakeCase import org.utbot.summary.fuzzer.names.TestSuggestedInfo import java.net.ServerSocket @@ -31,7 +32,6 @@ class PythonEngine( private val pythonPath: String, private val fuzzedConcreteValues: List, private val timeoutForRun: Long, - private val initialCoveredLines: Set, private val pythonTypeStorage: PythonTypeStorage, ) { @@ -81,17 +81,21 @@ class PythonEngine( } private fun handleSuccessResult( + arguments: List, types: List, evaluationResult: PythonEvaluationSuccess, methodUnderTestDescription: PythonMethodDescription, - hasThisObject: Boolean, - summary: List, ): FuzzingExecutionFeedback { val prohibitedExceptions = listOf( "builtins.AttributeError", "builtins.TypeError" ) + val summary = arguments + .zip(methodUnderTest.arguments) + .mapNotNull { it.first.summary?.replace("%var%", it.second.name) } + val hasThisObject = methodUnderTest.hasThisArgument + val resultModel = evaluationResult.stateAfter.getById(evaluationResult.resultId).toPythonTree(evaluationResult.stateAfter) if (evaluationResult.isException && (resultModel.type.name in prohibitedExceptions)) { // wrong type (sometimes mypy fails) @@ -139,27 +143,30 @@ class PythonEngine( ) } - fun fuzzing(parameters: List, isCancelled: () -> Boolean, until: Long): Flow = flow { + fun fuzzing(parameters: List, isCancelled: () -> Boolean, limitManager: TestGenerationLimitManager): Flow = flow { val additionalModules = parameters.flatMap { it.pythonModules() } - val coveredLines = initialCoveredLines.toMutableSet() ServerSocket(0).use { serverSocket -> - logger.info { "Server port: ${serverSocket.localPort}" } - val manager = PythonWorkerManager( - serverSocket, - pythonPath, - until, - { constructEvaluationInput(it) }, - timeoutForRun.toInt() - ) + logger.debug { "Server port: ${serverSocket.localPort}" } + val manager = try { + PythonWorkerManager( + serverSocket, + pythonPath, + limitManager.until, + { constructEvaluationInput(it) }, + timeoutForRun.toInt() + ) + } catch (_: TimeoutException) { + logger.info { "Cannot connect to python executor" } + return@flow + } logger.info { "Executor manager was created successfully" } - fun fuzzingResultHandler( - description: PythonMethodDescription, - arguments: List - ): PythonExecutionResult? { + fun runWithFuzzedValues( + arguments: List, + ): PythonEvaluationResult? { val argumentValues = arguments.map { PythonTreeModel(it.tree, it.tree.type) } - logger.debug(argumentValues.map { it.tree } .toString()) + logger.debug(argumentValues.map { it.tree }.toString()) val argumentModules = argumentValues .flatMap { it.allContainingClassIds } .map { it.moduleName } @@ -177,59 +184,83 @@ class PythonEngine( modelList, methodUnderTest.argumentsNames ) - try { - return when (val evaluationResult = manager.run(functionArguments, localAdditionalModules)) { - is PythonEvaluationError -> { - val utError = UtError( - "Error evaluation: ${evaluationResult.status}, ${evaluationResult.message}", - Throwable(evaluationResult.stackTrace.joinToString("\n")) - ) - logger.debug(evaluationResult.stackTrace.joinToString("\n")) - PythonExecutionResult(InvalidExecution(utError), PythonFeedback(control = Control.PASS)) - } + return try { + manager.run(functionArguments, localAdditionalModules) + } catch (_: TimeoutException) { + logger.info { "Fuzzing process was interrupted by timeout" } + null + } + } - is PythonEvaluationTimeout -> { - val utError = UtError(evaluationResult.message, Throwable()) - PythonExecutionResult(InvalidExecution(utError), PythonFeedback(control = Control.PASS)) - } + fun handleExecutionResult( + result: PythonEvaluationResult, + arguments: List, + description: PythonMethodDescription, + ): Pair { + val executionFeedback: FuzzingExecutionFeedback + val fuzzingFeedback: PythonFeedback - is PythonEvaluationSuccess -> { - val coveredInstructions = evaluationResult.coverage.coveredInstructions - coveredInstructions.forEach { coveredLines.add(it.lineNumber) } - - val summary = arguments - .zip(methodUnderTest.arguments) - .mapNotNull { it.first.summary?.replace("%var%", it.second.name) } - - val hasThisObject = methodUnderTest.hasThisArgument - - when (val result = handleSuccessResult( - parameters, - evaluationResult, - description, - hasThisObject, - summary - )) { - is ValidExecution -> { - val trieNode: Trie.Node = description.tracer.add(coveredInstructions) - PythonExecutionResult( - result, - PythonFeedback(control = Control.CONTINUE, result = trieNode) - ) - } + when(result) { + is PythonEvaluationError -> { + val utError = UtError( + "Error evaluation: ${result.status}, ${result.message}", + Throwable(result.stackTrace.joinToString("\n")) + ) + logger.debug(result.stackTrace.joinToString("\n")) - is ArgumentsTypeErrorFeedback, is TypeErrorFeedback -> { - PythonExecutionResult(result, PythonFeedback(control = Control.PASS)) - } + limitManager.addSuccessExecution() + executionFeedback = InvalidExecution(utError) + fuzzingFeedback = PythonFeedback(control = Control.PASS) + return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), true) + } + + is PythonEvaluationTimeout -> { + val utError = UtError(result.message, Throwable()) + limitManager.addInvalidExecution() + executionFeedback = InvalidExecution(utError) + fuzzingFeedback = PythonFeedback(control = Control.PASS) + return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false) + } + + is PythonEvaluationSuccess -> { + val coveredInstructions = result.coverage.coveredInstructions + executionFeedback = handleSuccessResult( + arguments, + parameters, + result, + description, + ) - is InvalidExecution -> { - PythonExecutionResult(result, PythonFeedback(control = Control.CONTINUE)) + val trieNode: Trie.Node = description.tracer.add(coveredInstructions) + when (executionFeedback) { + is ValidExecution -> { + limitManager.addSuccessExecution() + if (trieNode.count > 1) { + fuzzingFeedback = PythonFeedback(control = Control.CONTINUE, result = trieNode) + return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false) } } + + is ArgumentsTypeErrorFeedback -> { + fuzzingFeedback = PythonFeedback(control = Control.PASS) + return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false) + } + + is TypeErrorFeedback -> { + limitManager.addInvalidExecution() + fuzzingFeedback = PythonFeedback(control = Control.PASS) + return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false) + } + + is InvalidExecution -> { + limitManager.addInvalidExecution() + fuzzingFeedback = PythonFeedback(control = Control.CONTINUE) + return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false) + } } + fuzzingFeedback = PythonFeedback(control = Control.CONTINUE, result = trieNode) + return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), true) } - } catch (_: TimeoutException) { - return null } } @@ -242,43 +273,47 @@ class PythonEngine( ) if (parameters.isEmpty()) { - val result = fuzzingResultHandler(pmd, emptyList()) + val result = runWithFuzzedValues(emptyList()) result?.let { - emit(it.fuzzingExecutionFeedback) + val (executionResult, needToEmit) = handleExecutionResult(result, emptyList(), pmd) + if (needToEmit) { + emit(executionResult.fuzzingExecutionFeedback) + } } manager.disconnect() } else { - PythonFuzzing(pmd.pythonTypeStorage) { description, arguments -> - if (isCancelled()) { - logger.info { "Fuzzing process was interrupted" } - manager.disconnect() - return@PythonFuzzing PythonFeedback(control = Control.STOP) - } - if (System.currentTimeMillis() >= until) { - logger.info { "Fuzzing process was interrupted by timeout" } - manager.disconnect() - return@PythonFuzzing PythonFeedback(control = Control.STOP) - } + try { + PythonFuzzing(pmd.pythonTypeStorage) { description, arguments -> + if (isCancelled()) { + logger.info { "Fuzzing process was interrupted" } + manager.disconnect() + return@PythonFuzzing PythonFeedback(control = Control.STOP) + } - val pair = Pair(description, arguments.map { PythonTreeWrapper(it.tree) }) - val mem = cache.get(pair) - if (mem != null) { - logger.debug("Repeat in fuzzing") - emit(mem.fuzzingExecutionFeedback) - return@PythonFuzzing mem.fuzzingPlatformFeedback - } - val result = fuzzingResultHandler(description, arguments) - if (result == null) { // timeout - logger.info { "Fuzzing process was interrupted by timeout" } - manager.disconnect() - return@PythonFuzzing PythonFeedback(control = Control.STOP) - } + val pair = Pair(description, arguments.map { PythonTreeWrapper(it.tree) }) + val mem = cache.get(pair) + if (mem != null) { + logger.debug("Repeat in fuzzing") + return@PythonFuzzing mem.fuzzingPlatformFeedback + } - cache.add(pair, result) - emit(result.fuzzingExecutionFeedback) - return@PythonFuzzing result.fuzzingPlatformFeedback + val result = runWithFuzzedValues(arguments) + if (result == null) { // timeout + manager.disconnect() + return@PythonFuzzing PythonFeedback(control = Control.STOP) + } - }.fuzz(pmd) + val (executionResult, needToEmit) = handleExecutionResult(result, arguments, description) + cache.add(pair, executionResult) + if (needToEmit) { + emit(executionResult.fuzzingExecutionFeedback) + } + return@PythonFuzzing executionResult.fuzzingPlatformFeedback + }.fuzz(pmd) + } catch (ex: Exception) { // NoSeedValueException + logger.info { "Cannot fuzz values for types: $parameters" } + } + manager.disconnect() } } } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt index 9215bff3c2..33a419ecc2 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt @@ -22,15 +22,14 @@ import org.utbot.python.newtyping.mypy.MypyReportLine import org.utbot.python.newtyping.mypy.getErrorNumber import org.utbot.python.newtyping.utils.getOffsetLine import org.utbot.python.typing.MypyAnnotations +import org.utbot.python.utils.ExecutionWithTimeoutMode +import org.utbot.python.utils.TestGenerationLimitManager import org.utbot.python.utils.PriorityCartesianProduct +import org.utbot.python.utils.TimeoutMode import java.io.File private val logger = KotlinLogging.logger {} -private const val COVERAGE_LIMIT = 150 -private const val ADDITIONAL_LIMIT = 5 -private const val INVALID_EXECUTION_LIMIT = 10 - class PythonTestCaseGenerator( private val withMinimization: Boolean = true, private val directoriesForSysPath: Set, @@ -120,6 +119,10 @@ class PythonTestCaseGenerator( fun generate(method: PythonMethod, until: Long): PythonTestSet { storageForMypyMessages.clear() + val limitManager = TestGenerationLimitManager( + ExecutionWithTimeoutMode, + until, + ) val typeStorage = PythonTypeStorage.get(mypyStorage) @@ -135,10 +138,6 @@ class PythonTestCaseGenerator( val coveredLines = mutableSetOf() var generated = 0 - var additionalLimit = ADDITIONAL_LIMIT - val typeInferenceCancellation = - { isCancelled() || System.currentTimeMillis() >= until || additionalLimit <= 0 } - logger.info("Start test generation for ${method.name}") substituteTypeParameters(method, typeStorage).forEach { newMethod -> inferAnnotations( @@ -148,7 +147,7 @@ class PythonTestCaseGenerator( hintCollector, mypyReportLine, mypyConfigFile, - typeInferenceCancellation + limitManager, ) { functionType -> val args = (functionType as FunctionType).arguments @@ -161,25 +160,14 @@ class PythonTestCaseGenerator( pythonPath, constants, timeoutForRun, - coveredLines, PythonTypeStorage.get(mypyStorage) ) - var invalidExecutionLimit = INVALID_EXECUTION_LIMIT - var coverageLimit = COVERAGE_LIMIT - var coveredBefore = coveredLines.size - var feedback: InferredTypeFeedback = SuccessFeedback - val fuzzerCancellation = { - typeInferenceCancellation() - || coverageLimit == 0 - || additionalLimit == 0 - || invalidExecutionLimit == 0 - } - val startTime = System.currentTimeMillis() + val fuzzerCancellation = { isCancelled() || limitManager.isCancelled() } - engine.fuzzing(args, fuzzerCancellation, until).collect { + engine.fuzzing(args, fuzzerCancellation, limitManager).collect { generated += 1 when (it) { is ValidExecution -> { @@ -192,24 +180,15 @@ class PythonTestCaseGenerator( feedback = SuccessFeedback } is ArgumentsTypeErrorFeedback -> { - invalidExecutionLimit-- feedback = InvalidTypeFeedback } is TypeErrorFeedback -> { - invalidExecutionLimit-- feedback = InvalidTypeFeedback } } - if (missingLines?.size == 0) { - additionalLimit-- - } - val coveredAfter = coveredLines.size - if (coveredAfter == coveredBefore) { - coverageLimit-- - } - logger.info { "Time ${System.currentTimeMillis() - startTime}: $generated, $missingLines" } - coveredBefore = coveredAfter + limitManager.missedLines = missingLines?.size } + limitManager.restart() feedback } } @@ -250,7 +229,7 @@ class PythonTestCaseGenerator( hintCollector: HintCollector, report: List, mypyConfigFile: File, - isCancelled: () -> Boolean, + limitManager: TestGenerationLimitManager, annotationHandler: suspend (Type) -> InferredTypeFeedback, ) { val namesInModule = mypyStorage.names @@ -259,6 +238,7 @@ class PythonTestCaseGenerator( .filter { it.length < 4 || !it.startsWith("__") || !it.endsWith("__") } + val typeInferenceCancellation = { isCancelled() || limitManager.isCancelled() } val algo = BaselineAlgorithm( typeStorage, @@ -277,14 +257,15 @@ class PythonTestCaseGenerator( ) runBlocking breaking@{ - if (isCancelled()) { + if (typeInferenceCancellation()) { return@breaking } - algo.run(hintCollector.result, isCancelled, annotationHandler) + val iterationNumber = algo.run(hintCollector.result, typeInferenceCancellation, annotationHandler) - val existsAnnotation = method.definition.type - if (existsAnnotation.arguments.all { it.pythonTypeName() != "typing.Any" }) { + if (iterationNumber == 1) { + limitManager.mode = TimeoutMode + val existsAnnotation = method.definition.type annotationHandler(existsAnnotation) } } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt index fd98864bf8..31ca73db03 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt @@ -24,8 +24,8 @@ class PythonWorkerManager( var timeout: Long = 0 lateinit var process: Process - lateinit var workerSocket: Socket - lateinit var codeExecutor: PythonCodeExecutor + private lateinit var workerSocket: Socket + private lateinit var codeExecutor: PythonCodeExecutor init { connect() @@ -39,7 +39,7 @@ class PythonWorkerManager( "localhost", serverSocket.localPort.toString(), "--logfile", logfile.absolutePath, - //"--loglevel", "DEBUG", + "--loglevel", "INFO", // "DEBUG", "INFO", "ERROR" )) timeout = max(until - processStartTime, 0) workerSocket = try { diff --git a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/TypeInferenceAPI.kt b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/TypeInferenceAPI.kt index 8bd05be12c..babcd9a49b 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/TypeInferenceAPI.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/TypeInferenceAPI.kt @@ -8,7 +8,7 @@ abstract class TypeInferenceAlgorithm { hintCollectorResult: HintCollectorResult, isCancelled: () -> Boolean, annotationHandler: suspend (Type) -> InferredTypeFeedback, - ) + ): Int } sealed interface InferredTypeFeedback diff --git a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt index 4466a9abb8..48aa0cae02 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt @@ -39,20 +39,26 @@ class BaselineAlgorithm( hintCollectorResult: HintCollectorResult, isCancelled: () -> Boolean, annotationHandler: suspend (Type) -> InferredTypeFeedback, - ) { + ): Int { val generalRating = createGeneralTypeRating(hintCollectorResult, storage) val initialState = getInitialState(hintCollectorResult, generalRating) val states: MutableList = mutableListOf(initialState) val fileForMypyRuns = TemporaryFileManager.assignTemporaryFile(tag = "mypy.py") + var iterationCounter = 0 run breaking@ { while (states.isNotEmpty()) { if (isCancelled()) return@breaking logger.debug("State number: ${states.size}") + iterationCounter++ + val state = chooseState(states) val newState = expandState(state, storage) if (newState != null) { + if (iterationCounter == 1) { + annotationHandler(initialState.signature) + } logger.info("Checking ${newState.signature.pythonTypeRepresentation()}") if (checkSignature(newState.signature as FunctionType, fileForMypyRuns, configFile)) { logger.debug("Found new state!") @@ -71,6 +77,7 @@ class BaselineAlgorithm( } } } + return iterationCounter } private fun checkSignature(signature: FunctionType, fileForMypyRuns: File, configFile: File): Boolean { diff --git a/utbot-python/src/main/kotlin/org/utbot/python/utils/TestGenerationLimitManager.kt b/utbot-python/src/main/kotlin/org/utbot/python/utils/TestGenerationLimitManager.kt new file mode 100644 index 0000000000..fe2fd5a80d --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/utils/TestGenerationLimitManager.kt @@ -0,0 +1,76 @@ +package org.utbot.python.utils + +import kotlin.math.min + +class TestGenerationLimitManager( + // global settings + var mode: LimitManagerMode, + val until: Long, + + // local settings: one type inference iteration + var executions: Int = 150, + var invalidExecutions: Int = 10, + var additionalExecutions: Int = 5, + var missedLines: Int? = null, +) { + private val initExecution = executions + private val initInvalidExecutions = invalidExecutions + private val initAdditionalExecutions = additionalExecutions + private val initMissedLines = missedLines + + fun restart() { + executions = initExecution + invalidExecutions = initInvalidExecutions + additionalExecutions = initAdditionalExecutions + missedLines = initMissedLines + } + + fun addSuccessExecution() { + executions -= 1 + } + + fun addInvalidExecution() { + invalidExecutions -= 1 + } + + fun isCancelled(): Boolean { + return mode.isCancelled(this) + } +} + +interface LimitManagerMode { + fun isCancelled(manager: TestGenerationLimitManager): Boolean +} + +object MaxCoverageMode : LimitManagerMode { + override fun isCancelled(manager: TestGenerationLimitManager): Boolean { + return manager.missedLines?.equals(0) == true + } +} + +object TimeoutMode : LimitManagerMode { + override fun isCancelled(manager: TestGenerationLimitManager): Boolean { + return System.currentTimeMillis() >= manager.until + } +} + +object ExecutionMode : LimitManagerMode { + override fun isCancelled(manager: TestGenerationLimitManager): Boolean { + if (manager.invalidExecutions <= 0 || manager.executions <= 0) { + return min(manager.invalidExecutions, 0) + min(manager.executions, 0) <= manager.additionalExecutions + } + return false + } +} + +object MaxCoverageWithTimeoutMode : LimitManagerMode { + override fun isCancelled(manager: TestGenerationLimitManager): Boolean { + return MaxCoverageMode.isCancelled(manager) || TimeoutMode.isCancelled(manager) + } +} + +object ExecutionWithTimeoutMode : LimitManagerMode { + override fun isCancelled(manager: TestGenerationLimitManager): Boolean { + return ExecutionMode.isCancelled(manager) || TimeoutMode.isCancelled(manager) + } +}