diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index aecc2be86a..f7a5101b6a 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -216,14 +216,12 @@ class PythonEngine( PythonFeedback(control = Control.CONTINUE, result = trieNode) ) } - - is ArgumentsTypeErrorFeedback, is TypeErrorFeedback -> { - PythonExecutionResult(result, PythonFeedback(control = Control.PASS)) - } - is InvalidExecution -> { PythonExecutionResult(result, PythonFeedback(control = Control.CONTINUE)) } + else -> { + PythonExecutionResult(result, PythonFeedback(control = Control.PASS)) + } } } } @@ -262,7 +260,7 @@ class PythonEngine( if (arguments.any { PythonTree.containsFakeNode(it.tree) }) { logger.debug("FakeNode in Python model") - emit(InvalidExecution(UtError("Bad input object", Throwable()))) + emit(FakeNodeFeedback) return@PythonFuzzing PythonFeedback(control = Control.CONTINUE) } @@ -270,12 +268,11 @@ class PythonEngine( val mem = cache.get(pair) if (mem != null) { logger.debug("Repeat in fuzzing") - emit(mem.fuzzingExecutionFeedback) + emit(CachedExecutionFeedback(mem.fuzzingExecutionFeedback)) return@PythonFuzzing mem.fuzzingPlatformFeedback } val result = fuzzingResultHandler(description, arguments) if (result == null) { // timeout - logger.info { "Fuzzing process was interrupted by timeout" } manager.disconnect() return@PythonFuzzing PythonFeedback(control = Control.STOP) } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt index 81c335ea7e..c131d2779e 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt @@ -30,7 +30,6 @@ import java.io.File private val logger = KotlinLogging.logger {} private const val RANDOM_TYPE_FREQUENCY = 6 -private const val MAX_EXECUTIONS = 50000 class PythonTestCaseGenerator( private val withMinimization: Boolean = true, @@ -168,9 +167,7 @@ class PythonTestCaseGenerator( var feedback: InferredTypeFeedback = SuccessFeedback - val fuzzerCancellation = { - isCancelled() || limitManager.isCancelled() || (errors.size + executions.size) >= MAX_EXECUTIONS - } + val fuzzerCancellation = { isCancelled() || limitManager.isCancelled() } engine.fuzzing(args, fuzzerCancellation, until).collect { when (it) { @@ -182,8 +179,8 @@ class PythonTestCaseGenerator( } is InvalidExecution -> { errors += it.utError - feedback = SuccessFeedback - limitManager.addSuccessExecution() + feedback = InvalidTypeFeedback + limitManager.addInvalidExecution() } is ArgumentsTypeErrorFeedback -> { feedback = InvalidTypeFeedback @@ -193,6 +190,19 @@ class PythonTestCaseGenerator( feedback = InvalidTypeFeedback limitManager.addInvalidExecution() } + is CachedExecutionFeedback -> { + when (it.cachedFeedback) { + is ValidExecution -> { + limitManager.addSuccessExecution() + } + else -> { + limitManager.addInvalidExecution() + } + } + } + is FakeNodeFeedback -> { + limitManager.addFakeNodeExecutions() + } } limitManager.missedLines = missingLines?.size } @@ -213,28 +223,41 @@ class PythonTestCaseGenerator( val coveredLines = mutableSetOf() logger.info("Start test generation for ${method.name}") - val meta = method.definition.type.pythonDescription() as PythonCallableTypeDescription - val argKinds = meta.argumentKinds - if (argKinds.any { it != PythonCallableTypeDescription.ArgKind.ARG_POS }) { - val now = System.currentTimeMillis() - val firstUntil = (until - now) / 2 + now - val originalDef = method.definition - val shortType = meta.removeNonPositionalArgs(originalDef.type) - val shortMeta = PythonFuncItemDescription( - originalDef.meta.name, - originalDef.meta.args.take(shortType.arguments.size) - ) - val additionalVars = originalDef.meta.args - .drop(shortType.arguments.size) - .joinToString(separator="\n", prefix="\n") { arg -> - "${arg.name}: ${pythonAnyType.pythonTypeRepresentation()}" // TODO: better types - } - method.definition = PythonFunctionDefinition(shortMeta, shortType) - val missingLines = methodHandler(method, typeStorage, coveredLines, errors, executions, null, firstUntil, additionalVars) - method.definition = originalDef - methodHandler(method, typeStorage, coveredLines, errors, executions, missingLines, until) - } else { - methodHandler(method, typeStorage, coveredLines, errors, executions, null, until) + try { + val meta = method.definition.type.pythonDescription() as PythonCallableTypeDescription + val argKinds = meta.argumentKinds + if (argKinds.any { it != PythonCallableTypeDescription.ArgKind.ARG_POS }) { + val now = System.currentTimeMillis() + val firstUntil = (until - now) / 2 + now + val originalDef = method.definition + val shortType = meta.removeNonPositionalArgs(originalDef.type) + val shortMeta = PythonFuncItemDescription( + originalDef.meta.name, + originalDef.meta.args.take(shortType.arguments.size) + ) + val additionalVars = originalDef.meta.args + .drop(shortType.arguments.size) + .joinToString(separator = "\n", prefix = "\n") { arg -> + "${arg.name}: ${pythonAnyType.pythonTypeRepresentation()}" // TODO: better types + } + method.definition = PythonFunctionDefinition(shortMeta, shortType) + val missingLines = methodHandler( + method, + typeStorage, + coveredLines, + errors, + executions, + null, + firstUntil, + additionalVars + ) + method.definition = originalDef + methodHandler(method, typeStorage, coveredLines, errors, executions, missingLines, until) + } else { + methodHandler(method, typeStorage, coveredLines, errors, executions, null, until) + } + } catch (_: OutOfMemoryError) { + logger.info { "Out of memory error. Stop test generation process" } } logger.info("Collect all test executions for ${method.name}") diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt index d87c40a3da..9e6cae35a5 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt @@ -16,9 +16,7 @@ import org.utbot.fuzzing.utils.Trie import org.utbot.python.framework.api.python.PythonTree import org.utbot.python.fuzzing.provider.* import org.utbot.python.fuzzing.provider.utils.isAny -import org.utbot.python.fuzzing.provider.utils.isConcreteType import org.utbot.python.newtyping.* -import org.utbot.python.newtyping.general.DefaultSubstitutionProvider import org.utbot.python.newtyping.general.Type private val logger = KotlinLogging.logger {} @@ -42,6 +40,8 @@ class ValidExecution(val utFuzzedExecution: UtFuzzedExecution): FuzzingExecution class InvalidExecution(val utError: UtError): FuzzingExecutionFeedback class TypeErrorFeedback(val message: String) : FuzzingExecutionFeedback class ArgumentsTypeErrorFeedback(val message: String) : FuzzingExecutionFeedback +class CachedExecutionFeedback(val cachedFeedback: FuzzingExecutionFeedback) : FuzzingExecutionFeedback +object FakeNodeFeedback : FuzzingExecutionFeedback data class PythonExecutionResult( val fuzzingExecutionFeedback: FuzzingExecutionFeedback, diff --git a/utbot-python/src/main/kotlin/org/utbot/python/utils/TestGenerationLimitManager.kt b/utbot-python/src/main/kotlin/org/utbot/python/utils/TestGenerationLimitManager.kt index bf0fcc6b55..b731bf3ffd 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/utils/TestGenerationLimitManager.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/utils/TestGenerationLimitManager.kt @@ -1,7 +1,5 @@ package org.utbot.python.utils -import kotlin.math.min - class TestGenerationLimitManager( // global settings var mode: LimitManagerMode, @@ -10,18 +8,18 @@ class TestGenerationLimitManager( // local settings: one type inference iteration var executions: Int = 150, var invalidExecutions: Int = 10, - var additionalExecutions: Int = 5, + var fakeNodeExecutions: Int = 20, var missedLines: Int? = null, ) { private val initExecution = executions private val initInvalidExecutions = invalidExecutions - private val initAdditionalExecutions = additionalExecutions + private val initFakeNodeExecutions = fakeNodeExecutions private val initMissedLines = missedLines fun restart() { executions = initExecution invalidExecutions = initInvalidExecutions - additionalExecutions = initAdditionalExecutions + fakeNodeExecutions = initFakeNodeExecutions missedLines = initMissedLines } @@ -33,6 +31,10 @@ class TestGenerationLimitManager( invalidExecutions -= 1 } + fun addFakeNodeExecutions() { + fakeNodeExecutions -= 1 + } + fun isCancelled(): Boolean { return mode.isCancelled(this) } @@ -56,10 +58,7 @@ object TimeoutMode : LimitManagerMode { object ExecutionMode : LimitManagerMode { override fun isCancelled(manager: TestGenerationLimitManager): Boolean { - if (manager.invalidExecutions <= 0 || manager.executions <= 0) { - return min(manager.invalidExecutions, 0) + min(manager.executions, 0) + manager.additionalExecutions <= 0 - } - return false + return manager.invalidExecutions <= 0 || manager.executions <= 0 || manager.fakeNodeExecutions <= 0 } }