From 1669370f0636c1b5e4c7d8a4354fbbde3a051271 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Thu, 9 Mar 2023 14:43:23 +0300 Subject: [PATCH 1/3] Revert fuzzing minimization --- .../kotlin/org/utbot/python/PythonEngine.kt | 157 +++++++----------- .../utbot/python/PythonTestCaseGenerator.kt | 6 +- 2 files changed, 67 insertions(+), 96 deletions(-) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index a123d8a76a..840aa5aa05 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -18,7 +18,6 @@ import org.utbot.python.newtyping.PythonTypeStorage import org.utbot.python.newtyping.general.Type import org.utbot.python.newtyping.pythonModules import org.utbot.python.newtyping.pythonTypeRepresentation -import org.utbot.python.utils.TestGenerationLimitManager import org.utbot.python.utils.camelToSnakeCase import org.utbot.summary.fuzzer.names.TestSuggestedInfo import java.net.ServerSocket @@ -143,30 +142,30 @@ class PythonEngine( ) } - fun fuzzing(parameters: List, isCancelled: () -> Boolean, limitManager: TestGenerationLimitManager): Flow = flow { + fun fuzzing(parameters: List, isCancelled: () -> Boolean, until: Long): Flow = flow { val additionalModules = parameters.flatMap { it.pythonModules() } ServerSocket(0).use { serverSocket -> - logger.debug { "Server port: ${serverSocket.localPort}" } + logger.info { "Server port: ${serverSocket.localPort}" } val manager = try { PythonWorkerManager( serverSocket, pythonPath, - limitManager.until, + until, { constructEvaluationInput(it) }, timeoutForRun.toInt() ) } catch (_: TimeoutException) { - logger.info { "Cannot connect to python executor" } return@flow } logger.info { "Executor manager was created successfully" } - fun runWithFuzzedValues( - arguments: List, - ): PythonEvaluationResult? { + fun fuzzingResultHandler( + description: PythonMethodDescription, + arguments: List + ): PythonExecutionResult? { val argumentValues = arguments.map { PythonTreeModel(it.tree, it.tree.type) } - logger.debug(argumentValues.map { it.tree }.toString()) + logger.debug(argumentValues.map { it.tree } .toString()) val argumentModules = argumentValues .flatMap { it.allContainingClassIds } .map { it.moduleName } @@ -184,83 +183,52 @@ class PythonEngine( modelList, methodUnderTest.argumentsNames ) - return try { - manager.run(functionArguments, localAdditionalModules) - } catch (_: TimeoutException) { - logger.info { "Fuzzing process was interrupted by timeout" } - null - } - } - - fun handleExecutionResult( - result: PythonEvaluationResult, - arguments: List, - description: PythonMethodDescription, - ): Pair { - val executionFeedback: FuzzingExecutionFeedback - val fuzzingFeedback: PythonFeedback - - when(result) { - is PythonEvaluationError -> { - val utError = UtError( - "Error evaluation: ${result.status}, ${result.message}", - Throwable(result.stackTrace.joinToString("\n")) - ) - logger.debug(result.stackTrace.joinToString("\n")) - - limitManager.addSuccessExecution() - executionFeedback = InvalidExecution(utError) - fuzzingFeedback = PythonFeedback(control = Control.PASS) - return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), true) - } - - is PythonEvaluationTimeout -> { - val utError = UtError(result.message, Throwable()) - limitManager.addInvalidExecution() - executionFeedback = InvalidExecution(utError) - fuzzingFeedback = PythonFeedback(control = Control.PASS) - return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false) - } + try { + return when (val evaluationResult = manager.run(functionArguments, localAdditionalModules)) { + is PythonEvaluationError -> { + val utError = UtError( + "Error evaluation: ${evaluationResult.status}, ${evaluationResult.message}", + Throwable(evaluationResult.stackTrace.joinToString("\n")) + ) + logger.debug(evaluationResult.stackTrace.joinToString("\n")) + PythonExecutionResult(InvalidExecution(utError), PythonFeedback(control = Control.PASS)) + } - is PythonEvaluationSuccess -> { - val coveredInstructions = result.coverage.coveredInstructions - executionFeedback = handleSuccessResult( - arguments, - parameters, - result, - description, - ) + is PythonEvaluationTimeout -> { + val utError = UtError(evaluationResult.message, Throwable()) + PythonExecutionResult(InvalidExecution(utError), PythonFeedback(control = Control.PASS)) + } - val trieNode: Trie.Node = description.tracer.add(coveredInstructions) - when (executionFeedback) { - is ValidExecution -> { - limitManager.addSuccessExecution() - if (trieNode.count > 1) { - fuzzingFeedback = PythonFeedback(control = Control.CONTINUE, result = trieNode) - return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false) + is PythonEvaluationSuccess -> { + val coveredInstructions = evaluationResult.coverage.coveredInstructions + + when (val result = handleSuccessResult( + arguments, + parameters, + evaluationResult, + description, + )) { + is ValidExecution -> { + val trieNode: Trie.Node = description.tracer.add(coveredInstructions) + PythonExecutionResult( + result, + PythonFeedback(control = Control.CONTINUE, result = trieNode) + ) } - } - - is ArgumentsTypeErrorFeedback -> { - fuzzingFeedback = PythonFeedback(control = Control.PASS) - return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false) - } - is TypeErrorFeedback -> { - limitManager.addInvalidExecution() - fuzzingFeedback = PythonFeedback(control = Control.PASS) - return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false) - } + is ArgumentsTypeErrorFeedback, is TypeErrorFeedback -> { + PythonExecutionResult(result, PythonFeedback(control = Control.PASS)) + } - is InvalidExecution -> { - limitManager.addInvalidExecution() - fuzzingFeedback = PythonFeedback(control = Control.CONTINUE) - return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false) + is InvalidExecution -> { + PythonExecutionResult(result, PythonFeedback(control = Control.CONTINUE)) + } } } - fuzzingFeedback = PythonFeedback(control = Control.CONTINUE, result = trieNode) - return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), true) } + } catch (_: TimeoutException) { + logger.info { "Fuzzing process was interrupted by timeout" } + return null } } @@ -273,14 +241,10 @@ class PythonEngine( ) if (parameters.isEmpty()) { - val result = runWithFuzzedValues(emptyList()) + val result = fuzzingResultHandler(pmd, emptyList()) result?.let { - val (executionResult, needToEmit) = handleExecutionResult(result, emptyList(), pmd) - if (needToEmit) { - emit(executionResult.fuzzingExecutionFeedback) - } + emit(it.fuzzingExecutionFeedback) } - manager.disconnect() } else { try { PythonFuzzing(pmd.pythonTypeStorage) { description, arguments -> @@ -289,32 +253,35 @@ class PythonEngine( manager.disconnect() return@PythonFuzzing PythonFeedback(control = Control.STOP) } + if (System.currentTimeMillis() >= until) { + logger.info { "Fuzzing process was interrupted by timeout" } + manager.disconnect() + return@PythonFuzzing PythonFeedback(control = Control.STOP) + } val pair = Pair(description, arguments.map { PythonTreeWrapper(it.tree) }) val mem = cache.get(pair) if (mem != null) { logger.debug("Repeat in fuzzing") + emit(mem.fuzzingExecutionFeedback) return@PythonFuzzing mem.fuzzingPlatformFeedback } - - val result = runWithFuzzedValues(arguments) + val result = fuzzingResultHandler(description, arguments) if (result == null) { // timeout + logger.info { "Fuzzing process was interrupted by timeout" } manager.disconnect() return@PythonFuzzing PythonFeedback(control = Control.STOP) } - val (executionResult, needToEmit) = handleExecutionResult(result, arguments, description) - cache.add(pair, executionResult) - if (needToEmit) { - emit(executionResult.fuzzingExecutionFeedback) - } - return@PythonFuzzing executionResult.fuzzingPlatformFeedback + cache.add(pair, result) + emit(result.fuzzingExecutionFeedback) + return@PythonFuzzing result.fuzzingPlatformFeedback }.fuzz(pmd) - } catch (ex: Exception) { // NoSeedValueException + } catch (_: Exception) { // e.g. NoSeedValueException logger.info { "Cannot fuzz values for types: $parameters" } } - manager.disconnect() } + manager.disconnect() } } -} \ No newline at end of file +} diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt index 33a419ecc2..73c07d00ee 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt @@ -167,23 +167,27 @@ class PythonTestCaseGenerator( val fuzzerCancellation = { isCancelled() || limitManager.isCancelled() } - engine.fuzzing(args, fuzzerCancellation, limitManager).collect { + engine.fuzzing(args, fuzzerCancellation, until).collect { generated += 1 when (it) { is ValidExecution -> { executions += it.utFuzzedExecution missingLines = updateCoverage(it.utFuzzedExecution, coveredLines, missingLines) feedback = SuccessFeedback + limitManager.addSuccessExecution() } is InvalidExecution -> { errors += it.utError feedback = SuccessFeedback + limitManager.addSuccessExecution() } is ArgumentsTypeErrorFeedback -> { feedback = InvalidTypeFeedback + limitManager.addInvalidExecution() } is TypeErrorFeedback -> { feedback = InvalidTypeFeedback + limitManager.addInvalidExecution() } } limitManager.missedLines = missingLines?.size From 743061b8fad43c41b6e7056bf4cb37f6601cfa2b Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Thu, 9 Mar 2023 15:10:05 +0300 Subject: [PATCH 2/3] Fix limit manager bug --- .../kotlin/org/utbot/python/utils/TestGenerationLimitManager.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/utils/TestGenerationLimitManager.kt b/utbot-python/src/main/kotlin/org/utbot/python/utils/TestGenerationLimitManager.kt index fe2fd5a80d..bf0fcc6b55 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/utils/TestGenerationLimitManager.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/utils/TestGenerationLimitManager.kt @@ -57,7 +57,7 @@ object TimeoutMode : LimitManagerMode { object ExecutionMode : LimitManagerMode { override fun isCancelled(manager: TestGenerationLimitManager): Boolean { if (manager.invalidExecutions <= 0 || manager.executions <= 0) { - return min(manager.invalidExecutions, 0) + min(manager.executions, 0) <= manager.additionalExecutions + return min(manager.invalidExecutions, 0) + min(manager.executions, 0) + manager.additionalExecutions <= 0 } return false } From 1b7affe97e2d74cf4ae0f91bac9cbdf77fd9fb43 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Thu, 9 Mar 2023 15:12:12 +0300 Subject: [PATCH 3/3] Fix logging in python fuzzing and remove unused code --- .../org/utbot/python/fuzzing/PythonApi.kt | 40 +++++-------------- 1 file changed, 9 insertions(+), 31 deletions(-) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt index 6a7b27d176..9d75a2f453 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt @@ -4,8 +4,6 @@ import mu.KotlinLogging import org.utbot.framework.plugin.api.Instruction import org.utbot.framework.plugin.api.UtError import org.utbot.fuzzer.FuzzedContext -import org.utbot.fuzzer.IdGenerator -import org.utbot.fuzzer.IdentityPreservingIdGenerator import org.utbot.fuzzer.UtFuzzedExecution import org.utbot.fuzzing.Configuration import org.utbot.fuzzing.Control @@ -23,8 +21,6 @@ import org.utbot.python.newtyping.PythonSubtypeChecker import org.utbot.python.newtyping.PythonTypeStorage import org.utbot.python.newtyping.general.Type import org.utbot.python.newtyping.pythonTypeRepresentation -import java.util.* -import java.util.concurrent.atomic.AtomicLong private val logger = KotlinLogging.logger {} @@ -63,7 +59,7 @@ class PythonFuzzedValue( val summary: String? = null, ) -fun pythonDefaultValueProviders(idGenerator: IdGenerator) = listOf( +fun pythonDefaultValueProviders() = listOf( NoneValueProvider, BoolValueProvider, IntValueProvider, @@ -88,39 +84,38 @@ class PythonFuzzing( val execute: suspend (description: PythonMethodDescription, values: List) -> PythonFeedback, ) : Fuzzing { - private fun generateDefault(description: PythonMethodDescription, type: Type, idGenerator: IdGenerator): Sequence> { + private fun generateDefault(description: PythonMethodDescription, type: Type): Sequence> { var providers = emptyList>().asSequence() - pythonDefaultValueProviders(idGenerator).asSequence().forEach { provider -> + pythonDefaultValueProviders().asSequence().forEach { provider -> if (provider.accept(type)) { - logger.info { "Provider ${provider.javaClass.simpleName} accepts type ${type.pythonTypeRepresentation()}" } + logger.debug { "Provider ${provider.javaClass.simpleName} accepts type ${type.pythonTypeRepresentation()}" } providers += provider.generate(description, type) } } return providers } - private fun generateSubtype(description: PythonMethodDescription, type: Type, idGenerator: IdGenerator): Sequence> { + private fun generateSubtype(description: PythonMethodDescription, type: Type): Sequence> { var providers = emptyList>().asSequence() if (type.meta is PythonProtocolDescription) { val subtypes = pythonTypeStorage.allTypes.filter { PythonSubtypeChecker.checkIfRightIsSubtypeOfLeft(type, it, pythonTypeStorage) } subtypes.forEach { - providers += generateDefault(description, it, idGenerator) + providers += generateDefault(description, it) } } return providers } override fun generate(description: PythonMethodDescription, type: Type): Sequence> { - val idGenerator = PythonIdGenerator() var providers = emptyList>().asSequence() if (type.isAny()) { - logger.info("Any does not have provider") + logger.debug("Any does not have provider") } else { - providers += generateDefault(description, type, idGenerator) - providers += generateSubtype(description, type, idGenerator) + providers += generateDefault(description, type) + providers += generateSubtype(description, type) } return providers @@ -138,20 +133,3 @@ class PythonFuzzing( super.update(description, statistic, configuration) } } - -class PythonIdGenerator(lowerBound: Long = DEFAULT_LOWER_BOUND) : IdentityPreservingIdGenerator { - private val lastId: AtomicLong = AtomicLong(lowerBound) - private val cache: IdentityHashMap = IdentityHashMap() - - override fun getOrCreateIdForValue(value: Any): Long { - return cache.getOrPut(value) { createId() } - } - - override fun createId(): Long { - return lastId.incrementAndGet() - } - - companion object { - const val DEFAULT_LOWER_BOUND: Long = 1500_000_000 - } -}