Socket connection in python fuzzing

utbot-python/build.gradle.kts

@@ -21,7 +21,7 @@ tasks {
 }
 
 dependencies {
-    api(project(":utbot-fuzzers"))
+    api(project(":utbot-fuzzing"))
     api(project(":utbot-framework"))
     api(project(":utbot-python-parser"))
     testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.8.1")

utbot-python/samples/easy_samples/annotation_tests.py

@@ -10,7 +10,7 @@
 class A(Generic[XXX]):
     self_: XXX
 
-    def f(self, a, b: A[int]):
+    def f(self, a, b: 'A'[int]):
         self.y = b
         self.self_.x = b
         pass

utbot-python/samples/easy_samples/boruvka.py

@@ -58,7 +58,7 @@ def boruvka(self) -> None:
         component_size = []
         mst_weight = 0
 
-        minimum_weight_edge: list[Any] = [-1] * self.m_num_of_nodes
+        minimum_weight_edge: list = [-1] * self.m_num_of_nodes
 
         # A list of components (initialized to all of the nodes)
         for node in range(self.m_num_of_nodes):

utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt

@@ -3,47 +3,27 @@ package org.utbot.python
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.flow
 import mu.KotlinLogging
-import org.utbot.framework.plugin.api.DocRegularStmt
-import org.utbot.framework.plugin.api.EnvironmentModels
-import org.utbot.framework.plugin.api.Instruction
-import org.utbot.framework.plugin.api.UtError
-import org.utbot.framework.plugin.api.UtExecutionResult
-import org.utbot.framework.plugin.api.UtExecutionSuccess
-import org.utbot.framework.plugin.api.UtExplicitlyThrownException
-import org.utbot.framework.plugin.api.UtModel
-import org.utbot.fuzzer.FuzzedValue
+import org.utbot.framework.plugin.api.*
 import org.utbot.fuzzer.UtFuzzedExecution
 import org.utbot.fuzzing.Control
 import org.utbot.fuzzing.fuzz
 import org.utbot.fuzzing.utils.Trie
-import org.utbot.python.code.MemoryDump
-import org.utbot.python.code.toPythonTree
-import org.utbot.python.evaluation.PythonCodeExecutor
-import org.utbot.python.evaluation.PythonCodeExecutorImpl
-import org.utbot.python.evaluation.PythonEvaluationError
-import org.utbot.python.evaluation.PythonEvaluationSuccess
-import org.utbot.python.evaluation.PythonEvaluationTimeout
+import org.utbot.python.evaluation.*
+import org.utbot.python.evaluation.serialiation.MemoryDump
+import org.utbot.python.evaluation.serialiation.toPythonTree
 import org.utbot.python.framework.api.python.PythonTreeModel
-import org.utbot.python.fuzzing.PythonFeedback
-import org.utbot.python.fuzzing.PythonFuzzedConcreteValue
-import org.utbot.python.fuzzing.PythonFuzzedValue
-import org.utbot.python.fuzzing.PythonFuzzing
-import org.utbot.python.fuzzing.PythonMethodDescription
+import org.utbot.python.framework.api.python.PythonTreeWrapper
+import org.utbot.python.fuzzing.*
 import org.utbot.python.newtyping.PythonTypeStorage
 import org.utbot.python.newtyping.general.Type
 import org.utbot.python.newtyping.pythonModules
 import org.utbot.python.newtyping.pythonTypeRepresentation
 import org.utbot.python.utils.camelToSnakeCase
 import org.utbot.summary.fuzzer.names.TestSuggestedInfo
+import java.net.ServerSocket
 
 private val logger = KotlinLogging.logger {}
 
-sealed interface FuzzingExecutionFeedback
-class ValidExecution(val utFuzzedExecution: UtFuzzedExecution): FuzzingExecutionFeedback
-class InvalidExecution(val utError: UtError): FuzzingExecutionFeedback
-class TypeErrorFeedback(val message: String) : FuzzingExecutionFeedback
-class ArgumentsTypeErrorFeedback(val message: String) : FuzzingExecutionFeedback
-
 class PythonEngine(
     private val methodUnderTest: PythonMethod,
     private val directoriesForSysPath: Set<String>,
@@ -55,6 +35,8 @@ class PythonEngine(
     private val pythonTypeStorage: PythonTypeStorage,
 ) {
 
+    private val cache = EvaluationCache()
+
     private fun suggestExecutionName(
         description: PythonMethodDescription,
         executionResult: UtExecutionResult
@@ -146,109 +128,155 @@ class PythonEngine(
         return ValidExecution(utFuzzedExecution)
     }
 
-    private fun constructEvaluationInput(arguments: List<PythonFuzzedValue>, additionalModules: List<String>): PythonCodeExecutor {
-        val argumentValues = arguments.map { PythonTreeModel(it.tree, it.tree.type) }
-
-        val (thisObject, modelList) =
-            if (methodUnderTest.hasThisArgument)
-                Pair(argumentValues[0], argumentValues.drop(1))
-            else
-                Pair(null, argumentValues)
-
-        val argumentModules = argumentValues
-            .flatMap { it.allContainingClassIds }
-            .map { it.moduleName }
-            .filterNot { it.startsWith(moduleToImport) }
-        val localAdditionalModules = (additionalModules + argumentModules + moduleToImport).toSet()
-
-        return PythonCodeExecutorImpl(
+    private fun constructEvaluationInput(pythonWorker: PythonWorker): PythonCodeExecutor {
+        return PythonCodeSocketExecutor(
             methodUnderTest,
-            FunctionArguments(thisObject, methodUnderTest.thisObjectName, modelList, methodUnderTest.argumentsNames),
-            argumentValues.map { FuzzedValue(it) },
             moduleToImport,
-            localAdditionalModules,
             pythonPath,
             directoriesForSysPath,
             timeoutForRun,
+            pythonWorker,
         )
     }
 
     fun fuzzing(parameters: List<Type>, isCancelled: () -> Boolean, until: Long): Flow<FuzzingExecutionFeedback> = flow {
         val additionalModules = parameters.flatMap { it.pythonModules() }
         val coveredLines = initialCoveredLines.toMutableSet()
 
-        suspend fun fuzzingResultHandler(description: PythonMethodDescription, arguments: List<PythonFuzzedValue>): PythonFeedback {
-            val codeExecutor = constructEvaluationInput(arguments, additionalModules)
-            return when (val evaluationResult = codeExecutor.run()) {
-                is PythonEvaluationError -> {
-                    val utError = UtError(
-                        "Error evaluation: ${evaluationResult.status}, ${evaluationResult.message}",
-                        Throwable(evaluationResult.stackTrace.joinToString("\n"))
-                    )
-                    logger.debug(evaluationResult.stackTrace.joinToString("\n"))
-                    emit(InvalidExecution(utError))
-                    PythonFeedback(control = Control.PASS)
-                }
+        ServerSocket(0).use { serverSocket ->
+            logger.info { "Server port: ${serverSocket.localPort}" }
+            val manager = PythonWorkerManager(
+                serverSocket,
+                pythonPath,
+                until,
+                { constructEvaluationInput(it) },
+                timeoutForRun.toInt()
+            )
+            logger.info { "Executor manager was created successfully" }
 
-                is PythonEvaluationTimeout -> {
-                    val utError = UtError(evaluationResult.message, Throwable())
-                    emit(InvalidExecution(utError))
-                    PythonFeedback(control = Control.PASS)
-                }
+            fun fuzzingResultHandler(
+                description: PythonMethodDescription,
+                arguments: List<PythonFuzzedValue>
+            ): PythonExecutionResult? {
+                val argumentValues = arguments.map { PythonTreeModel(it.tree, it.tree.type) }
+                logger.debug(argumentValues.map { it.tree } .toString())
+                val argumentModules = argumentValues
+                    .flatMap { it.allContainingClassIds }
+                    .map { it.moduleName }
+                    .filterNot { it.startsWith(moduleToImport) }
+                val localAdditionalModules = (additionalModules + argumentModules + moduleToImport).toSet()
 
-                is PythonEvaluationSuccess -> {
-                    val coveredInstructions = evaluationResult.coverage.coveredInstructions
-                    coveredInstructions.forEach { coveredLines.add(it.lineNumber) }
+                val (thisObject, modelList) =
+                    if (methodUnderTest.hasThisArgument)
+                        Pair(argumentValues[0], argumentValues.drop(1))
+                    else
+                        Pair(null, argumentValues)
+                val functionArguments = FunctionArguments(
+                    thisObject,
+                    methodUnderTest.thisObjectName,
+                    modelList,
+                    methodUnderTest.argumentsNames
+                )
+                try {
+                    return when (val evaluationResult = manager.run(functionArguments, localAdditionalModules)) {
+                        is PythonEvaluationError -> {
+                            val utError = UtError(
+                                "Error evaluation: ${evaluationResult.status}, ${evaluationResult.message}",
+                                Throwable(evaluationResult.stackTrace.joinToString("\n"))
+                            )
+                            logger.debug(evaluationResult.stackTrace.joinToString("\n"))
+                            PythonExecutionResult(InvalidExecution(utError), PythonFeedback(control = Control.PASS))
+                        }
 
-                    val summary = arguments
-                        .zip(methodUnderTest.arguments)
-                        .mapNotNull { it.first.summary?.replace("%var%", it.second.name) }
+                        is PythonEvaluationTimeout -> {
+                            val utError = UtError(evaluationResult.message, Throwable())
+                            PythonExecutionResult(InvalidExecution(utError), PythonFeedback(control = Control.PASS))
+                        }
 
-                    val hasThisObject = codeExecutor.methodArguments.thisObject != null
+                        is PythonEvaluationSuccess -> {
+                            val coveredInstructions = evaluationResult.coverage.coveredInstructions
+                            coveredInstructions.forEach { coveredLines.add(it.lineNumber) }
 
-                    when (val result = handleSuccessResult(parameters, evaluationResult, description, hasThisObject, summary)) {
-                        is ValidExecution -> {
-                            logger.debug { arguments }
-                            val trieNode: Trie.Node<Instruction> = description.tracer.add(coveredInstructions)
-                            emit(result)
-                            PythonFeedback(control = Control.CONTINUE, result = trieNode)
-                        }
-                        is ArgumentsTypeErrorFeedback, is TypeErrorFeedback -> {
-                            emit(result)
-                            PythonFeedback(control = Control.PASS)
-                        }
-                        is InvalidExecution -> {
-                            emit(result)
-                            PythonFeedback(control = Control.CONTINUE)
+                            val summary = arguments
+                                .zip(methodUnderTest.arguments)
+                                .mapNotNull { it.first.summary?.replace("%var%", it.second.name) }
+
+                            val hasThisObject = methodUnderTest.hasThisArgument
+
+                            when (val result = handleSuccessResult(
+                                parameters,
+                                evaluationResult,
+                                description,
+                                hasThisObject,
+                                summary
+                            )) {
+                                is ValidExecution -> {
+                                    val trieNode: Trie.Node<Instruction> = description.tracer.add(coveredInstructions)
+                                    PythonExecutionResult(
+                                        result,
+                                        PythonFeedback(control = Control.CONTINUE, result = trieNode)
+                                    )
+                                }
+
+                                is ArgumentsTypeErrorFeedback, is TypeErrorFeedback -> {
+                                    PythonExecutionResult(result, PythonFeedback(control = Control.PASS))
+                                }
+
+                                is InvalidExecution -> {
+                                    PythonExecutionResult(result, PythonFeedback(control = Control.CONTINUE))
+                                }
+                            }
                         }
                     }
+                } catch (_: TimeoutException) {
+                    return null
                 }
             }
-        }
 
-        val pmd = PythonMethodDescription(
-            methodUnderTest.name,
-            parameters,
-            fuzzedConcreteValues,
-            pythonTypeStorage,
-            Trie(Instruction::id)
-        )
+            val pmd = PythonMethodDescription(
+                methodUnderTest.name,
+                parameters,
+                fuzzedConcreteValues,
+                pythonTypeStorage,
+                Trie(Instruction::id)
+            )
 
-        if (parameters.isEmpty()) {
-            fuzzingResultHandler(pmd, emptyList())
-        } else {
-            PythonFuzzing(pmd.pythonTypeStorage) { description, arguments ->
-                if (isCancelled()) {
-                    logger.info { "Fuzzing process was interrupted" }
-                    return@PythonFuzzing PythonFeedback(control = Control.STOP)
-                }
-                if (System.currentTimeMillis() >= until) {
-                    logger.info { "Fuzzing process was interrupted by timeout" }
-                    return@PythonFuzzing PythonFeedback(control = Control.STOP)
-                }
+            if (parameters.isEmpty()) {
+                fuzzingResultHandler(pmd, emptyList())
+                manager.disconnect()
+            } else {
+                PythonFuzzing(pmd.pythonTypeStorage) { description, arguments ->
+                    if (isCancelled()) {
+                        logger.info { "Fuzzing process was interrupted" }
+                        manager.disconnect()
+                        return@PythonFuzzing PythonFeedback(control = Control.STOP)
+                    }
+                    if (System.currentTimeMillis() >= until) {
+                        logger.info { "Fuzzing process was interrupted by timeout" }
+                        manager.disconnect()
+                        return@PythonFuzzing PythonFeedback(control = Control.STOP)
+                    }
 
-                return@PythonFuzzing fuzzingResultHandler(description, arguments)
-            }.fuzz(pmd)
+                    val pair = Pair(description, arguments.map { PythonTreeWrapper(it.tree) })
+                    val mem = cache.get(pair)
+                    if (mem != null) {
+                        logger.debug("Repeat in fuzzing")
+                        emit(mem.fuzzingExecutionFeedback)
+                        return@PythonFuzzing mem.fuzzingPlatformFeedback
+                    }
+                    val result = fuzzingResultHandler(description, arguments)
+                    if (result == null) {  // timeout
+                        logger.info { "Fuzzing process was interrupted by timeout" }
+                        manager.disconnect()
+                        return@PythonFuzzing PythonFeedback(control = Control.STOP)
+                    }
+
+                    cache.add(pair, result)
+                    emit(result.fuzzingExecutionFeedback)
+                    return@PythonFuzzing result.fuzzingPlatformFeedback
+
+                }.fuzz(pmd)
+            }
         }
     }
 }