From 991aa1b6f026a86e88efc9b05c8bf890984653f7 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Wed, 15 Feb 2023 15:35:20 +0300 Subject: [PATCH 01/32] Work on sockets --- .../kotlin/org/utbot/python/PythonEngine.kt | 4 +- .../python/code/PythonObjectDeserializer.kt | 135 ------ .../python/evaluation/CodeEvaluationApi.kt | 9 +- .../python/evaluation/ExecutionClient.kt | 52 +++ .../evaluation/PythonCodeExecutorImpl.kt | 17 +- .../evaluation/PythonCodeSocketExecutor.kt | 30 ++ .../ExecutionResultDeserializer.kt | 8 +- .../serialiation/PythonObjectParser.kt | 439 ++++++++++++++++++ .../python/framework/api/python/PythonTree.kt | 57 ++- .../api/python/util/PythonIdUtils.kt | 10 +- .../org/utbot/python/fuzzing/PythonApi.kt | 21 +- .../fuzzing/provider/ReduceValueProvider.kt | 6 +- 12 files changed, 609 insertions(+), 179 deletions(-) delete mode 100644 utbot-python/src/main/kotlin/org/utbot/python/code/PythonObjectDeserializer.kt create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/evaluation/ExecutionClient.kt create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt rename utbot-python/src/main/kotlin/org/utbot/python/evaluation/{ => serialiation}/ExecutionResultDeserializer.kt (85%) create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/PythonObjectParser.kt diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index e897c2ba51..71d387f89e 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -16,13 +16,13 @@ import org.utbot.fuzzer.UtFuzzedExecution import org.utbot.fuzzing.Control import org.utbot.fuzzing.fuzz import org.utbot.fuzzing.utils.Trie -import org.utbot.python.code.MemoryDump -import org.utbot.python.code.toPythonTree import org.utbot.python.evaluation.PythonCodeExecutor import org.utbot.python.evaluation.PythonCodeExecutorImpl import org.utbot.python.evaluation.PythonEvaluationError import org.utbot.python.evaluation.PythonEvaluationSuccess import org.utbot.python.evaluation.PythonEvaluationTimeout +import org.utbot.python.evaluation.serialiation.MemoryDump +import org.utbot.python.evaluation.serialiation.toPythonTree import org.utbot.python.framework.api.python.PythonTreeModel import org.utbot.python.fuzzing.PythonFeedback import org.utbot.python.fuzzing.PythonFuzzedConcreteValue diff --git a/utbot-python/src/main/kotlin/org/utbot/python/code/PythonObjectDeserializer.kt b/utbot-python/src/main/kotlin/org/utbot/python/code/PythonObjectDeserializer.kt deleted file mode 100644 index d840892a9c..0000000000 --- a/utbot-python/src/main/kotlin/org/utbot/python/code/PythonObjectDeserializer.kt +++ /dev/null @@ -1,135 +0,0 @@ -package org.utbot.python.code - -import com.squareup.moshi.Moshi -import com.squareup.moshi.adapters.PolymorphicJsonAdapterFactory -import com.squareup.moshi.kotlin.reflect.KotlinJsonAdapterFactory -import org.utbot.python.framework.api.python.PythonClassId -import org.utbot.python.framework.api.python.PythonTree - -object PythonObjectDeserializer { - private val moshi = Moshi.Builder() - .add( - PolymorphicJsonAdapterFactory.of(MemoryObject::class.java, "strategy") - .withSubtype(ReprMemoryObject::class.java, "repr") - .withSubtype(ListMemoryObject::class.java, "list") - .withSubtype(DictMemoryObject::class.java, "dict") - .withSubtype(ReduceMemoryObject::class.java, "reduce") - ) - .addLast(KotlinJsonAdapterFactory()) - .build() - - private val jsonAdapter = moshi.adapter(MemoryDump::class.java) - - fun parseDumpedObjects(jsonWithDump: String): MemoryDump { - return jsonAdapter.fromJson(jsonWithDump) ?: error("Couldn't parse json dump") - } -} - -class MemoryDump( - private val objects: Map -) { - fun getById(id: String): MemoryObject { - return objects[id]!! - } - - fun getByValue(value: MemoryObject): String { - return objects.filter { it.value == value }.keys.first() - } -} - -sealed class MemoryObject( - val kind: String, - val comparable: Boolean, -) - -class ReprMemoryObject( - kind: String, - comparable: Boolean, - val value: String, -): MemoryObject(kind, comparable) - -class ListMemoryObject( - kind: String, - comparable: Boolean, - val items: List, -): MemoryObject(kind, comparable) - -class DictMemoryObject( - kind: String, - comparable: Boolean, - val items: Map, -): MemoryObject(kind, comparable) - -class ReduceMemoryObject( - kind: String, - comparable: Boolean, - val constructor: String, - val args: String, - val state: String, - val listitems: String, - val dictitems: String -): MemoryObject(kind, comparable) - -fun MemoryObject.toPythonTree(memoryDump: MemoryDump): PythonTree.PythonTreeNode { - val obj = when(this) { - is ReprMemoryObject -> { - PythonTree.PrimitiveNode(PythonClassId(this.kind), this.value) - } - is DictMemoryObject -> { - PythonTree.DictNode( - items.entries.associate { - memoryDump.getById(it.key).toPythonTree(memoryDump) to memoryDump.getById(it.value).toPythonTree(memoryDump) - }.toMutableMap() - ) - } - is ListMemoryObject -> { - val elementsMap = items.withIndex().associate { - it.index to memoryDump.getById(it.value).toPythonTree(memoryDump) - }.toMutableMap() - when (this.kind) { - "builtins.tuple" -> { - PythonTree.TupleNode(elementsMap) - } - "builtins.set" -> { - PythonTree.SetNode(elementsMap.values.toMutableSet()) - } - else -> { - PythonTree.ListNode(elementsMap) - } - } - } - is ReduceMemoryObject -> { - val stateObjs = memoryDump.getById(state) as DictMemoryObject - val arguments = memoryDump.getById(args) as ListMemoryObject - val listitemsObjs = memoryDump.getById(listitems) as ListMemoryObject - val dictitemsObjs = memoryDump.getById(dictitems) as DictMemoryObject - PythonTree.ReduceNode( - memoryDump.getByValue(this).toLong(), - PythonClassId(this.kind), - PythonClassId(this.constructor), - arguments.items.map { memoryDump.getById(it).toPythonTree(memoryDump) }, - stateObjs.items.entries.associate { - (memoryDump.getById(it.key).toPythonTree(memoryDump) as PythonTree.PrimitiveNode).repr.drop(1).dropLast(1) to - memoryDump.getById(it.value).toPythonTree(memoryDump) - }.toMutableMap(), - listitemsObjs.items.map { memoryDump.getById(it).toPythonTree(memoryDump) }, - dictitemsObjs.items.entries.associate { - memoryDump.getById(it.key).toPythonTree(memoryDump) to - memoryDump.getById(it.value).toPythonTree(memoryDump) - }, - ) - } - } - obj.comparable = this.comparable - return obj -} - -fun main() { - val dump = """ - {"objects": {"140598295296000": {"strategy": "list", "kind": "builtins.list", "comparable": false, "items": ["140598416769264", "140598416769296", "140598416769328", "140598295298816", "140598427938368", "140598374175968"]}, "140598416769264": {"strategy": "repr", "kind": "builtins.int", "comparable": true, "value": "1"}, "140598416769296": {"strategy": "repr", "kind": "builtins.int", "comparable": true, "value": "2"}, "140598416769328": {"strategy": "repr", "kind": "builtins.int", "comparable": true, "value": "3"}, "140598295298816": {"strategy": "dict", "kind": "builtins.dict", "comparable": true, "items": {"140598416769264": "140598416769264"}}, "140598427938368": {"strategy": "repr", "kind": "types.NoneType", "comparable": true, "value": "None"}, "140598372733504": {"strategy": "list", "kind": "builtins.tuple", "comparable": true, "items": ["94206620040656", "140598427931232", "140598427938368"]}, "94206620040656": {"strategy": "repr", "kind": "builtins.type", "comparable": true, "value": "deep_serialization.example.B"}, "140598427931232": {"strategy": "repr", "kind": "builtins.type", "comparable": true, "value": "builtins.object"}, "140598374175968": {"strategy": "reduce", "kind": "deep_serialization.example.B", "comparable": false, "constructor": "copyreg._reconstructor", "args": "140598372733504", "state": "140598295238656", "listitems": "140598295487936", "dictitems": "140598295486336"}, "140598295238656": {"strategy": "dict", "kind": "builtins.dict", "comparable": true, "items": {"140598386103280": "140598416769264", "140598395465264": "140598416769296", "140598372712880": "140598416769328", "140598415768816": "140598374177584"}}, "140598386103280": {"strategy": "repr", "kind": "builtins.str", "comparable": true, "value": "\'b1\'"}, "140598395465264": {"strategy": "repr", "kind": "builtins.str", "comparable": true, "value": "\'b2\'"}, "140598372712880": {"strategy": "repr", "kind": "builtins.str", "comparable": true, "value": "\'b3\'"}, "140598415768816": {"strategy": "repr", "kind": "builtins.str", "comparable": true, "value": "\'time\'"}, "140598374184560": {"strategy": "list", "kind": "builtins.tuple", "comparable": true, "items": ["140598295162016"]}, "140598295162016": {"strategy": "repr", "kind": "builtins.bytes", "comparable": true, "value": "b\'\\\\x07\\\\xe7\\\\x01\\\\x13\\\\x11\\\\x01\\\\x1c\\\\x0e\\\\x921\'"}, "140598374177584": {"strategy": "reduce", "kind": "datetime.datetime", "comparable": true, "constructor": "datetime.datetime", "args": "140598374184560", "state": "140598295312768", "listitems": "140598295488000", "dictitems": "140598295485760"}, "140598295312768": {"strategy": "dict", "kind": "builtins.dict", "comparable": true, "items": {}}, "140598295488000": {"strategy": "list", "kind": "builtins.list", "comparable": true, "items": []}, "140598295485760": {"strategy": "dict", "kind": "builtins.dict", "comparable": true, "items": {}}, "140598295487936": {"strategy": "list", "kind": "builtins.list", "comparable": true, "items": []}, "140598295486336": {"strategy": "dict", "kind": "builtins.dict", "comparable": true, "items": {}}}} -""".trimIndent() - - val load = PythonObjectDeserializer.parseDumpedObjects(dump) - load.getById("140598295296000").toPythonTree(load) -} - diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt index ecb90c4096..afb1db6f4d 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt @@ -4,19 +4,20 @@ import org.utbot.framework.plugin.api.Coverage import org.utbot.fuzzer.FuzzedValue import org.utbot.python.FunctionArguments import org.utbot.python.PythonMethod -import org.utbot.python.code.MemoryDump +import org.utbot.python.evaluation.serialiation.MemoryDump interface PythonCodeExecutor { val method: PythonMethod val methodArguments: FunctionArguments - val fuzzedValues: List val moduleToImport: String - val additionalModulesToImport: Set val pythonPath: String val syspathDirectories: Set val executionTimeout: Long - fun run(): PythonEvaluationResult + fun run( + fuzzedValues: List, + additionalModulesToImport: Set + ): PythonEvaluationResult } sealed class PythonEvaluationResult diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/ExecutionClient.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/ExecutionClient.kt new file mode 100644 index 0000000000..4528906ca0 --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/ExecutionClient.kt @@ -0,0 +1,52 @@ +package org.utbot.python.evaluation + +import java.io.BufferedReader +import java.io.BufferedWriter +import java.io.InputStreamReader +import java.io.OutputStreamWriter +import java.io.PrintWriter +import java.net.Socket +import java.nio.CharBuffer + +class ExecutionClient( + hostname: String, + port: Int +) { + private val clientSocket: Socket = Socket(hostname, port) + private val outStream = BufferedWriter(OutputStreamWriter(clientSocket.getOutputStream())) + private val inStream = BufferedReader(InputStreamReader(clientSocket.getInputStream())) + + init { + runServer() + } + + private fun runServer() { + TODO() + } + + fun sendMessage(msg: String) { + outStream.write(msg) + outStream.flush() + outStream.write("END") + outStream.flush() + } + + fun receiveMessage(): String? { + return inStream.readLine() + } + + fun stopConnection() { + inStream.close() + outStream.close() + clientSocket.close() + } +} + +fun main() { + val client = ExecutionClient("localhost", 12011) + client.sendMessage("Firstfjlskdjf") + client.sendMessage("Second") + client.sendMessage("{123: 123}") + client.sendMessage("STOP") + client.stopConnection() +} \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeExecutorImpl.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeExecutorImpl.kt index 8fff56e926..b9de31646c 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeExecutorImpl.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeExecutorImpl.kt @@ -6,6 +6,9 @@ import org.utbot.fuzzer.FuzzedValue import org.utbot.python.FunctionArguments import org.utbot.python.PythonMethod import org.utbot.python.code.PythonCodeGenerator +import org.utbot.python.evaluation.serialiation.ExecutionResultDeserializer +import org.utbot.python.evaluation.serialiation.FailExecution +import org.utbot.python.evaluation.serialiation.SuccessExecution import org.utbot.python.framework.api.python.util.pythonAnyClassId import org.utbot.python.utils.TemporaryFileManager import org.utbot.python.utils.getResult @@ -20,19 +23,23 @@ data class EvaluationFiles( class PythonCodeExecutorImpl( override val method: PythonMethod, override val methodArguments: FunctionArguments, - override val fuzzedValues: List, override val moduleToImport: String, - override val additionalModulesToImport: Set, override val pythonPath: String, override val syspathDirectories: Set, override val executionTimeout: Long, ) : PythonCodeExecutor { - override fun run(): PythonEvaluationResult { - val evaluationFiles = generateExecutionCode() + + override fun run( + fuzzedValues: List, + additionalModulesToImport: Set, + ): PythonEvaluationResult { + val evaluationFiles = generateExecutionCode(additionalModulesToImport) return getEvaluationResult(evaluationFiles) } - private fun generateExecutionCode(): EvaluationFiles { + private fun generateExecutionCode( + additionalModulesToImport: Set, + ): EvaluationFiles { val fileForOutput = TemporaryFileManager.assignTemporaryFile( tag = "out_" + method.name + ".py", addToCleaner = false diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt new file mode 100644 index 0000000000..5f7137e074 --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt @@ -0,0 +1,30 @@ +package org.utbot.python.evaluation + +import org.utbot.fuzzer.FuzzedValue +import org.utbot.python.FunctionArguments +import org.utbot.python.PythonMethod + + +data class ExecutionDescription( + val functionName: String, + val imports: List, + val argumentsIds: List, + val serializedMemory: String +) + + +class PythonCodeSocketExecutor( + override val method: PythonMethod, + override val methodArguments: FunctionArguments, + override val moduleToImport: String, + override val pythonPath: String, + override val syspathDirectories: Set, + override val executionTimeout: Long, +) : PythonCodeExecutor { + val executionClient = ExecutionClient("localhost", 12011) // TODO: create port manager + override fun run( + fuzzedValues: List, + additionalModulesToImport: Set): PythonEvaluationResult { +// val input = + } +} \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/ExecutionResultDeserializer.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionResultDeserializer.kt similarity index 85% rename from utbot-python/src/main/kotlin/org/utbot/python/evaluation/ExecutionResultDeserializer.kt rename to utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionResultDeserializer.kt index f230f7d314..c6ec72d8c7 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/ExecutionResultDeserializer.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionResultDeserializer.kt @@ -1,14 +1,8 @@ -package org.utbot.python.evaluation +package org.utbot.python.evaluation.serialiation import com.squareup.moshi.Moshi import com.squareup.moshi.adapters.PolymorphicJsonAdapterFactory import com.squareup.moshi.kotlin.reflect.KotlinJsonAdapterFactory -import org.utbot.python.code.DictMemoryObject -import org.utbot.python.code.ListMemoryObject -import org.utbot.python.code.MemoryDump -import org.utbot.python.code.MemoryObject -import org.utbot.python.code.ReduceMemoryObject -import org.utbot.python.code.ReprMemoryObject object ExecutionResultDeserializer { private val moshi = Moshi.Builder() diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/PythonObjectParser.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/PythonObjectParser.kt new file mode 100644 index 0000000000..1bc7b376ac --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/PythonObjectParser.kt @@ -0,0 +1,439 @@ +package org.utbot.python.evaluation.serialiation + +import com.squareup.moshi.Moshi +import com.squareup.moshi.adapters.PolymorphicJsonAdapterFactory +import com.squareup.moshi.kotlin.reflect.KotlinJsonAdapterFactory +import org.utbot.python.framework.api.python.PythonClassId +import org.utbot.python.framework.api.python.PythonTree +import org.utbot.python.framework.api.python.util.pythonStrClassId + +object PythonObjectParser { + private val moshi = Moshi.Builder() + .add( + PolymorphicJsonAdapterFactory.of(MemoryObject::class.java, "strategy") + .withSubtype(ReprMemoryObject::class.java, "repr") + .withSubtype(ListMemoryObject::class.java, "list") + .withSubtype(DictMemoryObject::class.java, "dict") + .withSubtype(ReduceMemoryObject::class.java, "reduce") + ) + .addLast(KotlinJsonAdapterFactory()) + .build() + + private val jsonAdapter = moshi.adapter(MemoryDump::class.java) + + fun parseDumpedObjects(jsonWithDump: String): MemoryDump { + return jsonAdapter.fromJson(jsonWithDump) ?: error("Couldn't parse json dump") + } + + fun serializeMemory(memory: MemoryDump): String { + return jsonAdapter.toJson(memory) ?: error("Couldn't serialize dump to json") + } +} + +class MemoryDump( + private val objects: MutableMap +) { + fun getById(id: String): MemoryObject { + return objects[id]!! + } + + fun getByValue(value: MemoryObject): String { + return objects.filter { it.value == value }.keys.first() + } + + fun addObject(value: MemoryObject) { + objects[value.id] = value + } +} + +sealed class MemoryObject( + val id: String, + val kind: String, + val comparable: Boolean, +) + +class ReprMemoryObject( + id: String, + kind: String, + comparable: Boolean, + val value: String, +): MemoryObject(id, kind, comparable) + +class ListMemoryObject( + id: String, + kind: String, + comparable: Boolean, + val items: List, +): MemoryObject(id, kind, comparable) + +class DictMemoryObject( + id: String, + kind: String, + comparable: Boolean, + val items: Map, +): MemoryObject(id, kind, comparable) + +class ReduceMemoryObject( + id: String, + kind: String, + comparable: Boolean, + val constructor: String, + val args: String, + val state: String, + val listitems: String, + val dictitems: String +): MemoryObject(id, kind, comparable) + +fun PythonTree.PythonTreeNode.toMemoryObject(memoryDump: MemoryDump): String { + val obj = when(this) { + is PythonTree.PrimitiveNode -> { + ReprMemoryObject(this.id.toString(), this.type.name, this.comparable, this.repr) + } + is PythonTree.ListNode -> { + val items = this.items.entries + .sortedBy { it.key } + .map { it.value.toMemoryObject(memoryDump) } + ListMemoryObject(this.id.toString(), this.type.name, this.comparable, items) + } + is PythonTree.TupleNode -> { + val items = this.items.entries + .sortedBy { it.key } + .map { it.value.toMemoryObject(memoryDump) } + ListMemoryObject(this.id.toString(), this.type.name, this.comparable, items) + } + is PythonTree.SetNode -> { + val items = this.items.map { it.toMemoryObject(memoryDump) } + ListMemoryObject(this.id.toString(), this.type.name, this.comparable, items) + } + is PythonTree.DictNode -> { + val items = this.items.entries + .associate { it.key.toMemoryObject(memoryDump) to it.value.toMemoryObject(memoryDump) } + DictMemoryObject(this.id.toString(), this.type.name, this.comparable, items) + } + is PythonTree.ReduceNode -> { + val stateObjId = PythonTree.DictNode(this.state.entries.associate { PythonTree.PrimitiveNode(pythonStrClassId, it.key) to it.value }.toMutableMap()) + val argsIds = PythonTree.ListNode(this.args.withIndex().associate { it.index to it.value }.toMutableMap()) + val listItemsIds = PythonTree.ListNode(this.listitems.withIndex().associate { it.index to it.value }.toMutableMap()) + val dictItemsIds = PythonTree.DictNode(this.dictitems.toMutableMap()) + ReduceMemoryObject( + this.id.toString(), + this.type.name, + this.comparable, + this.constructor.name, + argsIds.toMemoryObject(memoryDump), + stateObjId.toMemoryObject(memoryDump), + listItemsIds.toMemoryObject(memoryDump), + dictItemsIds.toMemoryObject(memoryDump), + ) + } + else -> { + error("Invalid PythonTree.PythonTreeNode $this") + } + } + memoryDump.addObject(obj) + return obj.id +} + +fun MemoryObject.toPythonTree(memoryDump: MemoryDump): PythonTree.PythonTreeNode { + val obj = when(this) { + is ReprMemoryObject -> { + PythonTree.PrimitiveNode( + this.id.toLong(), + PythonClassId(this.kind), + this.value + ) + } + is DictMemoryObject -> { + PythonTree.DictNode( + this.id.toLong(), + items.entries.associate { + memoryDump.getById(it.key).toPythonTree(memoryDump) to memoryDump.getById(it.value).toPythonTree(memoryDump) + }.toMutableMap() + ) + } + is ListMemoryObject -> { + val elementsMap = items.withIndex().associate { + it.index to memoryDump.getById(it.value).toPythonTree(memoryDump) + }.toMutableMap() + when (this.kind) { + "builtins.tuple" -> { + PythonTree.TupleNode(this.id.toLong(), elementsMap) + } + "builtins.set" -> { + PythonTree.SetNode(this.id.toLong(), elementsMap.values.toMutableSet()) + } + else -> { + PythonTree.ListNode(this.id.toLong(), elementsMap) + } + } + } + is ReduceMemoryObject -> { + val stateObjs = memoryDump.getById(state) as DictMemoryObject + val arguments = memoryDump.getById(args) as ListMemoryObject + val listitemsObjs = memoryDump.getById(listitems) as ListMemoryObject + val dictitemsObjs = memoryDump.getById(dictitems) as DictMemoryObject + PythonTree.ReduceNode( + this.id.toLong(), + PythonClassId(this.kind), + PythonClassId(this.constructor), + arguments.items.map { memoryDump.getById(it).toPythonTree(memoryDump) }, + stateObjs.items.entries.associate { + (memoryDump.getById(it.key).toPythonTree(memoryDump) as PythonTree.PrimitiveNode).repr.drop(1).dropLast(1) to + memoryDump.getById(it.value).toPythonTree(memoryDump) + }.toMutableMap(), + listitemsObjs.items.map { memoryDump.getById(it).toPythonTree(memoryDump) }, + dictitemsObjs.items.entries.associate { + memoryDump.getById(it.key).toPythonTree(memoryDump) to + memoryDump.getById(it.value).toPythonTree(memoryDump) + }, + ) + } + } + obj.comparable = this.comparable + return obj +} + +fun serializeObjects(objs: List): Pair, String> { + val memoryDump = MemoryDump(emptyMap().toMutableMap()) + val ids = objs.map { it.toMemoryObject(memoryDump) } + return Pair(ids, PythonObjectParser.serializeMemory(memoryDump)) +} + +fun main() { + val dump = """ + { + "objects": { + "140239390887040": { + "strategy": "list", + "id": "140239390887040", + "kind": "builtins.list", + "comparable": false, + "items": [ + "140239394832624", + "140239394832656", + "140239392627184", + "140239394012784", + "140239392795520", + "140239406001728", + "140239392839840", + "140239390894848" + ] + }, + "140239394832624": { + "strategy": "repr", + "id": "140239394832624", + "kind": "builtins.int", + "comparable": true, + "value": "1" + }, + "140239394832656": { + "strategy": "repr", + "id": "140239394832656", + "kind": "builtins.int", + "comparable": true, + "value": "2" + }, + "140239392627184": { + "strategy": "repr", + "id": "140239392627184", + "kind": "builtins.float", + "comparable": true, + "value": "float('inf')" + }, + "140239394012784": { + "strategy": "repr", + "id": "140239394012784", + "kind": "builtins.str", + "comparable": true, + "value": "'abc'" + }, + "140239392795520": { + "strategy": "dict", + "id": "140239392795520", + "kind": "builtins.dict", + "comparable": true, + "items": { + "140239394832624": "140239394832624" + } + }, + "140239406001728": { + "strategy": "repr", + "id": "140239406001728", + "kind": "types.NoneType", + "comparable": true, + "value": "None" + }, + "140239391427840": { + "strategy": "list", + "id": "140239391427840", + "kind": "builtins.tuple", + "comparable": true, + "items": [ + "94246249326576", + "140239405994592", + "140239406001728" + ] + }, + "94246249326576": { + "strategy": "repr", + "id": "94246249326576", + "kind": "builtins.type", + "comparable": true, + "value": "deep_serialization.example.B" + }, + "140239405994592": { + "strategy": "repr", + "id": "140239405994592", + "kind": "builtins.type", + "comparable": true, + "value": "builtins.object" + }, + "140239392839840": { + "strategy": "reduce", + "id": "140239392839840", + "kind": "deep_serialization.example.B", + "comparable": false, + "constructor": "copyreg._reconstructor", + "args": "140239391427840", + "state": "140239392795712", + "listitems": "140239391672832", + "dictitems": "140239391673280" + }, + "140239392795712": { + "strategy": "dict", + "id": "140239392795712", + "kind": "builtins.dict", + "comparable": true, + "items": { + "140239392797168": "140239394832624", + "140239392797232": "140239394832656", + "140239392797296": "140239394832688", + "140239393831920": "140239392849760" + } + }, + "140239392797168": { + "strategy": "repr", + "id": "140239392797168", + "kind": "builtins.str", + "comparable": true, + "value": "'b1'" + }, + "140239392797232": { + "strategy": "repr", + "id": "140239392797232", + "kind": "builtins.str", + "comparable": true, + "value": "'b2'" + }, + "140239392797296": { + "strategy": "repr", + "id": "140239392797296", + "kind": "builtins.str", + "comparable": true, + "value": "'b3'" + }, + "140239394832688": { + "strategy": "repr", + "id": "140239394832688", + "kind": "builtins.int", + "comparable": true, + "value": "3" + }, + "140239393831920": { + "strategy": "repr", + "id": "140239393831920", + "kind": "builtins.str", + "comparable": true, + "value": "'time'" + }, + "140239394159488": { + "strategy": "list", + "id": "140239394159488", + "kind": "builtins.tuple", + "comparable": true, + "items": [ + "140239391514208" + ] + }, + "140239391514208": { + "strategy": "repr", + "id": "140239391514208", + "kind": "builtins.bytes", + "comparable": true, + "value": "b'\\x07\\xe7\\x02\\r\\x0c(3\\x06\\x1eA'" + }, + "140239392849760": { + "strategy": "reduce", + "id": "140239392849760", + "kind": "datetime.datetime", + "comparable": true, + "constructor": "datetime.datetime", + "args": "140239394159488", + "state": "140239391671232", + "listitems": "140239391671872", + "dictitems": "140239391672448" + }, + "140239391671232": { + "strategy": "dict", + "id": "140239391671232", + "kind": "builtins.dict", + "comparable": true, + "items": {} + }, + "140239391671872": { + "strategy": "list", + "id": "140239391671872", + "kind": "builtins.list", + "comparable": true, + "items": [] + }, + "140239391672448": { + "strategy": "dict", + "id": "140239391672448", + "kind": "builtins.dict", + "comparable": true, + "items": {} + }, + "140239391672832": { + "strategy": "list", + "id": "140239391672832", + "kind": "builtins.list", + "comparable": true, + "items": [] + }, + "140239391673280": { + "strategy": "dict", + "id": "140239391673280", + "kind": "builtins.dict", + "comparable": true, + "items": {} + }, + "140239390894848": { + "strategy": "list", + "id": "140239390894848", + "kind": "builtins.list", + "comparable": true, + "items": [ + "140239392797552" + ] + }, + "140239392797552": { + "strategy": "repr", + "id": "140239392797552", + "kind": "builtins.str", + "comparable": true, + "value": "'Alex'" + } + } + } +""".trimIndent() + + val load = PythonObjectParser.parseDumpedObjects(dump) + val obj = load.getById("140239390887040").toPythonTree(load) + val newMemory = MemoryDump(emptyMap().toMutableMap()) + val newId = obj.toMemoryObject(newMemory) + val newDump = PythonObjectParser.serializeMemory(newMemory) + val newLoad = PythonObjectParser.parseDumpedObjects(newDump) + val newObj = newLoad.getById(newId).toPythonTree(newLoad) +} + diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt index 3f793ae8f3..8f096e33b3 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt @@ -9,12 +9,18 @@ import org.utbot.python.newtyping.general.Type import org.utbot.python.newtyping.pythonTypeName import java.math.BigDecimal import java.math.BigInteger +import java.util.* +import java.util.concurrent.atomic.AtomicLong + object PythonTree { open class PythonTreeNode( + val id: Long, val type: PythonClassId, - var comparable: Boolean = true + var comparable: Boolean = true, ) { + constructor(type: PythonClassId, comparable: Boolean = true) : this(PythonIdGenerator.createId(), type, comparable) + open val children: List = emptyList() open fun typeEquals(other: Any?): Boolean { @@ -40,9 +46,11 @@ object PythonTree { } class PrimitiveNode( + id: Long, type: PythonClassId, val repr: String, - ) : PythonTreeNode(type) { + ) : PythonTreeNode(id, type) { + constructor(type: PythonClassId, repr: String) : this(PythonIdGenerator.getOrCreateIdForValue(repr), type, repr) override fun equals(other: Any?): Boolean { if (other !is PrimitiveNode) { return false @@ -58,8 +66,11 @@ object PythonTree { } class ListNode( + id: Long, val items: MutableMap - ) : PythonTreeNode(PythonClassId("builtins.list")) { + ) : PythonTreeNode(id, PythonClassId("builtins.list")) { + constructor(items: MutableMap) : this(PythonIdGenerator.createId(), items) + override val children: List get() = items.values.toList() @@ -86,8 +97,11 @@ object PythonTree { } class DictNode( + id: Long, val items: MutableMap - ) : PythonTreeNode(PythonClassId("builtins.dict")) { + ) : PythonTreeNode(id, PythonClassId("builtins.dict")) { + constructor(items: MutableMap) : this(PythonIdGenerator.createId(), items) + override val children: List get() = items.values + items.keys @@ -116,8 +130,11 @@ object PythonTree { } class SetNode( + id: Long, val items: MutableSet - ) : PythonTreeNode(PythonClassId("builtins.set")) { + ) : PythonTreeNode(id, PythonClassId("builtins.set")) { + constructor(items: MutableSet) : this(PythonIdGenerator.createId(), items) + override val children: List get() = items.toList() @@ -150,8 +167,11 @@ object PythonTree { } class TupleNode( + id: Long, val items: MutableMap - ) : PythonTreeNode(PythonClassId("builtins.tuple")) { + ) : PythonTreeNode(id, PythonClassId("builtins.tuple")) { + constructor(items: MutableMap) : this(PythonIdGenerator.createId(), items) + override val children: List get() = items.values.toList() @@ -180,20 +200,19 @@ object PythonTree { } class ReduceNode( - val id: Long, + id: Long, type: PythonClassId, val constructor: PythonClassId, val args: List, var state: MutableMap, var listitems: List, var dictitems: Map, - ) : PythonTreeNode(type) { + ) : PythonTreeNode(id, type) { constructor( - id: Long, type: PythonClassId, constructor: PythonClassId, args: List, - ) : this(id, type, constructor, args, emptyMap().toMutableMap(), emptyList(), emptyMap()) + ) : this(PythonIdGenerator.createId(), type, constructor, args, emptyMap().toMutableMap(), emptyList(), emptyMap()) override val children: List get() = args + state.values + listitems + dictitems.values + dictitems.keys + PythonTreeNode(constructor) @@ -310,4 +329,20 @@ object PythonTree { else -> null } } -} \ No newline at end of file +} + +object PythonIdGenerator { + private const val lower_bound: Long = 1500_000_000 + + private val lastId: AtomicLong = AtomicLong(lower_bound) + private val cache: IdentityHashMap = IdentityHashMap() + + fun getOrCreateIdForValue(value: Any): Long { + return cache.getOrPut(value) { createId() } + } + + fun createId(): Long { + return lastId.incrementAndGet() + } + +} diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonIdUtils.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonIdUtils.kt index 6ee9e07661..9a15f99dd4 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonIdUtils.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonIdUtils.kt @@ -16,11 +16,11 @@ val pythonIntClassId = PythonClassId("builtins.int") val pythonFloatClassId = PythonClassId("builtins.float") val pythonComplexClassId = PythonClassId("builtins.complex") val pythonStrClassId = PythonClassId("builtins.str") -val pythonBoolClassId = PythonBoolModel.classId +val pythonBoolClassId = PythonClassId("builtins.bool") val pythonRangeClassId = PythonClassId("builtins.range") -val pythonListClassId = PythonListModel.classId -val pythonTupleClassId = PythonTupleModel.classId -val pythonDictClassId = PythonDictModel.classId -val pythonSetClassId = PythonSetModel.classId +val pythonListClassId = PythonClassId("builtins.list") +val pythonTupleClassId = PythonClassId("builtins.tuple") +val pythonDictClassId = PythonClassId("builtins.dict") +val pythonSetClassId = PythonClassId("builtins.set") val pythonBytearrayClassId = PythonClassId("builtins.bytearray") val pythonBytesClassId = PythonClassId("builtins.bytes") diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt index e070410ca8..c8291d7ac9 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt @@ -4,6 +4,7 @@ import mu.KotlinLogging import org.utbot.framework.plugin.api.Instruction import org.utbot.fuzzer.FuzzedContext import org.utbot.fuzzer.IdGenerator +import org.utbot.fuzzer.IdentityPreservingIdGenerator import org.utbot.fuzzing.Configuration import org.utbot.fuzzing.Control import org.utbot.fuzzing.Description @@ -35,6 +36,8 @@ import org.utbot.python.newtyping.PythonSubtypeChecker import org.utbot.python.newtyping.PythonTypeStorage import org.utbot.python.newtyping.general.Type import org.utbot.python.newtyping.pythonTypeRepresentation +import java.util.* +import java.util.concurrent.atomic.AtomicLong private val logger = KotlinLogging.logger {} @@ -77,7 +80,7 @@ fun pythonDefaultValueProviders(idGenerator: IdGenerator) = listOf( UnionValueProvider, BytesValueProvider, BytearrayValueProvider, - ReduceValueProvider(idGenerator), + ReduceValueProvider, ConstantValueProvider, ) @@ -85,6 +88,7 @@ class PythonFuzzing( private val pythonTypeStorage: PythonTypeStorage, val execute: suspend (description: PythonMethodDescription, values: List) -> PythonFeedback, ) : Fuzzing { + private fun generateDefault(description: PythonMethodDescription, type: Type, idGenerator: IdGenerator): Sequence> { var providers = emptyList>().asSequence() pythonDefaultValueProviders(idGenerator).asSequence().forEach { provider -> @@ -136,12 +140,19 @@ class PythonFuzzing( } } -class PythonIdGenerator : IdGenerator { - private var _id: Long = 0 +class PythonIdGenerator(lowerBound: Long = DEFAULT_LOWER_BOUND) : IdentityPreservingIdGenerator { + private val lastId: AtomicLong = AtomicLong(lowerBound) + private val cache: IdentityHashMap = IdentityHashMap() + + override fun getOrCreateIdForValue(value: Any): Long { + return cache.getOrPut(value) { createId() } + } override fun createId(): Long { - _id += 1 - return _id + return lastId.incrementAndGet() } + companion object { + const val DEFAULT_LOWER_BOUND: Long = 1500_000_000 + } } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ReduceValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ReduceValueProvider.kt index f91f43075c..e8995bcad0 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ReduceValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ReduceValueProvider.kt @@ -1,6 +1,5 @@ package org.utbot.python.fuzzing.provider -import org.utbot.fuzzer.IdGenerator import org.utbot.fuzzing.Routine import org.utbot.fuzzing.Seed import org.utbot.fuzzing.ValueProvider @@ -12,9 +11,7 @@ import org.utbot.python.newtyping.* import org.utbot.python.newtyping.general.FunctionType import org.utbot.python.newtyping.general.Type -class ReduceValueProvider( - private val idGenerator: IdGenerator -) : ValueProvider { +object ReduceValueProvider : ValueProvider { private val unsupportedTypes = listOf( "builtins.list", "builtins.set", @@ -72,7 +69,6 @@ class ReduceValueProvider( construct = Routine.Create(nonSelfArgs) { v -> PythonFuzzedValue( PythonTree.ReduceNode( - idGenerator.createId(), PythonClassId(type.pythonTypeName()), PythonClassId(type.pythonTypeName()), v.map { it.tree }, From 3fd7906e27a6a878ddfb2e1cc71484d8c7938c17 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Wed, 15 Feb 2023 15:35:28 +0300 Subject: [PATCH 02/32] Work on sockets --- .../org/utbot/python/evaluation/PythonCodeSocketExecutor.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt index 5f7137e074..352afc936f 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt @@ -26,5 +26,6 @@ class PythonCodeSocketExecutor( fuzzedValues: List, additionalModulesToImport: Set): PythonEvaluationResult { // val input = + TODO() } } \ No newline at end of file From e606b2d5ad59bfe5785477b5bba6ad76d908471d Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Mon, 20 Feb 2023 13:46:07 +0300 Subject: [PATCH 03/32] Added new execution version with sockets --- .../kotlin/org/utbot/python/PythonEngine.kt | 46 ++++---- .../utbot/python/PythonTestCaseGenerator.kt | 2 +- .../kotlin/org/utbot/python/UTPythonAPI.kt | 8 +- .../python/evaluation/CodeEvaluationApi.kt | 4 +- .../python/evaluation/ExecutionClient.kt | 33 ++++-- .../evaluation/PythonCodeExecutorImpl.kt | 13 ++- .../evaluation/PythonCodeSocketExecutor.kt | 103 +++++++++++++++--- .../ExecutionRequestSerializer.kt | 26 +++++ 8 files changed, 171 insertions(+), 64 deletions(-) create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionRequestSerializer.kt diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index 71d387f89e..033454ef81 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -11,13 +11,12 @@ import org.utbot.framework.plugin.api.UtExecutionResult import org.utbot.framework.plugin.api.UtExecutionSuccess import org.utbot.framework.plugin.api.UtExplicitlyThrownException import org.utbot.framework.plugin.api.UtModel -import org.utbot.fuzzer.FuzzedValue import org.utbot.fuzzer.UtFuzzedExecution import org.utbot.fuzzing.Control import org.utbot.fuzzing.fuzz import org.utbot.fuzzing.utils.Trie import org.utbot.python.evaluation.PythonCodeExecutor -import org.utbot.python.evaluation.PythonCodeExecutorImpl +import org.utbot.python.evaluation.PythonCodeSocketExecutor import org.utbot.python.evaluation.PythonEvaluationError import org.utbot.python.evaluation.PythonEvaluationSuccess import org.utbot.python.evaluation.PythonEvaluationTimeout @@ -26,7 +25,6 @@ import org.utbot.python.evaluation.serialiation.toPythonTree import org.utbot.python.framework.api.python.PythonTreeModel import org.utbot.python.fuzzing.PythonFeedback import org.utbot.python.fuzzing.PythonFuzzedConcreteValue -import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonFuzzing import org.utbot.python.fuzzing.PythonMethodDescription import org.utbot.python.newtyping.PythonTypeStorage @@ -146,26 +144,10 @@ class PythonEngine( return ValidExecution(utFuzzedExecution) } - private fun constructEvaluationInput(arguments: List, additionalModules: List): PythonCodeExecutor { - val argumentValues = arguments.map { PythonTreeModel(it.tree, it.tree.type) } - - val (thisObject, modelList) = - if (methodUnderTest.hasThisArgument) - Pair(argumentValues[0], argumentValues.drop(1)) - else - Pair(null, argumentValues) - - val argumentModules = argumentValues - .flatMap { it.allContainingClassIds } - .map { it.moduleName } - val localAdditionalModules = (additionalModules + argumentModules).toSet() - - return PythonCodeExecutorImpl( + private fun constructEvaluationInput(): PythonCodeExecutor { + return PythonCodeSocketExecutor( methodUnderTest, - FunctionArguments(thisObject, methodUnderTest.thisObjectName, modelList, methodUnderTest.argumentsNames), - argumentValues.map { FuzzedValue(it) }, moduleToImport, - localAdditionalModules, pythonPath, directoriesForSysPath, timeoutForRun, @@ -185,6 +167,7 @@ class PythonEngine( val coveredLines = initialCoveredLines.toMutableSet() + val codeExecutor = constructEvaluationInput() PythonFuzzing(pmd.pythonTypeStorage) { description, arguments -> if (isCancelled()) { logger.info { "Fuzzing process was interrupted" } @@ -195,8 +178,20 @@ class PythonEngine( return@PythonFuzzing PythonFeedback(control = Control.STOP) } - val codeExecutor = constructEvaluationInput(arguments, additionalModules) - when (val evaluationResult = codeExecutor.run()) { + val argumentValues = arguments.map { PythonTreeModel(it.tree, it.tree.type) } + val argumentModules = argumentValues + .flatMap { it.allContainingClassIds } + .map { it.moduleName } + val localAdditionalModules = (additionalModules + argumentModules).toSet() + + val (thisObject, modelList) = + if (methodUnderTest.hasThisArgument) + Pair(argumentValues[0], argumentValues.drop(1)) + else + Pair(null, argumentValues) + val functionArguments = FunctionArguments(thisObject, methodUnderTest.thisObjectName, modelList, methodUnderTest.argumentsNames) + + when (val evaluationResult = codeExecutor.run(functionArguments, localAdditionalModules)) { is PythonEvaluationError -> { val utError = UtError( "Error evaluation: ${evaluationResult.status}, ${evaluationResult.message}", @@ -221,7 +216,7 @@ class PythonEngine( .zip(methodUnderTest.arguments) .mapNotNull { it.first.summary?.replace("%var%", it.second.name) } - val hasThisObject = codeExecutor.methodArguments.thisObject != null + val hasThisObject = codeExecutor.method.hasThisArgument when (val result = handleSuccessResult(parameters, evaluationResult, description, hasThisObject, summary)) { is ValidExecution -> { @@ -242,5 +237,8 @@ class PythonEngine( } } }.fuzz(pmd) + if (codeExecutor is PythonCodeSocketExecutor) { + codeExecutor.stop() + } } } \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt index 6187355209..a1c6a68554 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt @@ -29,7 +29,7 @@ import java.io.File private val logger = KotlinLogging.logger {} -private const val COVERAGE_LIMIT = 100 +private const val COVERAGE_LIMIT = 10_000 class PythonTestCaseGenerator( private val withMinimization: Boolean = true, diff --git a/utbot-python/src/main/kotlin/org/utbot/python/UTPythonAPI.kt b/utbot-python/src/main/kotlin/org/utbot/python/UTPythonAPI.kt index fa5bda5f57..187b78cc6d 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/UTPythonAPI.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/UTPythonAPI.kt @@ -4,7 +4,7 @@ import org.parsers.python.ast.Block import org.utbot.framework.plugin.api.UtError import org.utbot.framework.plugin.api.UtExecution import org.utbot.python.framework.api.python.PythonClassId -import org.utbot.python.framework.api.python.PythonModel +import org.utbot.python.framework.api.python.PythonTreeModel import org.utbot.python.framework.api.python.util.pythonAnyClassId import org.utbot.python.newtyping.* import org.utbot.python.typing.MypyAnnotations @@ -60,10 +60,10 @@ data class PythonTestSet( ) data class FunctionArguments( - val thisObject: PythonModel?, + val thisObject: PythonTreeModel?, val thisObjectName: String?, - val arguments: List, + val arguments: List, val names: List, ) { - val allArguments: List = (listOf(thisObject) + arguments).filterNotNull() + val allArguments: List = (listOf(thisObject) + arguments).filterNotNull() } \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt index afb1db6f4d..4e3534cefa 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt @@ -1,21 +1,19 @@ package org.utbot.python.evaluation import org.utbot.framework.plugin.api.Coverage -import org.utbot.fuzzer.FuzzedValue import org.utbot.python.FunctionArguments import org.utbot.python.PythonMethod import org.utbot.python.evaluation.serialiation.MemoryDump interface PythonCodeExecutor { val method: PythonMethod - val methodArguments: FunctionArguments val moduleToImport: String val pythonPath: String val syspathDirectories: Set val executionTimeout: Long fun run( - fuzzedValues: List, + fuzzedValues: FunctionArguments, additionalModulesToImport: Set ): PythonEvaluationResult } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/ExecutionClient.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/ExecutionClient.kt index 4528906ca0..e9f84032b7 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/ExecutionClient.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/ExecutionClient.kt @@ -1,27 +1,36 @@ package org.utbot.python.evaluation +import org.utbot.python.utils.startProcess import java.io.BufferedReader import java.io.BufferedWriter import java.io.InputStreamReader import java.io.OutputStreamWriter -import java.io.PrintWriter import java.net.Socket -import java.nio.CharBuffer class ExecutionClient( - hostname: String, - port: Int + private val hostname: String, + private val port: Int, + val pythonPath: String ) { private val clientSocket: Socket = Socket(hostname, port) private val outStream = BufferedWriter(OutputStreamWriter(clientSocket.getOutputStream())) private val inStream = BufferedReader(InputStreamReader(clientSocket.getInputStream())) - - init { - runServer() - } - - private fun runServer() { - TODO() + private val process = runServer() + + private fun runServer() = + startProcess( + listOf( + pythonPath, + "-m", + "utbot_executor", + hostname, + port.toString() + ) + ) + + fun stopServer() { + stopConnection() + process.destroy() // ? } fun sendMessage(msg: String) { @@ -43,7 +52,7 @@ class ExecutionClient( } fun main() { - val client = ExecutionClient("localhost", 12011) + val client = ExecutionClient("localhost", 12011, "python") client.sendMessage("Firstfjlskdjf") client.sendMessage("Second") client.sendMessage("{123: 123}") diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeExecutorImpl.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeExecutorImpl.kt index b9de31646c..e7c90fd747 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeExecutorImpl.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeExecutorImpl.kt @@ -2,13 +2,13 @@ package org.utbot.python.evaluation import org.utbot.framework.plugin.api.Coverage import org.utbot.framework.plugin.api.Instruction -import org.utbot.fuzzer.FuzzedValue import org.utbot.python.FunctionArguments import org.utbot.python.PythonMethod import org.utbot.python.code.PythonCodeGenerator import org.utbot.python.evaluation.serialiation.ExecutionResultDeserializer import org.utbot.python.evaluation.serialiation.FailExecution import org.utbot.python.evaluation.serialiation.SuccessExecution +import org.utbot.python.framework.api.python.PythonTreeModel import org.utbot.python.framework.api.python.util.pythonAnyClassId import org.utbot.python.utils.TemporaryFileManager import org.utbot.python.utils.getResult @@ -22,7 +22,6 @@ data class EvaluationFiles( class PythonCodeExecutorImpl( override val method: PythonMethod, - override val methodArguments: FunctionArguments, override val moduleToImport: String, override val pythonPath: String, override val syspathDirectories: Set, @@ -30,15 +29,19 @@ class PythonCodeExecutorImpl( ) : PythonCodeExecutor { override fun run( - fuzzedValues: List, + fuzzedValues: FunctionArguments, additionalModulesToImport: Set, ): PythonEvaluationResult { - val evaluationFiles = generateExecutionCode(additionalModulesToImport) + val evaluationFiles = generateExecutionCode( + additionalModulesToImport, + fuzzedValues.allArguments, + ) return getEvaluationResult(evaluationFiles) } private fun generateExecutionCode( additionalModulesToImport: Set, + methodArguments: List, ): EvaluationFiles { val fileForOutput = TemporaryFileManager.assignTemporaryFile( tag = "out_" + method.name + ".py", @@ -50,7 +53,7 @@ class PythonCodeExecutorImpl( ) val runCode = PythonCodeGenerator.generateRunFunctionCode( method, - methodArguments.allArguments, + methodArguments, syspathDirectories, moduleToImport, additionalModulesToImport, diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt index 352afc936f..a3829b4d9c 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt @@ -1,31 +1,104 @@ package org.utbot.python.evaluation -import org.utbot.fuzzer.FuzzedValue +import org.utbot.framework.plugin.api.Coverage +import org.utbot.framework.plugin.api.Instruction import org.utbot.python.FunctionArguments import org.utbot.python.PythonMethod - - -data class ExecutionDescription( - val functionName: String, - val imports: List, - val argumentsIds: List, - val serializedMemory: String -) +import org.utbot.python.evaluation.serialiation.ExecutionRequest +import org.utbot.python.evaluation.serialiation.ExecutionRequestSerializer +import org.utbot.python.evaluation.serialiation.ExecutionResultDeserializer +import org.utbot.python.evaluation.serialiation.FailExecution +import org.utbot.python.evaluation.serialiation.PythonExecutionResult +import org.utbot.python.evaluation.serialiation.SuccessExecution +import org.utbot.python.evaluation.serialiation.serializeObjects +import org.utbot.python.framework.api.python.util.pythonAnyClassId +import org.utbot.python.utils.TemporaryFileManager class PythonCodeSocketExecutor( override val method: PythonMethod, - override val methodArguments: FunctionArguments, override val moduleToImport: String, override val pythonPath: String, override val syspathDirectories: Set, override val executionTimeout: Long, ) : PythonCodeExecutor { - val executionClient = ExecutionClient("localhost", 12011) // TODO: create port manager + private val executionClient = ExecutionClient("localhost", 12011, pythonPath) // TODO: create port manager override fun run( - fuzzedValues: List, - additionalModulesToImport: Set): PythonEvaluationResult { -// val input = - TODO() + fuzzedValues: FunctionArguments, + additionalModulesToImport: Set + ): PythonEvaluationResult { + val (arguments, memory) = serializeObjects(fuzzedValues.allArguments.map { it.tree }) + val coverageDatabasePath = TemporaryFileManager.assignTemporaryFile( + tag = "coverage_db_" + method.name, + addToCleaner = false, + ) + val request = ExecutionRequest( + method.name, + (additionalModulesToImport + moduleToImport).toList(), + syspathDirectories.toList(), + arguments, + memory, + coverageDatabasePath.path, + method.moduleFilename, + ) + val message = ExecutionRequestSerializer.serializeRequest(request) ?: error("Cannot serialize request to python executor") + executionClient.sendMessage(message) + val response = executionClient.receiveMessage() ?: error("Cannot read response from python executor") + val executionResult = ExecutionResultDeserializer.parseExecutionResult(response) ?: error("Cannot parse execution result") + return parseExecutionResult(executionResult) + } + + private fun parseExecutionResult(executionResult: PythonExecutionResult): PythonEvaluationResult { + val parsingException = PythonEvaluationError( + 0, + "Incorrect format of output", + emptyList() + ) + return when (executionResult) { + is SuccessExecution -> { + val stateBefore = ExecutionResultDeserializer.parseMemoryDump(executionResult.stateBefore) ?: return parsingException + val stateAfter = ExecutionResultDeserializer.parseMemoryDump(executionResult.stateAfter) ?: return parsingException + PythonEvaluationSuccess( + executionResult.isException, + calculateCoverage(executionResult.statements, executionResult.missedStatements), + stateBefore, + stateAfter, + executionResult.argsIds + executionResult.kwargsIds, + executionResult.resultId, + ) + } + is FailExecution -> PythonEvaluationError( + 0, + executionResult.exception, + emptyList(), + ) + } + } + + private fun calculateCoverage(statements: List, missedStatements: List): Coverage { + val covered = statements.filter { it !in missedStatements } + return Coverage( + coveredInstructions=covered.map { + Instruction( + method.containingPythonClassId?.name ?: pythonAnyClassId.name, + method.methodSignature(), + it, + it.toLong() + ) + }, + instructionsCount = statements.size.toLong(), + missedInstructions = missedStatements.map { + Instruction( + method.containingPythonClassId?.name ?: pythonAnyClassId.name, + method.methodSignature(), + it, + it.toLong() + ) + } + ) + } + + fun stop() { + executionClient.stopServer() } } \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionRequestSerializer.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionRequestSerializer.kt new file mode 100644 index 0000000000..c554214236 --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionRequestSerializer.kt @@ -0,0 +1,26 @@ +package org.utbot.python.evaluation.serialiation + +import com.squareup.moshi.Moshi +import com.squareup.moshi.kotlin.reflect.KotlinJsonAdapterFactory + +object ExecutionRequestSerializer { + private val moshi = Moshi.Builder() + .addLast(KotlinJsonAdapterFactory()) + .build() + + private val jsonAdapter = moshi.adapter(ExecutionRequest::class.java) + + fun serializeRequest(request: ExecutionRequest): String? { + return jsonAdapter.toJson(request) + } +} + +data class ExecutionRequest( + val functionName: String, + val imports: List, + val syspaths: List, + val argumentsIds: List, + val serializedMemory: String, + val coverageDB: String, + val filepath: String, +) From 7ff12cd6dc247a450bac1995a07ad49533be337c Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Wed, 22 Feb 2023 12:59:17 +0300 Subject: [PATCH 04/32] Update socket version --- .../kotlin/org/utbot/python/PythonEngine.kt | 161 +++++++++++------- .../utbot/python/PythonTestCaseGenerator.kt | 13 +- .../python/evaluation/ExecutionClient.kt | 61 ------- .../evaluation/PythonCodeSocketExecutor.kt | 47 +++-- .../utbot/python/evaluation/PythonWorker.kt | 48 ++++++ .../ExecutionRequestSerializer.kt | 2 +- .../ExecutionResultDeserializer.kt | 8 +- .../utbot/python/newtyping/mypy/RunMypy.kt | 1 - 8 files changed, 197 insertions(+), 144 deletions(-) delete mode 100644 utbot-python/src/main/kotlin/org/utbot/python/evaluation/ExecutionClient.kt create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorker.kt diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index 033454ef81..f2cdc177d8 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -20,6 +20,7 @@ import org.utbot.python.evaluation.PythonCodeSocketExecutor import org.utbot.python.evaluation.PythonEvaluationError import org.utbot.python.evaluation.PythonEvaluationSuccess import org.utbot.python.evaluation.PythonEvaluationTimeout +import org.utbot.python.evaluation.PythonWorker import org.utbot.python.evaluation.serialiation.MemoryDump import org.utbot.python.evaluation.serialiation.toPythonTree import org.utbot.python.framework.api.python.PythonTreeModel @@ -32,7 +33,11 @@ import org.utbot.python.newtyping.general.Type import org.utbot.python.newtyping.pythonModules import org.utbot.python.newtyping.pythonTypeRepresentation import org.utbot.python.utils.camelToSnakeCase +import org.utbot.python.utils.startProcess import org.utbot.summary.fuzzer.names.TestSuggestedInfo +import java.net.ServerSocket +import java.net.SocketTimeoutException +import java.util.concurrent.TimeUnit private val logger = KotlinLogging.logger {} @@ -144,13 +149,14 @@ class PythonEngine( return ValidExecution(utFuzzedExecution) } - private fun constructEvaluationInput(): PythonCodeExecutor { + private fun constructEvaluationInput(pythonWorker: PythonWorker): PythonCodeExecutor { return PythonCodeSocketExecutor( methodUnderTest, moduleToImport, pythonPath, directoriesForSysPath, timeoutForRun, + pythonWorker, ) } @@ -167,78 +173,107 @@ class PythonEngine( val coveredLines = initialCoveredLines.toMutableSet() - val codeExecutor = constructEvaluationInput() - PythonFuzzing(pmd.pythonTypeStorage) { description, arguments -> - if (isCancelled()) { - logger.info { "Fuzzing process was interrupted" } - return@PythonFuzzing PythonFeedback(control = Control.STOP) - } - if (System.currentTimeMillis() >= until) { - logger.info { "Fuzzing process was interrupted by timeout" } - return@PythonFuzzing PythonFeedback(control = Control.STOP) - } - - val argumentValues = arguments.map { PythonTreeModel(it.tree, it.tree.type) } - val argumentModules = argumentValues - .flatMap { it.allContainingClassIds } - .map { it.moduleName } - val localAdditionalModules = (additionalModules + argumentModules).toSet() - - val (thisObject, modelList) = - if (methodUnderTest.hasThisArgument) - Pair(argumentValues[0], argumentValues.drop(1)) - else - Pair(null, argumentValues) - val functionArguments = FunctionArguments(thisObject, methodUnderTest.thisObjectName, modelList, methodUnderTest.argumentsNames) - - when (val evaluationResult = codeExecutor.run(functionArguments, localAdditionalModules)) { - is PythonEvaluationError -> { - val utError = UtError( - "Error evaluation: ${evaluationResult.status}, ${evaluationResult.message}", - Throwable(evaluationResult.stackTrace.joinToString("\n")) - ) - logger.debug(evaluationResult.stackTrace.joinToString("\n")) - emit(InvalidExecution(utError)) - return@PythonFuzzing PythonFeedback(control = Control.PASS) - } - - is PythonEvaluationTimeout -> { - val utError = UtError(evaluationResult.message, Throwable()) - emit(InvalidExecution(utError)) - return@PythonFuzzing PythonFeedback(control = Control.PASS) +// withContext(Dispatchers.IO) { + ServerSocket(0).use { serverSocket -> + logger.info { "Server port: ${serverSocket.localPort}" } + val processStartTime = System.currentTimeMillis() +// val outputFile = "/home/vyacheslav/utbot_profile" +// val process = startProcess(listOf(pythonPath, "-m", "cProfile", "-o", outputFile, "-m", "utbot_executor", "localhost", serverSocket.localPort.toString())) + val process = startProcess(listOf(pythonPath, "-m", "utbot_executor", "localhost", serverSocket.localPort.toString())) + val timeout = until - processStartTime + val workerSocket = try { + serverSocket.soTimeout = timeout.toInt() + serverSocket.accept() + } catch (e: SocketTimeoutException) { + val processHasExited = process.waitFor(timeout, TimeUnit.MILLISECONDS) + if (!processHasExited) { + process.destroy() + } + error("Worker not connected") } + logger.info { "Worker connected successfully" } + val pythonWorker = PythonWorker(workerSocket) + val codeExecutor = constructEvaluationInput(pythonWorker) + logger.info { "Executor was created successfully" } + PythonFuzzing(pmd.pythonTypeStorage) { description, arguments -> + if (isCancelled()) { + logger.info { "Fuzzing process was interrupted" } + return@PythonFuzzing PythonFeedback(control = Control.STOP) + } + if (System.currentTimeMillis() >= until) { + logger.info { "Fuzzing process was interrupted by timeout" } + return@PythonFuzzing PythonFeedback(control = Control.STOP) + } - is PythonEvaluationSuccess -> { - val coveredInstructions = evaluationResult.coverage.coveredInstructions - coveredInstructions.forEach { coveredLines.add(it.lineNumber) } - - val summary = arguments - .zip(methodUnderTest.arguments) - .mapNotNull { it.first.summary?.replace("%var%", it.second.name) } + val argumentValues = arguments.map { PythonTreeModel(it.tree, it.tree.type) } + val argumentModules = argumentValues + .flatMap { it.allContainingClassIds } + .map { it.moduleName } + val localAdditionalModules = (additionalModules + argumentModules).toSet() - val hasThisObject = codeExecutor.method.hasThisArgument + val (thisObject, modelList) = + if (methodUnderTest.hasThisArgument) + Pair(argumentValues[0], argumentValues.drop(1)) + else + Pair(null, argumentValues) + val functionArguments = FunctionArguments( + thisObject, + methodUnderTest.thisObjectName, + modelList, + methodUnderTest.argumentsNames + ) - when (val result = handleSuccessResult(parameters, evaluationResult, description, hasThisObject, summary)) { - is ValidExecution -> { - logger.debug { arguments } - val trieNode: Trie.Node = description.tracer.add(coveredInstructions) - emit(result) - return@PythonFuzzing PythonFeedback(control = Control.CONTINUE, result = trieNode) + when (val evaluationResult = codeExecutor.run(functionArguments, localAdditionalModules)) { + is PythonEvaluationError -> { + val utError = UtError( + "Error evaluation: ${evaluationResult.status}, ${evaluationResult.message}", + Throwable(evaluationResult.stackTrace.joinToString("\n")) + ) + logger.debug(evaluationResult.stackTrace.joinToString("\n")) + emit(InvalidExecution(utError)) + return@PythonFuzzing PythonFeedback(control = Control.PASS) } - is ArgumentsTypeErrorFeedback, is TypeErrorFeedback -> { - emit(result) + + is PythonEvaluationTimeout -> { + val utError = UtError(evaluationResult.message, Throwable()) + emit(InvalidExecution(utError)) return@PythonFuzzing PythonFeedback(control = Control.PASS) } - is InvalidExecution -> { - emit(result) - return@PythonFuzzing PythonFeedback(control = Control.CONTINUE) + + is PythonEvaluationSuccess -> { + val coveredInstructions = evaluationResult.coverage.coveredInstructions + coveredInstructions.forEach { coveredLines.add(it.lineNumber) } + + val summary = arguments + .zip(methodUnderTest.arguments) + .mapNotNull { it.first.summary?.replace("%var%", it.second.name) } + + val hasThisObject = codeExecutor.method.hasThisArgument + + when (val result = handleSuccessResult(parameters, evaluationResult, description, hasThisObject, summary)) { + is ValidExecution -> { + logger.debug { arguments } + val trieNode: Trie.Node = description.tracer.add(coveredInstructions) + emit(result) + return@PythonFuzzing PythonFeedback(control = Control.CONTINUE, result = trieNode) + } + is ArgumentsTypeErrorFeedback, is TypeErrorFeedback -> { + emit(result) + return@PythonFuzzing PythonFeedback(control = Control.PASS) + } + is InvalidExecution -> { + emit(result) + return@PythonFuzzing PythonFeedback(control = Control.CONTINUE) + } + } } } + }.fuzz(pmd) + if (codeExecutor is PythonCodeSocketExecutor) { + codeExecutor.stop() } } - }.fuzz(pmd) - if (codeExecutor is PythonCodeSocketExecutor) { - codeExecutor.stop() - } +// } + } } \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt index a1c6a68554..da206d09e5 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt @@ -29,7 +29,8 @@ import java.io.File private val logger = KotlinLogging.logger {} -private const val COVERAGE_LIMIT = 10_000 +private const val COVERAGE_LIMIT = 150 +private const val ADDITIONAL_LIMIT = 10 class PythonTestCaseGenerator( private val withMinimization: Boolean = true, @@ -108,7 +109,8 @@ class PythonTestCaseGenerator( var missingLines: Set? = null val coveredLines = mutableSetOf() var generated = 0 - val typeInferenceCancellation = { isCancelled() || System.currentTimeMillis() >= until || missingLines?.size == 0 } + var additionalLimit = ADDITIONAL_LIMIT + val typeInferenceCancellation = { isCancelled() || System.currentTimeMillis() >= until || (missingLines?.size == 0 && additionalLimit == 0) } inferAnnotations( method, @@ -140,7 +142,7 @@ class PythonTestCaseGenerator( var feedback: InferredTypeFeedback = SuccessFeedback val fuzzerCancellation = { typeInferenceCancellation() || coverageLimit == 0 } // || feedback is InvalidTypeFeedback } - + val startTime = System.currentTimeMillis() engine.fuzzing(args, fuzzerCancellation, until).collect { generated += 1 when (it) { @@ -160,16 +162,19 @@ class PythonTestCaseGenerator( feedback = InvalidTypeFeedback } } + if (missingLines?.size == 0) { + additionalLimit -= 1 + } val coveredAfter = coveredLines.size if (coveredAfter == coveredBefore) { coverageLimit -= 1 } + logger.info { "${System.currentTimeMillis() - startTime}: $generated, $missingLines" } coveredBefore = coveredAfter } feedback } - val (successfulExecutions, failedExecutions) = executions.partition { it.result is UtExecutionSuccess } return PythonTestSet( diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/ExecutionClient.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/ExecutionClient.kt deleted file mode 100644 index e9f84032b7..0000000000 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/ExecutionClient.kt +++ /dev/null @@ -1,61 +0,0 @@ -package org.utbot.python.evaluation - -import org.utbot.python.utils.startProcess -import java.io.BufferedReader -import java.io.BufferedWriter -import java.io.InputStreamReader -import java.io.OutputStreamWriter -import java.net.Socket - -class ExecutionClient( - private val hostname: String, - private val port: Int, - val pythonPath: String -) { - private val clientSocket: Socket = Socket(hostname, port) - private val outStream = BufferedWriter(OutputStreamWriter(clientSocket.getOutputStream())) - private val inStream = BufferedReader(InputStreamReader(clientSocket.getInputStream())) - private val process = runServer() - - private fun runServer() = - startProcess( - listOf( - pythonPath, - "-m", - "utbot_executor", - hostname, - port.toString() - ) - ) - - fun stopServer() { - stopConnection() - process.destroy() // ? - } - - fun sendMessage(msg: String) { - outStream.write(msg) - outStream.flush() - outStream.write("END") - outStream.flush() - } - - fun receiveMessage(): String? { - return inStream.readLine() - } - - fun stopConnection() { - inStream.close() - outStream.close() - clientSocket.close() - } -} - -fun main() { - val client = ExecutionClient("localhost", 12011, "python") - client.sendMessage("Firstfjlskdjf") - client.sendMessage("Second") - client.sendMessage("{123: 123}") - client.sendMessage("STOP") - client.stopConnection() -} \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt index a3829b4d9c..92fbb5a5b8 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt @@ -12,7 +12,6 @@ import org.utbot.python.evaluation.serialiation.PythonExecutionResult import org.utbot.python.evaluation.serialiation.SuccessExecution import org.utbot.python.evaluation.serialiation.serializeObjects import org.utbot.python.framework.api.python.util.pythonAnyClassId -import org.utbot.python.utils.TemporaryFileManager class PythonCodeSocketExecutor( @@ -22,29 +21,51 @@ class PythonCodeSocketExecutor( override val syspathDirectories: Set, override val executionTimeout: Long, ) : PythonCodeExecutor { - private val executionClient = ExecutionClient("localhost", 12011, pythonPath) // TODO: create port manager + private lateinit var pythonWorker: PythonWorker + + constructor( + method: PythonMethod, + moduleToImport: String, + pythonPath: String, + syspathDirectories: Set, + executionTimeout: Long, + pythonWorker: PythonWorker + ) : this( + method, + moduleToImport, + pythonPath, + syspathDirectories, + executionTimeout + ) { + this.pythonWorker = pythonWorker + } + override fun run( fuzzedValues: FunctionArguments, additionalModulesToImport: Set ): PythonEvaluationResult { val (arguments, memory) = serializeObjects(fuzzedValues.allArguments.map { it.tree }) - val coverageDatabasePath = TemporaryFileManager.assignTemporaryFile( - tag = "coverage_db_" + method.name, - addToCleaner = false, - ) + + val containingClass = method.containingPythonClassId + val functionTextName = + if (containingClass == null) + method.name + else + "${containingClass.simpleName}.${method.name}" + val request = ExecutionRequest( - method.name, - (additionalModulesToImport + moduleToImport).toList(), + functionTextName, + moduleToImport, + additionalModulesToImport.toList(), syspathDirectories.toList(), arguments, memory, - coverageDatabasePath.path, method.moduleFilename, ) val message = ExecutionRequestSerializer.serializeRequest(request) ?: error("Cannot serialize request to python executor") - executionClient.sendMessage(message) - val response = executionClient.receiveMessage() ?: error("Cannot read response from python executor") - val executionResult = ExecutionResultDeserializer.parseExecutionResult(response) ?: error("Cannot parse execution result") + pythonWorker.sendData(message) + val response = pythonWorker.receiveMessage() + val executionResult = ExecutionResultDeserializer.parseExecutionResult(response) ?: error("Cannot parse execution result: $response") return parseExecutionResult(executionResult) } @@ -99,6 +120,6 @@ class PythonCodeSocketExecutor( } fun stop() { - executionClient.stopServer() + pythonWorker.stopServer() } } \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorker.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorker.kt new file mode 100644 index 0000000000..26acf722e6 --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorker.kt @@ -0,0 +1,48 @@ +package org.utbot.python.evaluation + +import java.io.BufferedReader +import java.io.BufferedWriter +import java.io.InputStreamReader +import java.io.OutputStreamWriter +import java.net.Socket + +class PythonWorker( + private val clientSocket: Socket +) { + private val outStream = BufferedWriter(OutputStreamWriter(clientSocket.getOutputStream())) + private val inStream = BufferedReader(InputStreamReader(clientSocket.getInputStream())) + + fun stop() { + outStream.write("STOP") + outStream.flush() + } + + fun sendData(msg: String) { + outStream.write("DATA") + + val size = msg.length + outStream.write(size.toString().padStart(16)) + + outStream.write(msg) + outStream.flush() + } + + fun receiveMessage(): String { + val length = inStream.readLine().toInt() + val buffer = CharArray(length) + inStream.read(buffer) + return String(buffer) + } + + private fun stopConnection() { + inStream.close() + outStream.close() + clientSocket.close() + } + + fun stopServer() { + stop() + stopConnection() + } + +} \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionRequestSerializer.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionRequestSerializer.kt index c554214236..33d2251107 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionRequestSerializer.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionRequestSerializer.kt @@ -17,10 +17,10 @@ object ExecutionRequestSerializer { data class ExecutionRequest( val functionName: String, + val functionModule: String, val imports: List, val syspaths: List, val argumentsIds: List, val serializedMemory: String, - val coverageDB: String, val filepath: String, ) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionResultDeserializer.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionResultDeserializer.kt index c6ec72d8c7..3ffdbe7bce 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionResultDeserializer.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionResultDeserializer.kt @@ -1,5 +1,6 @@ package org.utbot.python.evaluation.serialiation +import com.squareup.moshi.JsonEncodingException import com.squareup.moshi.Moshi import com.squareup.moshi.adapters.PolymorphicJsonAdapterFactory import com.squareup.moshi.kotlin.reflect.KotlinJsonAdapterFactory @@ -25,7 +26,12 @@ object ExecutionResultDeserializer { private val jsonAdapterMemoryDump = moshi.adapter(MemoryDump::class.java) fun parseExecutionResult(content: String): PythonExecutionResult? { - return jsonAdapter.fromJson(content) + try { + return jsonAdapter.fromJson(content) ?: error("Parsing error with: $content") + } catch (_: JsonEncodingException) { + println(content) + } + return null } fun parseMemoryDump(content: String): MemoryDump? { diff --git a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/mypy/RunMypy.kt b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/mypy/RunMypy.kt index 92f5d9cff9..49e677789d 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/mypy/RunMypy.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/mypy/RunMypy.kt @@ -80,7 +80,6 @@ fun setConfigFile(directoriesForSysPath: Set): File { show_absolute_path = True cache_fine_grained = True check_untyped_defs = True - implicit_optional = True strict_optional = False disable_error_code = assignment,union-attr """.trimIndent() From 18a7768a582a00da6e20f945b50141f3229f7cd3 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Wed, 22 Feb 2023 13:00:51 +0300 Subject: [PATCH 05/32] Update requirements.txt --- utbot-python/src/main/resources/requirements.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/utbot-python/src/main/resources/requirements.txt b/utbot-python/src/main/resources/requirements.txt index ffca14c4e0..bbd111cccb 100644 --- a/utbot-python/src/main/resources/requirements.txt +++ b/utbot-python/src/main/resources/requirements.txt @@ -1,6 +1,4 @@ mypy==0.971 -astor -typeshed-client coverage -utbot-executor==0.2.4 +utbot-executor==1.0.21 utbot-mypy-runner==0.2.4 From 08dd9452163ff96f55e158e043be5bc24566f780 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Wed, 15 Feb 2023 15:35:20 +0300 Subject: [PATCH 06/32] Work on sockets --- .../kotlin/org/utbot/python/PythonEngine.kt | 4 +- .../python/code/PythonObjectDeserializer.kt | 135 ------ .../python/evaluation/CodeEvaluationApi.kt | 9 +- .../python/evaluation/ExecutionClient.kt | 52 +++ .../evaluation/PythonCodeExecutorImpl.kt | 17 +- .../evaluation/PythonCodeSocketExecutor.kt | 30 ++ .../ExecutionResultDeserializer.kt | 8 +- .../serialiation/PythonObjectParser.kt | 439 ++++++++++++++++++ .../python/framework/api/python/PythonTree.kt | 57 ++- .../api/python/util/PythonIdUtils.kt | 10 +- .../org/utbot/python/fuzzing/PythonApi.kt | 21 +- .../fuzzing/provider/ReduceValueProvider.kt | 6 +- 12 files changed, 609 insertions(+), 179 deletions(-) delete mode 100644 utbot-python/src/main/kotlin/org/utbot/python/code/PythonObjectDeserializer.kt create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/evaluation/ExecutionClient.kt create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt rename utbot-python/src/main/kotlin/org/utbot/python/evaluation/{ => serialiation}/ExecutionResultDeserializer.kt (85%) create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/PythonObjectParser.kt diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index 3d5c9c8a3a..cbbb571624 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -16,13 +16,13 @@ import org.utbot.fuzzer.UtFuzzedExecution import org.utbot.fuzzing.Control import org.utbot.fuzzing.fuzz import org.utbot.fuzzing.utils.Trie -import org.utbot.python.code.MemoryDump -import org.utbot.python.code.toPythonTree import org.utbot.python.evaluation.PythonCodeExecutor import org.utbot.python.evaluation.PythonCodeExecutorImpl import org.utbot.python.evaluation.PythonEvaluationError import org.utbot.python.evaluation.PythonEvaluationSuccess import org.utbot.python.evaluation.PythonEvaluationTimeout +import org.utbot.python.evaluation.serialiation.MemoryDump +import org.utbot.python.evaluation.serialiation.toPythonTree import org.utbot.python.framework.api.python.PythonTreeModel import org.utbot.python.fuzzing.PythonFeedback import org.utbot.python.fuzzing.PythonFuzzedConcreteValue diff --git a/utbot-python/src/main/kotlin/org/utbot/python/code/PythonObjectDeserializer.kt b/utbot-python/src/main/kotlin/org/utbot/python/code/PythonObjectDeserializer.kt deleted file mode 100644 index d840892a9c..0000000000 --- a/utbot-python/src/main/kotlin/org/utbot/python/code/PythonObjectDeserializer.kt +++ /dev/null @@ -1,135 +0,0 @@ -package org.utbot.python.code - -import com.squareup.moshi.Moshi -import com.squareup.moshi.adapters.PolymorphicJsonAdapterFactory -import com.squareup.moshi.kotlin.reflect.KotlinJsonAdapterFactory -import org.utbot.python.framework.api.python.PythonClassId -import org.utbot.python.framework.api.python.PythonTree - -object PythonObjectDeserializer { - private val moshi = Moshi.Builder() - .add( - PolymorphicJsonAdapterFactory.of(MemoryObject::class.java, "strategy") - .withSubtype(ReprMemoryObject::class.java, "repr") - .withSubtype(ListMemoryObject::class.java, "list") - .withSubtype(DictMemoryObject::class.java, "dict") - .withSubtype(ReduceMemoryObject::class.java, "reduce") - ) - .addLast(KotlinJsonAdapterFactory()) - .build() - - private val jsonAdapter = moshi.adapter(MemoryDump::class.java) - - fun parseDumpedObjects(jsonWithDump: String): MemoryDump { - return jsonAdapter.fromJson(jsonWithDump) ?: error("Couldn't parse json dump") - } -} - -class MemoryDump( - private val objects: Map -) { - fun getById(id: String): MemoryObject { - return objects[id]!! - } - - fun getByValue(value: MemoryObject): String { - return objects.filter { it.value == value }.keys.first() - } -} - -sealed class MemoryObject( - val kind: String, - val comparable: Boolean, -) - -class ReprMemoryObject( - kind: String, - comparable: Boolean, - val value: String, -): MemoryObject(kind, comparable) - -class ListMemoryObject( - kind: String, - comparable: Boolean, - val items: List, -): MemoryObject(kind, comparable) - -class DictMemoryObject( - kind: String, - comparable: Boolean, - val items: Map, -): MemoryObject(kind, comparable) - -class ReduceMemoryObject( - kind: String, - comparable: Boolean, - val constructor: String, - val args: String, - val state: String, - val listitems: String, - val dictitems: String -): MemoryObject(kind, comparable) - -fun MemoryObject.toPythonTree(memoryDump: MemoryDump): PythonTree.PythonTreeNode { - val obj = when(this) { - is ReprMemoryObject -> { - PythonTree.PrimitiveNode(PythonClassId(this.kind), this.value) - } - is DictMemoryObject -> { - PythonTree.DictNode( - items.entries.associate { - memoryDump.getById(it.key).toPythonTree(memoryDump) to memoryDump.getById(it.value).toPythonTree(memoryDump) - }.toMutableMap() - ) - } - is ListMemoryObject -> { - val elementsMap = items.withIndex().associate { - it.index to memoryDump.getById(it.value).toPythonTree(memoryDump) - }.toMutableMap() - when (this.kind) { - "builtins.tuple" -> { - PythonTree.TupleNode(elementsMap) - } - "builtins.set" -> { - PythonTree.SetNode(elementsMap.values.toMutableSet()) - } - else -> { - PythonTree.ListNode(elementsMap) - } - } - } - is ReduceMemoryObject -> { - val stateObjs = memoryDump.getById(state) as DictMemoryObject - val arguments = memoryDump.getById(args) as ListMemoryObject - val listitemsObjs = memoryDump.getById(listitems) as ListMemoryObject - val dictitemsObjs = memoryDump.getById(dictitems) as DictMemoryObject - PythonTree.ReduceNode( - memoryDump.getByValue(this).toLong(), - PythonClassId(this.kind), - PythonClassId(this.constructor), - arguments.items.map { memoryDump.getById(it).toPythonTree(memoryDump) }, - stateObjs.items.entries.associate { - (memoryDump.getById(it.key).toPythonTree(memoryDump) as PythonTree.PrimitiveNode).repr.drop(1).dropLast(1) to - memoryDump.getById(it.value).toPythonTree(memoryDump) - }.toMutableMap(), - listitemsObjs.items.map { memoryDump.getById(it).toPythonTree(memoryDump) }, - dictitemsObjs.items.entries.associate { - memoryDump.getById(it.key).toPythonTree(memoryDump) to - memoryDump.getById(it.value).toPythonTree(memoryDump) - }, - ) - } - } - obj.comparable = this.comparable - return obj -} - -fun main() { - val dump = """ - {"objects": {"140598295296000": {"strategy": "list", "kind": "builtins.list", "comparable": false, "items": ["140598416769264", "140598416769296", "140598416769328", "140598295298816", "140598427938368", "140598374175968"]}, "140598416769264": {"strategy": "repr", "kind": "builtins.int", "comparable": true, "value": "1"}, "140598416769296": {"strategy": "repr", "kind": "builtins.int", "comparable": true, "value": "2"}, "140598416769328": {"strategy": "repr", "kind": "builtins.int", "comparable": true, "value": "3"}, "140598295298816": {"strategy": "dict", "kind": "builtins.dict", "comparable": true, "items": {"140598416769264": "140598416769264"}}, "140598427938368": {"strategy": "repr", "kind": "types.NoneType", "comparable": true, "value": "None"}, "140598372733504": {"strategy": "list", "kind": "builtins.tuple", "comparable": true, "items": ["94206620040656", "140598427931232", "140598427938368"]}, "94206620040656": {"strategy": "repr", "kind": "builtins.type", "comparable": true, "value": "deep_serialization.example.B"}, "140598427931232": {"strategy": "repr", "kind": "builtins.type", "comparable": true, "value": "builtins.object"}, "140598374175968": {"strategy": "reduce", "kind": "deep_serialization.example.B", "comparable": false, "constructor": "copyreg._reconstructor", "args": "140598372733504", "state": "140598295238656", "listitems": "140598295487936", "dictitems": "140598295486336"}, "140598295238656": {"strategy": "dict", "kind": "builtins.dict", "comparable": true, "items": {"140598386103280": "140598416769264", "140598395465264": "140598416769296", "140598372712880": "140598416769328", "140598415768816": "140598374177584"}}, "140598386103280": {"strategy": "repr", "kind": "builtins.str", "comparable": true, "value": "\'b1\'"}, "140598395465264": {"strategy": "repr", "kind": "builtins.str", "comparable": true, "value": "\'b2\'"}, "140598372712880": {"strategy": "repr", "kind": "builtins.str", "comparable": true, "value": "\'b3\'"}, "140598415768816": {"strategy": "repr", "kind": "builtins.str", "comparable": true, "value": "\'time\'"}, "140598374184560": {"strategy": "list", "kind": "builtins.tuple", "comparable": true, "items": ["140598295162016"]}, "140598295162016": {"strategy": "repr", "kind": "builtins.bytes", "comparable": true, "value": "b\'\\\\x07\\\\xe7\\\\x01\\\\x13\\\\x11\\\\x01\\\\x1c\\\\x0e\\\\x921\'"}, "140598374177584": {"strategy": "reduce", "kind": "datetime.datetime", "comparable": true, "constructor": "datetime.datetime", "args": "140598374184560", "state": "140598295312768", "listitems": "140598295488000", "dictitems": "140598295485760"}, "140598295312768": {"strategy": "dict", "kind": "builtins.dict", "comparable": true, "items": {}}, "140598295488000": {"strategy": "list", "kind": "builtins.list", "comparable": true, "items": []}, "140598295485760": {"strategy": "dict", "kind": "builtins.dict", "comparable": true, "items": {}}, "140598295487936": {"strategy": "list", "kind": "builtins.list", "comparable": true, "items": []}, "140598295486336": {"strategy": "dict", "kind": "builtins.dict", "comparable": true, "items": {}}}} -""".trimIndent() - - val load = PythonObjectDeserializer.parseDumpedObjects(dump) - load.getById("140598295296000").toPythonTree(load) -} - diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt index ecb90c4096..afb1db6f4d 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt @@ -4,19 +4,20 @@ import org.utbot.framework.plugin.api.Coverage import org.utbot.fuzzer.FuzzedValue import org.utbot.python.FunctionArguments import org.utbot.python.PythonMethod -import org.utbot.python.code.MemoryDump +import org.utbot.python.evaluation.serialiation.MemoryDump interface PythonCodeExecutor { val method: PythonMethod val methodArguments: FunctionArguments - val fuzzedValues: List val moduleToImport: String - val additionalModulesToImport: Set val pythonPath: String val syspathDirectories: Set val executionTimeout: Long - fun run(): PythonEvaluationResult + fun run( + fuzzedValues: List, + additionalModulesToImport: Set + ): PythonEvaluationResult } sealed class PythonEvaluationResult diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/ExecutionClient.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/ExecutionClient.kt new file mode 100644 index 0000000000..4528906ca0 --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/ExecutionClient.kt @@ -0,0 +1,52 @@ +package org.utbot.python.evaluation + +import java.io.BufferedReader +import java.io.BufferedWriter +import java.io.InputStreamReader +import java.io.OutputStreamWriter +import java.io.PrintWriter +import java.net.Socket +import java.nio.CharBuffer + +class ExecutionClient( + hostname: String, + port: Int +) { + private val clientSocket: Socket = Socket(hostname, port) + private val outStream = BufferedWriter(OutputStreamWriter(clientSocket.getOutputStream())) + private val inStream = BufferedReader(InputStreamReader(clientSocket.getInputStream())) + + init { + runServer() + } + + private fun runServer() { + TODO() + } + + fun sendMessage(msg: String) { + outStream.write(msg) + outStream.flush() + outStream.write("END") + outStream.flush() + } + + fun receiveMessage(): String? { + return inStream.readLine() + } + + fun stopConnection() { + inStream.close() + outStream.close() + clientSocket.close() + } +} + +fun main() { + val client = ExecutionClient("localhost", 12011) + client.sendMessage("Firstfjlskdjf") + client.sendMessage("Second") + client.sendMessage("{123: 123}") + client.sendMessage("STOP") + client.stopConnection() +} \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeExecutorImpl.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeExecutorImpl.kt index 8fff56e926..b9de31646c 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeExecutorImpl.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeExecutorImpl.kt @@ -6,6 +6,9 @@ import org.utbot.fuzzer.FuzzedValue import org.utbot.python.FunctionArguments import org.utbot.python.PythonMethod import org.utbot.python.code.PythonCodeGenerator +import org.utbot.python.evaluation.serialiation.ExecutionResultDeserializer +import org.utbot.python.evaluation.serialiation.FailExecution +import org.utbot.python.evaluation.serialiation.SuccessExecution import org.utbot.python.framework.api.python.util.pythonAnyClassId import org.utbot.python.utils.TemporaryFileManager import org.utbot.python.utils.getResult @@ -20,19 +23,23 @@ data class EvaluationFiles( class PythonCodeExecutorImpl( override val method: PythonMethod, override val methodArguments: FunctionArguments, - override val fuzzedValues: List, override val moduleToImport: String, - override val additionalModulesToImport: Set, override val pythonPath: String, override val syspathDirectories: Set, override val executionTimeout: Long, ) : PythonCodeExecutor { - override fun run(): PythonEvaluationResult { - val evaluationFiles = generateExecutionCode() + + override fun run( + fuzzedValues: List, + additionalModulesToImport: Set, + ): PythonEvaluationResult { + val evaluationFiles = generateExecutionCode(additionalModulesToImport) return getEvaluationResult(evaluationFiles) } - private fun generateExecutionCode(): EvaluationFiles { + private fun generateExecutionCode( + additionalModulesToImport: Set, + ): EvaluationFiles { val fileForOutput = TemporaryFileManager.assignTemporaryFile( tag = "out_" + method.name + ".py", addToCleaner = false diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt new file mode 100644 index 0000000000..5f7137e074 --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt @@ -0,0 +1,30 @@ +package org.utbot.python.evaluation + +import org.utbot.fuzzer.FuzzedValue +import org.utbot.python.FunctionArguments +import org.utbot.python.PythonMethod + + +data class ExecutionDescription( + val functionName: String, + val imports: List, + val argumentsIds: List, + val serializedMemory: String +) + + +class PythonCodeSocketExecutor( + override val method: PythonMethod, + override val methodArguments: FunctionArguments, + override val moduleToImport: String, + override val pythonPath: String, + override val syspathDirectories: Set, + override val executionTimeout: Long, +) : PythonCodeExecutor { + val executionClient = ExecutionClient("localhost", 12011) // TODO: create port manager + override fun run( + fuzzedValues: List, + additionalModulesToImport: Set): PythonEvaluationResult { +// val input = + } +} \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/ExecutionResultDeserializer.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionResultDeserializer.kt similarity index 85% rename from utbot-python/src/main/kotlin/org/utbot/python/evaluation/ExecutionResultDeserializer.kt rename to utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionResultDeserializer.kt index f230f7d314..c6ec72d8c7 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/ExecutionResultDeserializer.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionResultDeserializer.kt @@ -1,14 +1,8 @@ -package org.utbot.python.evaluation +package org.utbot.python.evaluation.serialiation import com.squareup.moshi.Moshi import com.squareup.moshi.adapters.PolymorphicJsonAdapterFactory import com.squareup.moshi.kotlin.reflect.KotlinJsonAdapterFactory -import org.utbot.python.code.DictMemoryObject -import org.utbot.python.code.ListMemoryObject -import org.utbot.python.code.MemoryDump -import org.utbot.python.code.MemoryObject -import org.utbot.python.code.ReduceMemoryObject -import org.utbot.python.code.ReprMemoryObject object ExecutionResultDeserializer { private val moshi = Moshi.Builder() diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/PythonObjectParser.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/PythonObjectParser.kt new file mode 100644 index 0000000000..1bc7b376ac --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/PythonObjectParser.kt @@ -0,0 +1,439 @@ +package org.utbot.python.evaluation.serialiation + +import com.squareup.moshi.Moshi +import com.squareup.moshi.adapters.PolymorphicJsonAdapterFactory +import com.squareup.moshi.kotlin.reflect.KotlinJsonAdapterFactory +import org.utbot.python.framework.api.python.PythonClassId +import org.utbot.python.framework.api.python.PythonTree +import org.utbot.python.framework.api.python.util.pythonStrClassId + +object PythonObjectParser { + private val moshi = Moshi.Builder() + .add( + PolymorphicJsonAdapterFactory.of(MemoryObject::class.java, "strategy") + .withSubtype(ReprMemoryObject::class.java, "repr") + .withSubtype(ListMemoryObject::class.java, "list") + .withSubtype(DictMemoryObject::class.java, "dict") + .withSubtype(ReduceMemoryObject::class.java, "reduce") + ) + .addLast(KotlinJsonAdapterFactory()) + .build() + + private val jsonAdapter = moshi.adapter(MemoryDump::class.java) + + fun parseDumpedObjects(jsonWithDump: String): MemoryDump { + return jsonAdapter.fromJson(jsonWithDump) ?: error("Couldn't parse json dump") + } + + fun serializeMemory(memory: MemoryDump): String { + return jsonAdapter.toJson(memory) ?: error("Couldn't serialize dump to json") + } +} + +class MemoryDump( + private val objects: MutableMap +) { + fun getById(id: String): MemoryObject { + return objects[id]!! + } + + fun getByValue(value: MemoryObject): String { + return objects.filter { it.value == value }.keys.first() + } + + fun addObject(value: MemoryObject) { + objects[value.id] = value + } +} + +sealed class MemoryObject( + val id: String, + val kind: String, + val comparable: Boolean, +) + +class ReprMemoryObject( + id: String, + kind: String, + comparable: Boolean, + val value: String, +): MemoryObject(id, kind, comparable) + +class ListMemoryObject( + id: String, + kind: String, + comparable: Boolean, + val items: List, +): MemoryObject(id, kind, comparable) + +class DictMemoryObject( + id: String, + kind: String, + comparable: Boolean, + val items: Map, +): MemoryObject(id, kind, comparable) + +class ReduceMemoryObject( + id: String, + kind: String, + comparable: Boolean, + val constructor: String, + val args: String, + val state: String, + val listitems: String, + val dictitems: String +): MemoryObject(id, kind, comparable) + +fun PythonTree.PythonTreeNode.toMemoryObject(memoryDump: MemoryDump): String { + val obj = when(this) { + is PythonTree.PrimitiveNode -> { + ReprMemoryObject(this.id.toString(), this.type.name, this.comparable, this.repr) + } + is PythonTree.ListNode -> { + val items = this.items.entries + .sortedBy { it.key } + .map { it.value.toMemoryObject(memoryDump) } + ListMemoryObject(this.id.toString(), this.type.name, this.comparable, items) + } + is PythonTree.TupleNode -> { + val items = this.items.entries + .sortedBy { it.key } + .map { it.value.toMemoryObject(memoryDump) } + ListMemoryObject(this.id.toString(), this.type.name, this.comparable, items) + } + is PythonTree.SetNode -> { + val items = this.items.map { it.toMemoryObject(memoryDump) } + ListMemoryObject(this.id.toString(), this.type.name, this.comparable, items) + } + is PythonTree.DictNode -> { + val items = this.items.entries + .associate { it.key.toMemoryObject(memoryDump) to it.value.toMemoryObject(memoryDump) } + DictMemoryObject(this.id.toString(), this.type.name, this.comparable, items) + } + is PythonTree.ReduceNode -> { + val stateObjId = PythonTree.DictNode(this.state.entries.associate { PythonTree.PrimitiveNode(pythonStrClassId, it.key) to it.value }.toMutableMap()) + val argsIds = PythonTree.ListNode(this.args.withIndex().associate { it.index to it.value }.toMutableMap()) + val listItemsIds = PythonTree.ListNode(this.listitems.withIndex().associate { it.index to it.value }.toMutableMap()) + val dictItemsIds = PythonTree.DictNode(this.dictitems.toMutableMap()) + ReduceMemoryObject( + this.id.toString(), + this.type.name, + this.comparable, + this.constructor.name, + argsIds.toMemoryObject(memoryDump), + stateObjId.toMemoryObject(memoryDump), + listItemsIds.toMemoryObject(memoryDump), + dictItemsIds.toMemoryObject(memoryDump), + ) + } + else -> { + error("Invalid PythonTree.PythonTreeNode $this") + } + } + memoryDump.addObject(obj) + return obj.id +} + +fun MemoryObject.toPythonTree(memoryDump: MemoryDump): PythonTree.PythonTreeNode { + val obj = when(this) { + is ReprMemoryObject -> { + PythonTree.PrimitiveNode( + this.id.toLong(), + PythonClassId(this.kind), + this.value + ) + } + is DictMemoryObject -> { + PythonTree.DictNode( + this.id.toLong(), + items.entries.associate { + memoryDump.getById(it.key).toPythonTree(memoryDump) to memoryDump.getById(it.value).toPythonTree(memoryDump) + }.toMutableMap() + ) + } + is ListMemoryObject -> { + val elementsMap = items.withIndex().associate { + it.index to memoryDump.getById(it.value).toPythonTree(memoryDump) + }.toMutableMap() + when (this.kind) { + "builtins.tuple" -> { + PythonTree.TupleNode(this.id.toLong(), elementsMap) + } + "builtins.set" -> { + PythonTree.SetNode(this.id.toLong(), elementsMap.values.toMutableSet()) + } + else -> { + PythonTree.ListNode(this.id.toLong(), elementsMap) + } + } + } + is ReduceMemoryObject -> { + val stateObjs = memoryDump.getById(state) as DictMemoryObject + val arguments = memoryDump.getById(args) as ListMemoryObject + val listitemsObjs = memoryDump.getById(listitems) as ListMemoryObject + val dictitemsObjs = memoryDump.getById(dictitems) as DictMemoryObject + PythonTree.ReduceNode( + this.id.toLong(), + PythonClassId(this.kind), + PythonClassId(this.constructor), + arguments.items.map { memoryDump.getById(it).toPythonTree(memoryDump) }, + stateObjs.items.entries.associate { + (memoryDump.getById(it.key).toPythonTree(memoryDump) as PythonTree.PrimitiveNode).repr.drop(1).dropLast(1) to + memoryDump.getById(it.value).toPythonTree(memoryDump) + }.toMutableMap(), + listitemsObjs.items.map { memoryDump.getById(it).toPythonTree(memoryDump) }, + dictitemsObjs.items.entries.associate { + memoryDump.getById(it.key).toPythonTree(memoryDump) to + memoryDump.getById(it.value).toPythonTree(memoryDump) + }, + ) + } + } + obj.comparable = this.comparable + return obj +} + +fun serializeObjects(objs: List): Pair, String> { + val memoryDump = MemoryDump(emptyMap().toMutableMap()) + val ids = objs.map { it.toMemoryObject(memoryDump) } + return Pair(ids, PythonObjectParser.serializeMemory(memoryDump)) +} + +fun main() { + val dump = """ + { + "objects": { + "140239390887040": { + "strategy": "list", + "id": "140239390887040", + "kind": "builtins.list", + "comparable": false, + "items": [ + "140239394832624", + "140239394832656", + "140239392627184", + "140239394012784", + "140239392795520", + "140239406001728", + "140239392839840", + "140239390894848" + ] + }, + "140239394832624": { + "strategy": "repr", + "id": "140239394832624", + "kind": "builtins.int", + "comparable": true, + "value": "1" + }, + "140239394832656": { + "strategy": "repr", + "id": "140239394832656", + "kind": "builtins.int", + "comparable": true, + "value": "2" + }, + "140239392627184": { + "strategy": "repr", + "id": "140239392627184", + "kind": "builtins.float", + "comparable": true, + "value": "float('inf')" + }, + "140239394012784": { + "strategy": "repr", + "id": "140239394012784", + "kind": "builtins.str", + "comparable": true, + "value": "'abc'" + }, + "140239392795520": { + "strategy": "dict", + "id": "140239392795520", + "kind": "builtins.dict", + "comparable": true, + "items": { + "140239394832624": "140239394832624" + } + }, + "140239406001728": { + "strategy": "repr", + "id": "140239406001728", + "kind": "types.NoneType", + "comparable": true, + "value": "None" + }, + "140239391427840": { + "strategy": "list", + "id": "140239391427840", + "kind": "builtins.tuple", + "comparable": true, + "items": [ + "94246249326576", + "140239405994592", + "140239406001728" + ] + }, + "94246249326576": { + "strategy": "repr", + "id": "94246249326576", + "kind": "builtins.type", + "comparable": true, + "value": "deep_serialization.example.B" + }, + "140239405994592": { + "strategy": "repr", + "id": "140239405994592", + "kind": "builtins.type", + "comparable": true, + "value": "builtins.object" + }, + "140239392839840": { + "strategy": "reduce", + "id": "140239392839840", + "kind": "deep_serialization.example.B", + "comparable": false, + "constructor": "copyreg._reconstructor", + "args": "140239391427840", + "state": "140239392795712", + "listitems": "140239391672832", + "dictitems": "140239391673280" + }, + "140239392795712": { + "strategy": "dict", + "id": "140239392795712", + "kind": "builtins.dict", + "comparable": true, + "items": { + "140239392797168": "140239394832624", + "140239392797232": "140239394832656", + "140239392797296": "140239394832688", + "140239393831920": "140239392849760" + } + }, + "140239392797168": { + "strategy": "repr", + "id": "140239392797168", + "kind": "builtins.str", + "comparable": true, + "value": "'b1'" + }, + "140239392797232": { + "strategy": "repr", + "id": "140239392797232", + "kind": "builtins.str", + "comparable": true, + "value": "'b2'" + }, + "140239392797296": { + "strategy": "repr", + "id": "140239392797296", + "kind": "builtins.str", + "comparable": true, + "value": "'b3'" + }, + "140239394832688": { + "strategy": "repr", + "id": "140239394832688", + "kind": "builtins.int", + "comparable": true, + "value": "3" + }, + "140239393831920": { + "strategy": "repr", + "id": "140239393831920", + "kind": "builtins.str", + "comparable": true, + "value": "'time'" + }, + "140239394159488": { + "strategy": "list", + "id": "140239394159488", + "kind": "builtins.tuple", + "comparable": true, + "items": [ + "140239391514208" + ] + }, + "140239391514208": { + "strategy": "repr", + "id": "140239391514208", + "kind": "builtins.bytes", + "comparable": true, + "value": "b'\\x07\\xe7\\x02\\r\\x0c(3\\x06\\x1eA'" + }, + "140239392849760": { + "strategy": "reduce", + "id": "140239392849760", + "kind": "datetime.datetime", + "comparable": true, + "constructor": "datetime.datetime", + "args": "140239394159488", + "state": "140239391671232", + "listitems": "140239391671872", + "dictitems": "140239391672448" + }, + "140239391671232": { + "strategy": "dict", + "id": "140239391671232", + "kind": "builtins.dict", + "comparable": true, + "items": {} + }, + "140239391671872": { + "strategy": "list", + "id": "140239391671872", + "kind": "builtins.list", + "comparable": true, + "items": [] + }, + "140239391672448": { + "strategy": "dict", + "id": "140239391672448", + "kind": "builtins.dict", + "comparable": true, + "items": {} + }, + "140239391672832": { + "strategy": "list", + "id": "140239391672832", + "kind": "builtins.list", + "comparable": true, + "items": [] + }, + "140239391673280": { + "strategy": "dict", + "id": "140239391673280", + "kind": "builtins.dict", + "comparable": true, + "items": {} + }, + "140239390894848": { + "strategy": "list", + "id": "140239390894848", + "kind": "builtins.list", + "comparable": true, + "items": [ + "140239392797552" + ] + }, + "140239392797552": { + "strategy": "repr", + "id": "140239392797552", + "kind": "builtins.str", + "comparable": true, + "value": "'Alex'" + } + } + } +""".trimIndent() + + val load = PythonObjectParser.parseDumpedObjects(dump) + val obj = load.getById("140239390887040").toPythonTree(load) + val newMemory = MemoryDump(emptyMap().toMutableMap()) + val newId = obj.toMemoryObject(newMemory) + val newDump = PythonObjectParser.serializeMemory(newMemory) + val newLoad = PythonObjectParser.parseDumpedObjects(newDump) + val newObj = newLoad.getById(newId).toPythonTree(newLoad) +} + diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt index 3f793ae8f3..8f096e33b3 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt @@ -9,12 +9,18 @@ import org.utbot.python.newtyping.general.Type import org.utbot.python.newtyping.pythonTypeName import java.math.BigDecimal import java.math.BigInteger +import java.util.* +import java.util.concurrent.atomic.AtomicLong + object PythonTree { open class PythonTreeNode( + val id: Long, val type: PythonClassId, - var comparable: Boolean = true + var comparable: Boolean = true, ) { + constructor(type: PythonClassId, comparable: Boolean = true) : this(PythonIdGenerator.createId(), type, comparable) + open val children: List = emptyList() open fun typeEquals(other: Any?): Boolean { @@ -40,9 +46,11 @@ object PythonTree { } class PrimitiveNode( + id: Long, type: PythonClassId, val repr: String, - ) : PythonTreeNode(type) { + ) : PythonTreeNode(id, type) { + constructor(type: PythonClassId, repr: String) : this(PythonIdGenerator.getOrCreateIdForValue(repr), type, repr) override fun equals(other: Any?): Boolean { if (other !is PrimitiveNode) { return false @@ -58,8 +66,11 @@ object PythonTree { } class ListNode( + id: Long, val items: MutableMap - ) : PythonTreeNode(PythonClassId("builtins.list")) { + ) : PythonTreeNode(id, PythonClassId("builtins.list")) { + constructor(items: MutableMap) : this(PythonIdGenerator.createId(), items) + override val children: List get() = items.values.toList() @@ -86,8 +97,11 @@ object PythonTree { } class DictNode( + id: Long, val items: MutableMap - ) : PythonTreeNode(PythonClassId("builtins.dict")) { + ) : PythonTreeNode(id, PythonClassId("builtins.dict")) { + constructor(items: MutableMap) : this(PythonIdGenerator.createId(), items) + override val children: List get() = items.values + items.keys @@ -116,8 +130,11 @@ object PythonTree { } class SetNode( + id: Long, val items: MutableSet - ) : PythonTreeNode(PythonClassId("builtins.set")) { + ) : PythonTreeNode(id, PythonClassId("builtins.set")) { + constructor(items: MutableSet) : this(PythonIdGenerator.createId(), items) + override val children: List get() = items.toList() @@ -150,8 +167,11 @@ object PythonTree { } class TupleNode( + id: Long, val items: MutableMap - ) : PythonTreeNode(PythonClassId("builtins.tuple")) { + ) : PythonTreeNode(id, PythonClassId("builtins.tuple")) { + constructor(items: MutableMap) : this(PythonIdGenerator.createId(), items) + override val children: List get() = items.values.toList() @@ -180,20 +200,19 @@ object PythonTree { } class ReduceNode( - val id: Long, + id: Long, type: PythonClassId, val constructor: PythonClassId, val args: List, var state: MutableMap, var listitems: List, var dictitems: Map, - ) : PythonTreeNode(type) { + ) : PythonTreeNode(id, type) { constructor( - id: Long, type: PythonClassId, constructor: PythonClassId, args: List, - ) : this(id, type, constructor, args, emptyMap().toMutableMap(), emptyList(), emptyMap()) + ) : this(PythonIdGenerator.createId(), type, constructor, args, emptyMap().toMutableMap(), emptyList(), emptyMap()) override val children: List get() = args + state.values + listitems + dictitems.values + dictitems.keys + PythonTreeNode(constructor) @@ -310,4 +329,20 @@ object PythonTree { else -> null } } -} \ No newline at end of file +} + +object PythonIdGenerator { + private const val lower_bound: Long = 1500_000_000 + + private val lastId: AtomicLong = AtomicLong(lower_bound) + private val cache: IdentityHashMap = IdentityHashMap() + + fun getOrCreateIdForValue(value: Any): Long { + return cache.getOrPut(value) { createId() } + } + + fun createId(): Long { + return lastId.incrementAndGet() + } + +} diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonIdUtils.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonIdUtils.kt index 6ee9e07661..9a15f99dd4 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonIdUtils.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonIdUtils.kt @@ -16,11 +16,11 @@ val pythonIntClassId = PythonClassId("builtins.int") val pythonFloatClassId = PythonClassId("builtins.float") val pythonComplexClassId = PythonClassId("builtins.complex") val pythonStrClassId = PythonClassId("builtins.str") -val pythonBoolClassId = PythonBoolModel.classId +val pythonBoolClassId = PythonClassId("builtins.bool") val pythonRangeClassId = PythonClassId("builtins.range") -val pythonListClassId = PythonListModel.classId -val pythonTupleClassId = PythonTupleModel.classId -val pythonDictClassId = PythonDictModel.classId -val pythonSetClassId = PythonSetModel.classId +val pythonListClassId = PythonClassId("builtins.list") +val pythonTupleClassId = PythonClassId("builtins.tuple") +val pythonDictClassId = PythonClassId("builtins.dict") +val pythonSetClassId = PythonClassId("builtins.set") val pythonBytearrayClassId = PythonClassId("builtins.bytearray") val pythonBytesClassId = PythonClassId("builtins.bytes") diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt index e070410ca8..c8291d7ac9 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt @@ -4,6 +4,7 @@ import mu.KotlinLogging import org.utbot.framework.plugin.api.Instruction import org.utbot.fuzzer.FuzzedContext import org.utbot.fuzzer.IdGenerator +import org.utbot.fuzzer.IdentityPreservingIdGenerator import org.utbot.fuzzing.Configuration import org.utbot.fuzzing.Control import org.utbot.fuzzing.Description @@ -35,6 +36,8 @@ import org.utbot.python.newtyping.PythonSubtypeChecker import org.utbot.python.newtyping.PythonTypeStorage import org.utbot.python.newtyping.general.Type import org.utbot.python.newtyping.pythonTypeRepresentation +import java.util.* +import java.util.concurrent.atomic.AtomicLong private val logger = KotlinLogging.logger {} @@ -77,7 +80,7 @@ fun pythonDefaultValueProviders(idGenerator: IdGenerator) = listOf( UnionValueProvider, BytesValueProvider, BytearrayValueProvider, - ReduceValueProvider(idGenerator), + ReduceValueProvider, ConstantValueProvider, ) @@ -85,6 +88,7 @@ class PythonFuzzing( private val pythonTypeStorage: PythonTypeStorage, val execute: suspend (description: PythonMethodDescription, values: List) -> PythonFeedback, ) : Fuzzing { + private fun generateDefault(description: PythonMethodDescription, type: Type, idGenerator: IdGenerator): Sequence> { var providers = emptyList>().asSequence() pythonDefaultValueProviders(idGenerator).asSequence().forEach { provider -> @@ -136,12 +140,19 @@ class PythonFuzzing( } } -class PythonIdGenerator : IdGenerator { - private var _id: Long = 0 +class PythonIdGenerator(lowerBound: Long = DEFAULT_LOWER_BOUND) : IdentityPreservingIdGenerator { + private val lastId: AtomicLong = AtomicLong(lowerBound) + private val cache: IdentityHashMap = IdentityHashMap() + + override fun getOrCreateIdForValue(value: Any): Long { + return cache.getOrPut(value) { createId() } + } override fun createId(): Long { - _id += 1 - return _id + return lastId.incrementAndGet() } + companion object { + const val DEFAULT_LOWER_BOUND: Long = 1500_000_000 + } } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ReduceValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ReduceValueProvider.kt index f91f43075c..e8995bcad0 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ReduceValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ReduceValueProvider.kt @@ -1,6 +1,5 @@ package org.utbot.python.fuzzing.provider -import org.utbot.fuzzer.IdGenerator import org.utbot.fuzzing.Routine import org.utbot.fuzzing.Seed import org.utbot.fuzzing.ValueProvider @@ -12,9 +11,7 @@ import org.utbot.python.newtyping.* import org.utbot.python.newtyping.general.FunctionType import org.utbot.python.newtyping.general.Type -class ReduceValueProvider( - private val idGenerator: IdGenerator -) : ValueProvider { +object ReduceValueProvider : ValueProvider { private val unsupportedTypes = listOf( "builtins.list", "builtins.set", @@ -72,7 +69,6 @@ class ReduceValueProvider( construct = Routine.Create(nonSelfArgs) { v -> PythonFuzzedValue( PythonTree.ReduceNode( - idGenerator.createId(), PythonClassId(type.pythonTypeName()), PythonClassId(type.pythonTypeName()), v.map { it.tree }, From 40c1454b12154f91f2dff49c5c76d84a130a9f16 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Wed, 15 Feb 2023 15:35:28 +0300 Subject: [PATCH 07/32] Work on sockets --- .../org/utbot/python/evaluation/PythonCodeSocketExecutor.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt index 5f7137e074..352afc936f 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt @@ -26,5 +26,6 @@ class PythonCodeSocketExecutor( fuzzedValues: List, additionalModulesToImport: Set): PythonEvaluationResult { // val input = + TODO() } } \ No newline at end of file From 6088f9f1b74110b22d68a8a009b99e5492149cb4 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Mon, 20 Feb 2023 13:46:07 +0300 Subject: [PATCH 08/32] Added new execution version with sockets --- .../kotlin/org/utbot/python/PythonEngine.kt | 47 ++++---- .../utbot/python/PythonTestCaseGenerator.kt | 2 +- .../kotlin/org/utbot/python/UTPythonAPI.kt | 8 +- .../python/evaluation/CodeEvaluationApi.kt | 4 +- .../python/evaluation/ExecutionClient.kt | 33 ++++-- .../evaluation/PythonCodeExecutorImpl.kt | 13 ++- .../evaluation/PythonCodeSocketExecutor.kt | 103 +++++++++++++++--- .../ExecutionRequestSerializer.kt | 26 +++++ 8 files changed, 172 insertions(+), 64 deletions(-) create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionRequestSerializer.kt diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index cbbb571624..c290ccf3bc 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -11,13 +11,12 @@ import org.utbot.framework.plugin.api.UtExecutionResult import org.utbot.framework.plugin.api.UtExecutionSuccess import org.utbot.framework.plugin.api.UtExplicitlyThrownException import org.utbot.framework.plugin.api.UtModel -import org.utbot.fuzzer.FuzzedValue import org.utbot.fuzzer.UtFuzzedExecution import org.utbot.fuzzing.Control import org.utbot.fuzzing.fuzz import org.utbot.fuzzing.utils.Trie import org.utbot.python.evaluation.PythonCodeExecutor -import org.utbot.python.evaluation.PythonCodeExecutorImpl +import org.utbot.python.evaluation.PythonCodeSocketExecutor import org.utbot.python.evaluation.PythonEvaluationError import org.utbot.python.evaluation.PythonEvaluationSuccess import org.utbot.python.evaluation.PythonEvaluationTimeout @@ -146,26 +145,10 @@ class PythonEngine( return ValidExecution(utFuzzedExecution) } - private fun constructEvaluationInput(arguments: List, additionalModules: List): PythonCodeExecutor { - val argumentValues = arguments.map { PythonTreeModel(it.tree, it.tree.type) } - - val (thisObject, modelList) = - if (methodUnderTest.hasThisArgument) - Pair(argumentValues[0], argumentValues.drop(1)) - else - Pair(null, argumentValues) - - val argumentModules = argumentValues - .flatMap { it.allContainingClassIds } - .map { it.moduleName } - val localAdditionalModules = (additionalModules + argumentModules).toSet() - - return PythonCodeExecutorImpl( + private fun constructEvaluationInput(): PythonCodeExecutor { + return PythonCodeSocketExecutor( methodUnderTest, - FunctionArguments(thisObject, methodUnderTest.thisObjectName, modelList, methodUnderTest.argumentsNames), - argumentValues.map { FuzzedValue(it) }, moduleToImport, - localAdditionalModules, pythonPath, directoriesForSysPath, timeoutForRun, @@ -175,10 +158,23 @@ class PythonEngine( fun fuzzing(parameters: List, isCancelled: () -> Boolean, until: Long): Flow = flow { val additionalModules = parameters.flatMap { it.pythonModules() } val coveredLines = initialCoveredLines.toMutableSet() + val codeExecutor = constructEvaluationInput() suspend fun fuzzingResultHandler(description: PythonMethodDescription, arguments: List): PythonFeedback { - val codeExecutor = constructEvaluationInput(arguments, additionalModules) - return when (val evaluationResult = codeExecutor.run()) { + val argumentValues = arguments.map { PythonTreeModel(it.tree, it.tree.type) } + val argumentModules = argumentValues + .flatMap { it.allContainingClassIds } + .map { it.moduleName } + val localAdditionalModules = (additionalModules + argumentModules).toSet() + + val (thisObject, modelList) = + if (methodUnderTest.hasThisArgument) + Pair(argumentValues[0], argumentValues.drop(1)) + else + Pair(null, argumentValues) + val functionArguments = FunctionArguments(thisObject, methodUnderTest.thisObjectName, modelList, methodUnderTest.argumentsNames) + + return when (val evaluationResult = codeExecutor.run(functionArguments, localAdditionalModules)) { is PythonEvaluationError -> { val utError = UtError( "Error evaluation: ${evaluationResult.status}, ${evaluationResult.message}", @@ -203,7 +199,7 @@ class PythonEngine( .zip(methodUnderTest.arguments) .mapNotNull { it.first.summary?.replace("%var%", it.second.name) } - val hasThisObject = codeExecutor.methodArguments.thisObject != null + val hasThisObject = codeExecutor.method.hasThisArgument when (val result = handleSuccessResult(parameters, evaluationResult, description, hasThisObject, summary)) { is ValidExecution -> { @@ -245,9 +241,12 @@ class PythonEngine( logger.info { "Fuzzing process was interrupted by timeout" } return@PythonFuzzing PythonFeedback(control = Control.STOP) } - + return@PythonFuzzing fuzzingResultHandler(description, arguments) }.fuzz(pmd) + if (codeExecutor is PythonCodeSocketExecutor) { + codeExecutor.stop() + } } } } \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt index 6187355209..a1c6a68554 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt @@ -29,7 +29,7 @@ import java.io.File private val logger = KotlinLogging.logger {} -private const val COVERAGE_LIMIT = 100 +private const val COVERAGE_LIMIT = 10_000 class PythonTestCaseGenerator( private val withMinimization: Boolean = true, diff --git a/utbot-python/src/main/kotlin/org/utbot/python/UTPythonAPI.kt b/utbot-python/src/main/kotlin/org/utbot/python/UTPythonAPI.kt index fa5bda5f57..187b78cc6d 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/UTPythonAPI.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/UTPythonAPI.kt @@ -4,7 +4,7 @@ import org.parsers.python.ast.Block import org.utbot.framework.plugin.api.UtError import org.utbot.framework.plugin.api.UtExecution import org.utbot.python.framework.api.python.PythonClassId -import org.utbot.python.framework.api.python.PythonModel +import org.utbot.python.framework.api.python.PythonTreeModel import org.utbot.python.framework.api.python.util.pythonAnyClassId import org.utbot.python.newtyping.* import org.utbot.python.typing.MypyAnnotations @@ -60,10 +60,10 @@ data class PythonTestSet( ) data class FunctionArguments( - val thisObject: PythonModel?, + val thisObject: PythonTreeModel?, val thisObjectName: String?, - val arguments: List, + val arguments: List, val names: List, ) { - val allArguments: List = (listOf(thisObject) + arguments).filterNotNull() + val allArguments: List = (listOf(thisObject) + arguments).filterNotNull() } \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt index afb1db6f4d..4e3534cefa 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt @@ -1,21 +1,19 @@ package org.utbot.python.evaluation import org.utbot.framework.plugin.api.Coverage -import org.utbot.fuzzer.FuzzedValue import org.utbot.python.FunctionArguments import org.utbot.python.PythonMethod import org.utbot.python.evaluation.serialiation.MemoryDump interface PythonCodeExecutor { val method: PythonMethod - val methodArguments: FunctionArguments val moduleToImport: String val pythonPath: String val syspathDirectories: Set val executionTimeout: Long fun run( - fuzzedValues: List, + fuzzedValues: FunctionArguments, additionalModulesToImport: Set ): PythonEvaluationResult } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/ExecutionClient.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/ExecutionClient.kt index 4528906ca0..e9f84032b7 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/ExecutionClient.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/ExecutionClient.kt @@ -1,27 +1,36 @@ package org.utbot.python.evaluation +import org.utbot.python.utils.startProcess import java.io.BufferedReader import java.io.BufferedWriter import java.io.InputStreamReader import java.io.OutputStreamWriter -import java.io.PrintWriter import java.net.Socket -import java.nio.CharBuffer class ExecutionClient( - hostname: String, - port: Int + private val hostname: String, + private val port: Int, + val pythonPath: String ) { private val clientSocket: Socket = Socket(hostname, port) private val outStream = BufferedWriter(OutputStreamWriter(clientSocket.getOutputStream())) private val inStream = BufferedReader(InputStreamReader(clientSocket.getInputStream())) - - init { - runServer() - } - - private fun runServer() { - TODO() + private val process = runServer() + + private fun runServer() = + startProcess( + listOf( + pythonPath, + "-m", + "utbot_executor", + hostname, + port.toString() + ) + ) + + fun stopServer() { + stopConnection() + process.destroy() // ? } fun sendMessage(msg: String) { @@ -43,7 +52,7 @@ class ExecutionClient( } fun main() { - val client = ExecutionClient("localhost", 12011) + val client = ExecutionClient("localhost", 12011, "python") client.sendMessage("Firstfjlskdjf") client.sendMessage("Second") client.sendMessage("{123: 123}") diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeExecutorImpl.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeExecutorImpl.kt index b9de31646c..e7c90fd747 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeExecutorImpl.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeExecutorImpl.kt @@ -2,13 +2,13 @@ package org.utbot.python.evaluation import org.utbot.framework.plugin.api.Coverage import org.utbot.framework.plugin.api.Instruction -import org.utbot.fuzzer.FuzzedValue import org.utbot.python.FunctionArguments import org.utbot.python.PythonMethod import org.utbot.python.code.PythonCodeGenerator import org.utbot.python.evaluation.serialiation.ExecutionResultDeserializer import org.utbot.python.evaluation.serialiation.FailExecution import org.utbot.python.evaluation.serialiation.SuccessExecution +import org.utbot.python.framework.api.python.PythonTreeModel import org.utbot.python.framework.api.python.util.pythonAnyClassId import org.utbot.python.utils.TemporaryFileManager import org.utbot.python.utils.getResult @@ -22,7 +22,6 @@ data class EvaluationFiles( class PythonCodeExecutorImpl( override val method: PythonMethod, - override val methodArguments: FunctionArguments, override val moduleToImport: String, override val pythonPath: String, override val syspathDirectories: Set, @@ -30,15 +29,19 @@ class PythonCodeExecutorImpl( ) : PythonCodeExecutor { override fun run( - fuzzedValues: List, + fuzzedValues: FunctionArguments, additionalModulesToImport: Set, ): PythonEvaluationResult { - val evaluationFiles = generateExecutionCode(additionalModulesToImport) + val evaluationFiles = generateExecutionCode( + additionalModulesToImport, + fuzzedValues.allArguments, + ) return getEvaluationResult(evaluationFiles) } private fun generateExecutionCode( additionalModulesToImport: Set, + methodArguments: List, ): EvaluationFiles { val fileForOutput = TemporaryFileManager.assignTemporaryFile( tag = "out_" + method.name + ".py", @@ -50,7 +53,7 @@ class PythonCodeExecutorImpl( ) val runCode = PythonCodeGenerator.generateRunFunctionCode( method, - methodArguments.allArguments, + methodArguments, syspathDirectories, moduleToImport, additionalModulesToImport, diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt index 352afc936f..a3829b4d9c 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt @@ -1,31 +1,104 @@ package org.utbot.python.evaluation -import org.utbot.fuzzer.FuzzedValue +import org.utbot.framework.plugin.api.Coverage +import org.utbot.framework.plugin.api.Instruction import org.utbot.python.FunctionArguments import org.utbot.python.PythonMethod - - -data class ExecutionDescription( - val functionName: String, - val imports: List, - val argumentsIds: List, - val serializedMemory: String -) +import org.utbot.python.evaluation.serialiation.ExecutionRequest +import org.utbot.python.evaluation.serialiation.ExecutionRequestSerializer +import org.utbot.python.evaluation.serialiation.ExecutionResultDeserializer +import org.utbot.python.evaluation.serialiation.FailExecution +import org.utbot.python.evaluation.serialiation.PythonExecutionResult +import org.utbot.python.evaluation.serialiation.SuccessExecution +import org.utbot.python.evaluation.serialiation.serializeObjects +import org.utbot.python.framework.api.python.util.pythonAnyClassId +import org.utbot.python.utils.TemporaryFileManager class PythonCodeSocketExecutor( override val method: PythonMethod, - override val methodArguments: FunctionArguments, override val moduleToImport: String, override val pythonPath: String, override val syspathDirectories: Set, override val executionTimeout: Long, ) : PythonCodeExecutor { - val executionClient = ExecutionClient("localhost", 12011) // TODO: create port manager + private val executionClient = ExecutionClient("localhost", 12011, pythonPath) // TODO: create port manager override fun run( - fuzzedValues: List, - additionalModulesToImport: Set): PythonEvaluationResult { -// val input = - TODO() + fuzzedValues: FunctionArguments, + additionalModulesToImport: Set + ): PythonEvaluationResult { + val (arguments, memory) = serializeObjects(fuzzedValues.allArguments.map { it.tree }) + val coverageDatabasePath = TemporaryFileManager.assignTemporaryFile( + tag = "coverage_db_" + method.name, + addToCleaner = false, + ) + val request = ExecutionRequest( + method.name, + (additionalModulesToImport + moduleToImport).toList(), + syspathDirectories.toList(), + arguments, + memory, + coverageDatabasePath.path, + method.moduleFilename, + ) + val message = ExecutionRequestSerializer.serializeRequest(request) ?: error("Cannot serialize request to python executor") + executionClient.sendMessage(message) + val response = executionClient.receiveMessage() ?: error("Cannot read response from python executor") + val executionResult = ExecutionResultDeserializer.parseExecutionResult(response) ?: error("Cannot parse execution result") + return parseExecutionResult(executionResult) + } + + private fun parseExecutionResult(executionResult: PythonExecutionResult): PythonEvaluationResult { + val parsingException = PythonEvaluationError( + 0, + "Incorrect format of output", + emptyList() + ) + return when (executionResult) { + is SuccessExecution -> { + val stateBefore = ExecutionResultDeserializer.parseMemoryDump(executionResult.stateBefore) ?: return parsingException + val stateAfter = ExecutionResultDeserializer.parseMemoryDump(executionResult.stateAfter) ?: return parsingException + PythonEvaluationSuccess( + executionResult.isException, + calculateCoverage(executionResult.statements, executionResult.missedStatements), + stateBefore, + stateAfter, + executionResult.argsIds + executionResult.kwargsIds, + executionResult.resultId, + ) + } + is FailExecution -> PythonEvaluationError( + 0, + executionResult.exception, + emptyList(), + ) + } + } + + private fun calculateCoverage(statements: List, missedStatements: List): Coverage { + val covered = statements.filter { it !in missedStatements } + return Coverage( + coveredInstructions=covered.map { + Instruction( + method.containingPythonClassId?.name ?: pythonAnyClassId.name, + method.methodSignature(), + it, + it.toLong() + ) + }, + instructionsCount = statements.size.toLong(), + missedInstructions = missedStatements.map { + Instruction( + method.containingPythonClassId?.name ?: pythonAnyClassId.name, + method.methodSignature(), + it, + it.toLong() + ) + } + ) + } + + fun stop() { + executionClient.stopServer() } } \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionRequestSerializer.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionRequestSerializer.kt new file mode 100644 index 0000000000..c554214236 --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionRequestSerializer.kt @@ -0,0 +1,26 @@ +package org.utbot.python.evaluation.serialiation + +import com.squareup.moshi.Moshi +import com.squareup.moshi.kotlin.reflect.KotlinJsonAdapterFactory + +object ExecutionRequestSerializer { + private val moshi = Moshi.Builder() + .addLast(KotlinJsonAdapterFactory()) + .build() + + private val jsonAdapter = moshi.adapter(ExecutionRequest::class.java) + + fun serializeRequest(request: ExecutionRequest): String? { + return jsonAdapter.toJson(request) + } +} + +data class ExecutionRequest( + val functionName: String, + val imports: List, + val syspaths: List, + val argumentsIds: List, + val serializedMemory: String, + val coverageDB: String, + val filepath: String, +) From f5df9e71aa222039f80b5b726ff9939e37ad99b9 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Wed, 22 Feb 2023 12:59:17 +0300 Subject: [PATCH 09/32] Update socket version --- .../kotlin/org/utbot/python/PythonEngine.kt | 203 +++++++++++------- .../utbot/python/PythonTestCaseGenerator.kt | 13 +- .../python/evaluation/ExecutionClient.kt | 61 ------ .../evaluation/PythonCodeSocketExecutor.kt | 47 ++-- .../utbot/python/evaluation/PythonWorker.kt | 48 +++++ .../ExecutionRequestSerializer.kt | 2 +- .../ExecutionResultDeserializer.kt | 8 +- 7 files changed, 225 insertions(+), 157 deletions(-) delete mode 100644 utbot-python/src/main/kotlin/org/utbot/python/evaluation/ExecutionClient.kt create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorker.kt diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index c290ccf3bc..6873bcf9e1 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -20,6 +20,7 @@ import org.utbot.python.evaluation.PythonCodeSocketExecutor import org.utbot.python.evaluation.PythonEvaluationError import org.utbot.python.evaluation.PythonEvaluationSuccess import org.utbot.python.evaluation.PythonEvaluationTimeout +import org.utbot.python.evaluation.PythonWorker import org.utbot.python.evaluation.serialiation.MemoryDump import org.utbot.python.evaluation.serialiation.toPythonTree import org.utbot.python.framework.api.python.PythonTreeModel @@ -33,7 +34,11 @@ import org.utbot.python.newtyping.general.Type import org.utbot.python.newtyping.pythonModules import org.utbot.python.newtyping.pythonTypeRepresentation import org.utbot.python.utils.camelToSnakeCase +import org.utbot.python.utils.startProcess import org.utbot.summary.fuzzer.names.TestSuggestedInfo +import java.net.ServerSocket +import java.net.SocketTimeoutException +import java.util.concurrent.TimeUnit private val logger = KotlinLogging.logger {} @@ -145,107 +150,151 @@ class PythonEngine( return ValidExecution(utFuzzedExecution) } - private fun constructEvaluationInput(): PythonCodeExecutor { + private fun constructEvaluationInput(pythonWorker: PythonWorker): PythonCodeExecutor { return PythonCodeSocketExecutor( methodUnderTest, moduleToImport, pythonPath, directoriesForSysPath, timeoutForRun, + pythonWorker, ) } + + private conndectToExecutor(serverSocket: Socket): PythonWorker { + logger.info { "Server port: ${serverSocket.localPort}" } + val processStartTime = System.currentTimeMillis() +// val outputFile = "/home/vyacheslav/utbot_profile" +// val process = startProcess(listOf(pythonPath, "-m", "cProfile", "-o", outputFile, "-m", "utbot_executor", "localhost", serverSocket.localPort.toString())) + val process = startProcess(listOf(pythonPath, "-m", "utbot_executor", "localhost", serverSocket.localPort.toString())) + val timeout = until - processStartTime + val workerSocket = try { + serverSocket.soTimeout = timeout.toInt() + serverSocket.accept() + } catch (e: SocketTimeoutException) { + val processHasExited = process.waitFor(timeout, TimeUnit.MILLISECONDS) + if (!processHasExited) { + process.destroy() + } + error("Worker not connected") + } + logger.info { "Worker connected successfully" } + return PythonWorker(workerSocket) + } fun fuzzing(parameters: List, isCancelled: () -> Boolean, until: Long): Flow = flow { val additionalModules = parameters.flatMap { it.pythonModules() } val coveredLines = initialCoveredLines.toMutableSet() - val codeExecutor = constructEvaluationInput() - - suspend fun fuzzingResultHandler(description: PythonMethodDescription, arguments: List): PythonFeedback { - val argumentValues = arguments.map { PythonTreeModel(it.tree, it.tree.type) } - val argumentModules = argumentValues - .flatMap { it.allContainingClassIds } - .map { it.moduleName } - val localAdditionalModules = (additionalModules + argumentModules).toSet() - - val (thisObject, modelList) = - if (methodUnderTest.hasThisArgument) - Pair(argumentValues[0], argumentValues.drop(1)) - else - Pair(null, argumentValues) - val functionArguments = FunctionArguments(thisObject, methodUnderTest.thisObjectName, modelList, methodUnderTest.argumentsNames) - - return when (val evaluationResult = codeExecutor.run(functionArguments, localAdditionalModules)) { - is PythonEvaluationError -> { - val utError = UtError( - "Error evaluation: ${evaluationResult.status}, ${evaluationResult.message}", - Throwable(evaluationResult.stackTrace.joinToString("\n")) - ) - logger.debug(evaluationResult.stackTrace.joinToString("\n")) - emit(InvalidExecution(utError)) - PythonFeedback(control = Control.PASS) - } - is PythonEvaluationTimeout -> { - val utError = UtError(evaluationResult.message, Throwable()) - emit(InvalidExecution(utError)) - PythonFeedback(control = Control.PASS) - } +// withContext(Dispatchers.IO) { + ServerSocket(0).use { serverSocket -> + val pythonWorker = connectToExecutor(serverSocket) + val codeExecutor = constructEvaluationInput(pythonWorker) + logger.info { "Executor was created successfully" } + + suspend fun fuzzingResultHandler( + description: PythonMethodDescription, + arguments: List + ): PythonFeedback { + val argumentValues = arguments.map { PythonTreeModel(it.tree, it.tree.type) } + val argumentModules = argumentValues + .flatMap { it.allContainingClassIds } + .map { it.moduleName } + val localAdditionalModules = (additionalModules + argumentModules).toSet() - is PythonEvaluationSuccess -> { - val coveredInstructions = evaluationResult.coverage.coveredInstructions - coveredInstructions.forEach { coveredLines.add(it.lineNumber) } + val (thisObject, modelList) = + if (methodUnderTest.hasThisArgument) + Pair(argumentValues[0], argumentValues.drop(1)) + else + Pair(null, argumentValues) + val functionArguments = FunctionArguments( + thisObject, + methodUnderTest.thisObjectName, + modelList, + methodUnderTest.argumentsNames + ) - val summary = arguments - .zip(methodUnderTest.arguments) - .mapNotNull { it.first.summary?.replace("%var%", it.second.name) } + return when (val evaluationResult = codeExecutor.run(functionArguments, localAdditionalModules)) { + is PythonEvaluationError -> { + val utError = UtError( + "Error evaluation: ${evaluationResult.status}, ${evaluationResult.message}", + Throwable(evaluationResult.stackTrace.joinToString("\n")) + ) + logger.debug(evaluationResult.stackTrace.joinToString("\n")) + emit(InvalidExecution(utError)) + PythonFeedback(control = Control.PASS) + } - val hasThisObject = codeExecutor.method.hasThisArgument + is PythonEvaluationTimeout -> { + val utError = UtError(evaluationResult.message, Throwable()) + emit(InvalidExecution(utError)) + PythonFeedback(control = Control.PASS) + } - when (val result = handleSuccessResult(parameters, evaluationResult, description, hasThisObject, summary)) { - is ValidExecution -> { - logger.debug { arguments } - val trieNode: Trie.Node = description.tracer.add(coveredInstructions) - emit(result) - PythonFeedback(control = Control.CONTINUE, result = trieNode) - } - is ArgumentsTypeErrorFeedback, is TypeErrorFeedback -> { - emit(result) - PythonFeedback(control = Control.PASS) - } - is InvalidExecution -> { - emit(result) - PythonFeedback(control = Control.CONTINUE) + is PythonEvaluationSuccess -> { + val coveredInstructions = evaluationResult.coverage.coveredInstructions + coveredInstructions.forEach { coveredLines.add(it.lineNumber) } + + val summary = arguments + .zip(methodUnderTest.arguments) + .mapNotNull { it.first.summary?.replace("%var%", it.second.name) } + + val hasThisObject = codeExecutor.method.hasThisArgument + + when (val result = handleSuccessResult( + parameters, + evaluationResult, + description, + hasThisObject, + summary + )) { + is ValidExecution -> { + logger.debug { arguments } + val trieNode: Trie.Node = description.tracer.add(coveredInstructions) + emit(result) + PythonFeedback(control = Control.CONTINUE, result = trieNode) + } + + is ArgumentsTypeErrorFeedback, is TypeErrorFeedback -> { + emit(result) + PythonFeedback(control = Control.PASS) + } + + is InvalidExecution -> { + emit(result) + PythonFeedback(control = Control.CONTINUE) + } } } } } - } - val pmd = PythonMethodDescription( - methodUnderTest.name, - parameters, - fuzzedConcreteValues, - pythonTypeStorage, - Trie(Instruction::id) - ) + val pmd = PythonMethodDescription( + methodUnderTest.name, + parameters, + fuzzedConcreteValues, + pythonTypeStorage, + Trie(Instruction::id) + ) - if (parameters.isEmpty()) { - fuzzingResultHandler(pmd, emptyList()) - } else { - PythonFuzzing(pmd.pythonTypeStorage) { description, arguments -> - if (isCancelled()) { - logger.info { "Fuzzing process was interrupted" } - return@PythonFuzzing PythonFeedback(control = Control.STOP) - } - if (System.currentTimeMillis() >= until) { - logger.info { "Fuzzing process was interrupted by timeout" } - return@PythonFuzzing PythonFeedback(control = Control.STOP) + if (parameters.isEmpty()) { + fuzzingResultHandler(pmd, emptyList()) + } else { + PythonFuzzing(pmd.pythonTypeStorage) { description, arguments -> + if (isCancelled()) { + logger.info { "Fuzzing process was interrupted" } + return@PythonFuzzing PythonFeedback(control = Control.STOP) + } + if (System.currentTimeMillis() >= until) { + logger.info { "Fuzzing process was interrupted by timeout" } + return@PythonFuzzing PythonFeedback(control = Control.STOP) + } + + return@PythonFuzzing fuzzingResultHandler(description, arguments) + }.fuzz(pmd) + if (codeExecutor is PythonCodeSocketExecutor) { + codeExecutor.stop() } - - return@PythonFuzzing fuzzingResultHandler(description, arguments) - }.fuzz(pmd) - if (codeExecutor is PythonCodeSocketExecutor) { - codeExecutor.stop() } } } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt index a1c6a68554..da206d09e5 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt @@ -29,7 +29,8 @@ import java.io.File private val logger = KotlinLogging.logger {} -private const val COVERAGE_LIMIT = 10_000 +private const val COVERAGE_LIMIT = 150 +private const val ADDITIONAL_LIMIT = 10 class PythonTestCaseGenerator( private val withMinimization: Boolean = true, @@ -108,7 +109,8 @@ class PythonTestCaseGenerator( var missingLines: Set? = null val coveredLines = mutableSetOf() var generated = 0 - val typeInferenceCancellation = { isCancelled() || System.currentTimeMillis() >= until || missingLines?.size == 0 } + var additionalLimit = ADDITIONAL_LIMIT + val typeInferenceCancellation = { isCancelled() || System.currentTimeMillis() >= until || (missingLines?.size == 0 && additionalLimit == 0) } inferAnnotations( method, @@ -140,7 +142,7 @@ class PythonTestCaseGenerator( var feedback: InferredTypeFeedback = SuccessFeedback val fuzzerCancellation = { typeInferenceCancellation() || coverageLimit == 0 } // || feedback is InvalidTypeFeedback } - + val startTime = System.currentTimeMillis() engine.fuzzing(args, fuzzerCancellation, until).collect { generated += 1 when (it) { @@ -160,16 +162,19 @@ class PythonTestCaseGenerator( feedback = InvalidTypeFeedback } } + if (missingLines?.size == 0) { + additionalLimit -= 1 + } val coveredAfter = coveredLines.size if (coveredAfter == coveredBefore) { coverageLimit -= 1 } + logger.info { "${System.currentTimeMillis() - startTime}: $generated, $missingLines" } coveredBefore = coveredAfter } feedback } - val (successfulExecutions, failedExecutions) = executions.partition { it.result is UtExecutionSuccess } return PythonTestSet( diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/ExecutionClient.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/ExecutionClient.kt deleted file mode 100644 index e9f84032b7..0000000000 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/ExecutionClient.kt +++ /dev/null @@ -1,61 +0,0 @@ -package org.utbot.python.evaluation - -import org.utbot.python.utils.startProcess -import java.io.BufferedReader -import java.io.BufferedWriter -import java.io.InputStreamReader -import java.io.OutputStreamWriter -import java.net.Socket - -class ExecutionClient( - private val hostname: String, - private val port: Int, - val pythonPath: String -) { - private val clientSocket: Socket = Socket(hostname, port) - private val outStream = BufferedWriter(OutputStreamWriter(clientSocket.getOutputStream())) - private val inStream = BufferedReader(InputStreamReader(clientSocket.getInputStream())) - private val process = runServer() - - private fun runServer() = - startProcess( - listOf( - pythonPath, - "-m", - "utbot_executor", - hostname, - port.toString() - ) - ) - - fun stopServer() { - stopConnection() - process.destroy() // ? - } - - fun sendMessage(msg: String) { - outStream.write(msg) - outStream.flush() - outStream.write("END") - outStream.flush() - } - - fun receiveMessage(): String? { - return inStream.readLine() - } - - fun stopConnection() { - inStream.close() - outStream.close() - clientSocket.close() - } -} - -fun main() { - val client = ExecutionClient("localhost", 12011, "python") - client.sendMessage("Firstfjlskdjf") - client.sendMessage("Second") - client.sendMessage("{123: 123}") - client.sendMessage("STOP") - client.stopConnection() -} \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt index a3829b4d9c..92fbb5a5b8 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt @@ -12,7 +12,6 @@ import org.utbot.python.evaluation.serialiation.PythonExecutionResult import org.utbot.python.evaluation.serialiation.SuccessExecution import org.utbot.python.evaluation.serialiation.serializeObjects import org.utbot.python.framework.api.python.util.pythonAnyClassId -import org.utbot.python.utils.TemporaryFileManager class PythonCodeSocketExecutor( @@ -22,29 +21,51 @@ class PythonCodeSocketExecutor( override val syspathDirectories: Set, override val executionTimeout: Long, ) : PythonCodeExecutor { - private val executionClient = ExecutionClient("localhost", 12011, pythonPath) // TODO: create port manager + private lateinit var pythonWorker: PythonWorker + + constructor( + method: PythonMethod, + moduleToImport: String, + pythonPath: String, + syspathDirectories: Set, + executionTimeout: Long, + pythonWorker: PythonWorker + ) : this( + method, + moduleToImport, + pythonPath, + syspathDirectories, + executionTimeout + ) { + this.pythonWorker = pythonWorker + } + override fun run( fuzzedValues: FunctionArguments, additionalModulesToImport: Set ): PythonEvaluationResult { val (arguments, memory) = serializeObjects(fuzzedValues.allArguments.map { it.tree }) - val coverageDatabasePath = TemporaryFileManager.assignTemporaryFile( - tag = "coverage_db_" + method.name, - addToCleaner = false, - ) + + val containingClass = method.containingPythonClassId + val functionTextName = + if (containingClass == null) + method.name + else + "${containingClass.simpleName}.${method.name}" + val request = ExecutionRequest( - method.name, - (additionalModulesToImport + moduleToImport).toList(), + functionTextName, + moduleToImport, + additionalModulesToImport.toList(), syspathDirectories.toList(), arguments, memory, - coverageDatabasePath.path, method.moduleFilename, ) val message = ExecutionRequestSerializer.serializeRequest(request) ?: error("Cannot serialize request to python executor") - executionClient.sendMessage(message) - val response = executionClient.receiveMessage() ?: error("Cannot read response from python executor") - val executionResult = ExecutionResultDeserializer.parseExecutionResult(response) ?: error("Cannot parse execution result") + pythonWorker.sendData(message) + val response = pythonWorker.receiveMessage() + val executionResult = ExecutionResultDeserializer.parseExecutionResult(response) ?: error("Cannot parse execution result: $response") return parseExecutionResult(executionResult) } @@ -99,6 +120,6 @@ class PythonCodeSocketExecutor( } fun stop() { - executionClient.stopServer() + pythonWorker.stopServer() } } \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorker.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorker.kt new file mode 100644 index 0000000000..26acf722e6 --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorker.kt @@ -0,0 +1,48 @@ +package org.utbot.python.evaluation + +import java.io.BufferedReader +import java.io.BufferedWriter +import java.io.InputStreamReader +import java.io.OutputStreamWriter +import java.net.Socket + +class PythonWorker( + private val clientSocket: Socket +) { + private val outStream = BufferedWriter(OutputStreamWriter(clientSocket.getOutputStream())) + private val inStream = BufferedReader(InputStreamReader(clientSocket.getInputStream())) + + fun stop() { + outStream.write("STOP") + outStream.flush() + } + + fun sendData(msg: String) { + outStream.write("DATA") + + val size = msg.length + outStream.write(size.toString().padStart(16)) + + outStream.write(msg) + outStream.flush() + } + + fun receiveMessage(): String { + val length = inStream.readLine().toInt() + val buffer = CharArray(length) + inStream.read(buffer) + return String(buffer) + } + + private fun stopConnection() { + inStream.close() + outStream.close() + clientSocket.close() + } + + fun stopServer() { + stop() + stopConnection() + } + +} \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionRequestSerializer.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionRequestSerializer.kt index c554214236..33d2251107 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionRequestSerializer.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionRequestSerializer.kt @@ -17,10 +17,10 @@ object ExecutionRequestSerializer { data class ExecutionRequest( val functionName: String, + val functionModule: String, val imports: List, val syspaths: List, val argumentsIds: List, val serializedMemory: String, - val coverageDB: String, val filepath: String, ) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionResultDeserializer.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionResultDeserializer.kt index c6ec72d8c7..3ffdbe7bce 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionResultDeserializer.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionResultDeserializer.kt @@ -1,5 +1,6 @@ package org.utbot.python.evaluation.serialiation +import com.squareup.moshi.JsonEncodingException import com.squareup.moshi.Moshi import com.squareup.moshi.adapters.PolymorphicJsonAdapterFactory import com.squareup.moshi.kotlin.reflect.KotlinJsonAdapterFactory @@ -25,7 +26,12 @@ object ExecutionResultDeserializer { private val jsonAdapterMemoryDump = moshi.adapter(MemoryDump::class.java) fun parseExecutionResult(content: String): PythonExecutionResult? { - return jsonAdapter.fromJson(content) + try { + return jsonAdapter.fromJson(content) ?: error("Parsing error with: $content") + } catch (_: JsonEncodingException) { + println(content) + } + return null } fun parseMemoryDump(content: String): MemoryDump? { From 4cec02cf4f7b5716b2764213b41904c867776f56 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Wed, 22 Feb 2023 13:00:51 +0300 Subject: [PATCH 10/32] Update requirements.txt --- utbot-python/src/main/resources/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utbot-python/src/main/resources/requirements.txt b/utbot-python/src/main/resources/requirements.txt index da790dfa02..2896fba020 100644 --- a/utbot-python/src/main/resources/requirements.txt +++ b/utbot-python/src/main/resources/requirements.txt @@ -1,4 +1,4 @@ mypy==1.0.0 coverage -utbot-executor==0.2.4 +utbot-executor==1.0.21 utbot-mypy-runner==0.2.6 From c965458c92c7e5c79840d22f9c667d489e6482e2 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Wed, 22 Feb 2023 15:48:48 +0300 Subject: [PATCH 11/32] Fix rebase --- .../kotlin/org/utbot/python/PythonEngine.kt | 41 ++++++++----------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index 6873bcf9e1..13ab4c2bef 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -161,34 +161,29 @@ class PythonEngine( ) } - private conndectToExecutor(serverSocket: Socket): PythonWorker { - logger.info { "Server port: ${serverSocket.localPort}" } - val processStartTime = System.currentTimeMillis() -// val outputFile = "/home/vyacheslav/utbot_profile" -// val process = startProcess(listOf(pythonPath, "-m", "cProfile", "-o", outputFile, "-m", "utbot_executor", "localhost", serverSocket.localPort.toString())) - val process = startProcess(listOf(pythonPath, "-m", "utbot_executor", "localhost", serverSocket.localPort.toString())) - val timeout = until - processStartTime - val workerSocket = try { - serverSocket.soTimeout = timeout.toInt() - serverSocket.accept() - } catch (e: SocketTimeoutException) { - val processHasExited = process.waitFor(timeout, TimeUnit.MILLISECONDS) - if (!processHasExited) { - process.destroy() - } - error("Worker not connected") - } - logger.info { "Worker connected successfully" } - return PythonWorker(workerSocket) - } - fun fuzzing(parameters: List, isCancelled: () -> Boolean, until: Long): Flow = flow { val additionalModules = parameters.flatMap { it.pythonModules() } val coveredLines = initialCoveredLines.toMutableSet() -// withContext(Dispatchers.IO) { ServerSocket(0).use { serverSocket -> - val pythonWorker = connectToExecutor(serverSocket) + logger.info { "Server port: ${serverSocket.localPort}" } + val processStartTime = System.currentTimeMillis() +// val outputFile = "/home/vyacheslav/utbot_profile" +// val process = startProcess(listOf(pythonPath, "-m", "cProfile", "-o", outputFile, "-m", "utbot_executor", "localhost", serverSocket.localPort.toString())) + val process = startProcess(listOf(pythonPath, "-m", "utbot_executor", "localhost", serverSocket.localPort.toString())) + val timeout = until - processStartTime + val workerSocket = try { + serverSocket.soTimeout = timeout.toInt() + serverSocket.accept() + } catch (e: SocketTimeoutException) { + val processHasExited = process.waitFor(timeout, TimeUnit.MILLISECONDS) + if (!processHasExited) { + process.destroy() + } + error("Worker not connected") + } + logger.info { "Worker connected successfully" } + val pythonWorker = PythonWorker(workerSocket) val codeExecutor = constructEvaluationInput(pythonWorker) logger.info { "Executor was created successfully" } From a1c3f9ae14c4db902c1ba838bf89830279880190 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Mon, 27 Feb 2023 13:54:14 +0300 Subject: [PATCH 12/32] Fixed merge bugs --- .../src/main/kotlin/org/utbot/python/PythonEngine.kt | 7 ++++--- .../utbot/python/evaluation/PythonCodeSocketExecutor.kt | 9 +++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index d8c130e1d0..364bc7c42f 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -33,6 +33,7 @@ import org.utbot.python.newtyping.PythonTypeStorage import org.utbot.python.newtyping.general.Type import org.utbot.python.newtyping.pythonModules import org.utbot.python.newtyping.pythonTypeRepresentation +import org.utbot.python.utils.TemporaryFileManager import org.utbot.python.utils.camelToSnakeCase import org.utbot.python.utils.startProcess import org.utbot.summary.fuzzer.names.TestSuggestedInfo @@ -164,13 +165,13 @@ class PythonEngine( fun fuzzing(parameters: List, isCancelled: () -> Boolean, until: Long): Flow = flow { val additionalModules = parameters.flatMap { it.pythonModules() } val coveredLines = initialCoveredLines.toMutableSet() + val logfile = TemporaryFileManager.createTemporaryFile("","utbot_executor", "log", true) ServerSocket(0).use { serverSocket -> logger.info { "Server port: ${serverSocket.localPort}" } val processStartTime = System.currentTimeMillis() -// val outputFile = "/home/vyacheslav/utbot_profile" -// val process = startProcess(listOf(pythonPath, "-m", "cProfile", "-o", outputFile, "-m", "utbot_executor", "localhost", serverSocket.localPort.toString())) - val process = startProcess(listOf(pythonPath, "-m", "utbot_executor", "localhost", serverSocket.localPort.toString())) +// val process = startProcess(listOf(pythonPath, "-m", "cProfile", "-o", outputFile, "-m", "utbot_executor", "localhost", serverSocket.localPort.toString())) + val process = startProcess(listOf(pythonPath, "-m", "utbot_executor", "localhost", serverSocket.localPort.toString(), logfile.absolutePath)) val timeout = until - processStartTime val workerSocket = try { serverSocket.soTimeout = timeout.toInt() diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt index 92fbb5a5b8..5e321aff1c 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt @@ -12,6 +12,7 @@ import org.utbot.python.evaluation.serialiation.PythonExecutionResult import org.utbot.python.evaluation.serialiation.SuccessExecution import org.utbot.python.evaluation.serialiation.serializeObjects import org.utbot.python.framework.api.python.util.pythonAnyClassId +import org.utbot.python.newtyping.pythonTypeRepresentation class PythonCodeSocketExecutor( @@ -46,12 +47,12 @@ class PythonCodeSocketExecutor( ): PythonEvaluationResult { val (arguments, memory) = serializeObjects(fuzzedValues.allArguments.map { it.tree }) - val containingClass = method.containingPythonClassId + val containingClass = method.containingPythonClass val functionTextName = if (containingClass == null) method.name else - "${containingClass.simpleName}.${method.name}" + "${containingClass.pythonTypeRepresentation()}.${method.name}" val request = ExecutionRequest( functionTextName, @@ -101,7 +102,7 @@ class PythonCodeSocketExecutor( return Coverage( coveredInstructions=covered.map { Instruction( - method.containingPythonClassId?.name ?: pythonAnyClassId.name, + method.containingPythonClass?.pythonTypeRepresentation() ?: pythonAnyClassId.name, method.methodSignature(), it, it.toLong() @@ -110,7 +111,7 @@ class PythonCodeSocketExecutor( instructionsCount = statements.size.toLong(), missedInstructions = missedStatements.map { Instruction( - method.containingPythonClassId?.name ?: pythonAnyClassId.name, + method.containingPythonClass?.pythonTypeRepresentation() ?: pythonAnyClassId.name, method.methodSignature(), it, it.toLong() From 9f92d64b5cbcada51be4f6b6296421767596db0f Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Mon, 27 Feb 2023 13:54:35 +0300 Subject: [PATCH 13/32] Fixed data size bug --- .../src/main/kotlin/org/utbot/python/evaluation/PythonWorker.kt | 2 +- utbot-python/src/main/resources/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorker.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorker.kt index 26acf722e6..db5d24d612 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorker.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorker.kt @@ -20,7 +20,7 @@ class PythonWorker( fun sendData(msg: String) { outStream.write("DATA") - val size = msg.length + val size = msg.encodeToByteArray().size outStream.write(size.toString().padStart(16)) outStream.write(msg) diff --git a/utbot-python/src/main/resources/requirements.txt b/utbot-python/src/main/resources/requirements.txt index 5b200fc408..11e53756ea 100644 --- a/utbot-python/src/main/resources/requirements.txt +++ b/utbot-python/src/main/resources/requirements.txt @@ -1,4 +1,4 @@ mypy==1.0.0 coverage -utbot-executor==1.0.21 +utbot-executor==1.0.25 utbot-mypy-runner==0.2.8 From bdbe677f0233bb0a657447a86e1e549a18049812 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Mon, 27 Feb 2023 14:59:56 +0300 Subject: [PATCH 14/32] Update socket execution logic --- .../main/kotlin/org/utbot/python/PythonEngine.kt | 6 +++--- .../utbot/python/PythonTestGenerationProcessor.kt | 13 +++++++++++-- utbot-python/src/main/resources/requirements.txt | 2 +- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index 364bc7c42f..37ad4f596d 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -6,6 +6,7 @@ import mu.KotlinLogging import org.utbot.framework.plugin.api.DocRegularStmt import org.utbot.framework.plugin.api.EnvironmentModels import org.utbot.framework.plugin.api.Instruction +import org.utbot.framework.plugin.api.TimeoutException import org.utbot.framework.plugin.api.UtError import org.utbot.framework.plugin.api.UtExecutionResult import org.utbot.framework.plugin.api.UtExecutionSuccess @@ -170,8 +171,7 @@ class PythonEngine( ServerSocket(0).use { serverSocket -> logger.info { "Server port: ${serverSocket.localPort}" } val processStartTime = System.currentTimeMillis() -// val process = startProcess(listOf(pythonPath, "-m", "cProfile", "-o", outputFile, "-m", "utbot_executor", "localhost", serverSocket.localPort.toString())) - val process = startProcess(listOf(pythonPath, "-m", "utbot_executor", "localhost", serverSocket.localPort.toString(), logfile.absolutePath)) + val process = startProcess(listOf(pythonPath, "-m", "utbot_executor", "localhost", serverSocket.localPort.toString(), "--logfile", logfile.absolutePath)) val timeout = until - processStartTime val workerSocket = try { serverSocket.soTimeout = timeout.toInt() @@ -181,7 +181,7 @@ class PythonEngine( if (!processHasExited) { process.destroy() } - error("Worker not connected") + throw TimeoutException("Worker not connected") } logger.info { "Worker connected successfully" } val pythonWorker = PythonWorker(workerSocket) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt index 8edc5eeaf7..54ea0aaee2 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt @@ -2,12 +2,14 @@ package org.utbot.python import com.squareup.moshi.Moshi import com.squareup.moshi.kotlin.reflect.KotlinJsonAdapterFactory +import mu.KotlinLogging import org.utbot.python.framework.codegen.model.PythonSysPathImport import org.utbot.python.framework.codegen.model.PythonSystemImport import org.utbot.python.framework.codegen.model.PythonUserImport import org.utbot.framework.codegen.domain.TestFramework import org.utbot.framework.codegen.domain.models.CgMethodTestSet import org.utbot.framework.plugin.api.ExecutableId +import org.utbot.framework.plugin.api.TimeoutException import org.utbot.framework.plugin.api.UtExecutionSuccess import org.utbot.framework.plugin.api.util.UtContext import org.utbot.framework.plugin.api.util.withUtContext @@ -28,6 +30,8 @@ import java.nio.file.Path import kotlin.io.path.Path import kotlin.io.path.pathString +private val logger = KotlinLogging.logger {} + object PythonTestGenerationProcessor { fun processTestGeneration( pythonPath: String, @@ -108,8 +112,13 @@ object PythonTestGenerationProcessor { val tests = pythonMethods.mapIndexed { index, method -> val methodsLeft = pythonMethods.size - index val localUntil = (until - System.currentTimeMillis()) / methodsLeft + System.currentTimeMillis() - testCaseGenerator.generate(method, localUntil) - } + try { + testCaseGenerator.generate(method, localUntil) + } catch (_: TimeoutException) { + logger.warn { "Cannot connect to python code executor for method ${method.name}" } + null + } + }.filterNotNull() val (notEmptyTests, emptyTestSets) = tests.partition { it.executions.isNotEmpty() } diff --git a/utbot-python/src/main/resources/requirements.txt b/utbot-python/src/main/resources/requirements.txt index 11e53756ea..09e0bd7b9a 100644 --- a/utbot-python/src/main/resources/requirements.txt +++ b/utbot-python/src/main/resources/requirements.txt @@ -1,4 +1,4 @@ mypy==1.0.0 coverage -utbot-executor==1.0.25 +utbot-executor==1.1.1 utbot-mypy-runner==0.2.8 From 1c7b07ab883c4da6384ca77952bcf561a690be02 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Mon, 27 Feb 2023 16:02:01 +0300 Subject: [PATCH 15/32] Add feedback from execution, INVALID_EXECUTION_LIMIT and debug in python --- .../kotlin/org/utbot/python/PythonEngine.kt | 12 ++++++---- .../utbot/python/PythonTestCaseGenerator.kt | 23 +++++++++++-------- .../inference/baseline/BaselineAlgorithm.kt | 6 ++--- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index 37ad4f596d..871ed4cc0a 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -171,7 +171,14 @@ class PythonEngine( ServerSocket(0).use { serverSocket -> logger.info { "Server port: ${serverSocket.localPort}" } val processStartTime = System.currentTimeMillis() - val process = startProcess(listOf(pythonPath, "-m", "utbot_executor", "localhost", serverSocket.localPort.toString(), "--logfile", logfile.absolutePath)) + val process = startProcess(listOf( + pythonPath, + "-m", "utbot_executor", + "localhost", + serverSocket.localPort.toString(), + "--logfile", logfile.absolutePath, + "--loglevel", "DEBUG", + )) val timeout = until - processStartTime val workerSocket = try { serverSocket.soTimeout = timeout.toInt() @@ -246,17 +253,14 @@ class PythonEngine( summary )) { is ValidExecution -> { - logger.debug { arguments } val trieNode: Trie.Node = description.tracer.add(coveredInstructions) emit(result) PythonFeedback(control = Control.CONTINUE, result = trieNode) } - is ArgumentsTypeErrorFeedback, is TypeErrorFeedback -> { emit(result) PythonFeedback(control = Control.PASS) } - is InvalidExecution -> { emit(result) PythonFeedback(control = Control.CONTINUE) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt index d6dd164768..d628e43e24 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt @@ -30,7 +30,8 @@ import java.io.File private val logger = KotlinLogging.logger {} private const val COVERAGE_LIMIT = 150 -private const val ADDITIONAL_LIMIT = 10 +private const val ADDITIONAL_LIMIT = 5 +private const val INVALID_EXECUTION_LIMIT = 10 class PythonTestCaseGenerator( private val withMinimization: Boolean = true, @@ -162,7 +163,6 @@ class PythonTestCaseGenerator( var missingLines: Set? = null val coveredLines = mutableSetOf() var generated = 0 - var additionalLimit = ADDITIONAL_LIMIT val typeInferenceCancellation = { isCancelled() || System.currentTimeMillis() >= until || missingLines?.size == 0 } @@ -192,13 +192,19 @@ class PythonTestCaseGenerator( PythonTypeStorage.get(mypyStorage) ) + var invalidExecutionLimit = INVALID_EXECUTION_LIMIT var coverageLimit = COVERAGE_LIMIT + var additionalLimit = ADDITIONAL_LIMIT var coveredBefore = coveredLines.size var feedback: InferredTypeFeedback = SuccessFeedback - val fuzzerCancellation = - { typeInferenceCancellation() || coverageLimit == 0 } // || feedback is InvalidTypeFeedback } + val fuzzerCancellation = { + typeInferenceCancellation() + || coverageLimit == 0 + || additionalLimit == 0 + || invalidExecutionLimit == 0 + } val startTime = System.currentTimeMillis() engine.fuzzing(args, fuzzerCancellation, until).collect { @@ -209,26 +215,25 @@ class PythonTestCaseGenerator( missingLines = updateCoverage(it.utFuzzedExecution, coveredLines, missingLines) feedback = SuccessFeedback } - is InvalidExecution -> { errors += it.utError feedback = SuccessFeedback } - is ArgumentsTypeErrorFeedback -> { + invalidExecutionLimit-- feedback = InvalidTypeFeedback } - is TypeErrorFeedback -> { + invalidExecutionLimit-- feedback = InvalidTypeFeedback } } if (missingLines?.size == 0) { - additionalLimit -= 1 + additionalLimit-- } val coveredAfter = coveredLines.size if (coveredAfter == coveredBefore) { - coverageLimit -= 1 + coverageLimit-- } logger.info { "${System.currentTimeMillis() - startTime}: $generated, $missingLines" } coveredBefore = coveredAfter diff --git a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt index 57ca920f4a..2c5ef45432 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt @@ -52,16 +52,14 @@ class BaselineAlgorithm( logger.info("Checking ${newState.signature.pythonTypeRepresentation()}") if (checkSignature(newState.signature as FunctionType, fileForMypyRuns, configFile)) { logger.debug("Found new state!") - annotationHandler(newState.signature) - states.add(newState) - /* +// annotationHandler(newState.signature) +// states.add(newState) when (annotationHandler(newState.signature)) { SuccessFeedback -> { states.add(newState) } InvalidTypeFeedback -> {} } - */ } } else { states.remove(state) From 6cbdeef8919e3dc13adb50d60ed93db44700d704 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Mon, 27 Feb 2023 18:03:35 +0300 Subject: [PATCH 16/32] Update constants and requirements.txt --- .../main/resources/preprocessed_values.json | 46 +------------------ .../src/main/resources/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 46 deletions(-) diff --git a/utbot-python/src/main/resources/preprocessed_values.json b/utbot-python/src/main/resources/preprocessed_values.json index 577bdb9595..03f768cfd9 100644 --- a/utbot-python/src/main/resources/preprocessed_values.json +++ b/utbot-python/src/main/resources/preprocessed_values.json @@ -8,8 +8,6 @@ "1", "10", "100", - "10000000000000000159028911097599180468360808563945281389781327557747838772170381060813469985856815104", - "111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111", "1144132807", "123", "1267650600228229401496703205376", @@ -37,22 +35,10 @@ "name": "builtins.str", "instances": [ "''", - "'(1+3j)'", - "'(1.5+3.5j)'", - "'(3+0j)'", - "'(3.2+0j)'", - "'-123456789'", - "'-1234567890'", "'1001'", - "'10715086071862673209484250490600018105614048117055336074437503883703510511249361224931983788156958581275946729175531468251871452856923140435984577574698574803934567774824230985421074605062371141877954182153046474983581941267398767559165543946077062914571196477686542167660429831652624386837205668069376'", "'3'", "'3j'", "'500'", - "'6442450944'", - "''", - "'False'", - "'True'", - "'a = 1'", "'abcdefghijklmnopqrst'", "'foo'", "'global'", @@ -103,33 +89,26 @@ "instances": [ "range(127, 256)", "range(24)", - "range(0, 2 ** 100 + 1, 2)", - "range(1, 25 + 1)", "range(0, 3)", "range(-10, 10)", "range(10, -11, -1)", "range(240)", - "range(2 ** 200, 2 ** 201, 2 ** 100)", "range(200)", "range(50, 400)", "range(150)", "range(9, -1, -2)", - "range(3 * 5 * 7 * 11)", "range(4, 16)", - "range(0, 2 ** 100 - 1, 2)", "range(0, 55296)", "range(101)", "range(5000)", "range(65536, 1114112)", - "range((1 << 16) - 1)", "range(1500)", "range(1, 9)", "range(512)", "range(0, -20, -1)", "range(32, 127)", "range(52, 64)", - "range(1 << 1000)", - "range(70000)" + "range(70)" ] }, { @@ -141,7 +120,6 @@ "complex(0.0, 3.14)", "complex(1, 2)", "complex(float('inf'), float('inf'))", - "complex('1' * 500)", "complex(float('inf'), -1)", "complex(0.0, float('nan'))", "complex(10.0)", @@ -252,23 +230,16 @@ "bytearray(range(10))", "bytearray(b'a\\x80b')", "bytearray(b':a:b::c')", - "bytearray(b'\\x00' * 15 + b'\\x01')", "bytearray(b'hash this!')", "bytearray(b'bar')", - "bytearray(b'x' * 4)", - "bytearray([0, 1, 2, 42, 42, 42, 3, 4, 5, 6, 7, 8, 9])", "bytearray(b'----')", - "bytearray([i for i in range(256)])", "bytearray(b'bytearray')", "bytearray(b'spam')", - "bytearray([10, 100, 200])", "bytearray(b'abcdefghijk')", "bytearray(b'msssspp')", "bytearray(b'no error')", "bytearray(b'YWJj\\n')", - "bytearray([1, 1, 1, 1, 1, 5, 6, 7, 8, 9])", "bytearray(b'foobaz')", - "bytearray([0])", "bytearray(b'little lamb---')", "bytearray(b'abc\\xe9\\x00xxx')", "bytearray(b'def')", @@ -281,33 +252,21 @@ "bytearray(b'file.py')", "bytearray(b'ab')", "bytearray(b'this is a random bytearray object')", - "bytearray(b'x' * 8)", "bytearray(b' world\\n\\n\\n')", - "bytearray([1, 100, 200])", - "bytearray([102, 111, 111, 111, 111])", - "bytearray(range(1, 9))", - "bytearray([126, 127, 128, 129])", "bytearray(5)", - "bytearray([0, 1, 2, 102, 111, 111])", "bytearray(b'\\x00python\\x00test\\x00')", "bytearray(9)", "bytearray(b'abcde')", "bytearray(b'x')", "bytearray(b'0123456789')", - "bytearray([102, 111, 111, 102, 111, 111])", - "bytearray(2 ** 16)", "bytearray(b'python')", "bytearray(8192)", - "bytearray(list(range(8)) + list(range(256)))", "bytearray(b'\\xff\\x00\\x00')", "bytearray(b'hello1')", - "bytearray(range(16))", "bytearray(b'xyz')", "bytearray(b'\\xaaU\\xaaU')", "bytearray(b'Z')", "bytearray(8)", - "bytearray([1, 1, 1, 1, 1])", - "bytearray([0, 1, 2, 3, 4])", "bytearray(b'ghi')", "bytearray(b'[ytearra]')", "bytearray(b'abc')", @@ -316,11 +275,8 @@ "bytearray()", "bytearray(b'abcdefghijklmnopqrstuvwxyz')", "bytearray(b'my dog has fleas')", - "bytearray([1, 100, 3])", "bytearray(b'g\\xfcrk')", "bytearray(b'hello world')", - "bytearray([26, 43, 48])", - "bytearray([1, 2, 3, 4, 6, 7, 8])", "bytearray(b'0102abcdef')", "bytearray(1)", "bytearray(b'--------------')", diff --git a/utbot-python/src/main/resources/requirements.txt b/utbot-python/src/main/resources/requirements.txt index 09e0bd7b9a..cddb74e56f 100644 --- a/utbot-python/src/main/resources/requirements.txt +++ b/utbot-python/src/main/resources/requirements.txt @@ -1,4 +1,4 @@ mypy==1.0.0 coverage -utbot-executor==1.1.1 +utbot-executor==1.1.2 utbot-mypy-runner==0.2.8 From 634b6971ea69db44a293f2a5a28ea6d247ee1baa Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Wed, 1 Mar 2023 06:21:31 -0800 Subject: [PATCH 17/32] Change string constants and add encoding in socket streams --- .../main/kotlin/org/utbot/python/evaluation/PythonWorker.kt | 5 +++-- .../org/utbot/python/fuzzing/provider/StrValueProvider.kt | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorker.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorker.kt index db5d24d612..6dff965b9d 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorker.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorker.kt @@ -9,8 +9,8 @@ import java.net.Socket class PythonWorker( private val clientSocket: Socket ) { - private val outStream = BufferedWriter(OutputStreamWriter(clientSocket.getOutputStream())) - private val inStream = BufferedReader(InputStreamReader(clientSocket.getInputStream())) + private val outStream = BufferedWriter(OutputStreamWriter(clientSocket.getOutputStream(), "UTF8")) + private val inStream = BufferedReader(InputStreamReader(clientSocket.getInputStream(), "UTF8")) fun stop() { outStream.write("STOP") @@ -22,6 +22,7 @@ class PythonWorker( val size = msg.encodeToByteArray().size outStream.write(size.toString().padStart(16)) + outStream.flush() outStream.write(msg) outStream.flush() diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/StrValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/StrValueProvider.kt index 8b6ca3b691..4769aef332 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/StrValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/StrValueProvider.kt @@ -28,8 +28,8 @@ object StrValueProvider : ValueProvider Date: Wed, 1 Mar 2023 17:32:09 +0300 Subject: [PATCH 18/32] fuzzed value memorization + new type inference heuristic --- .../kotlin/org/utbot/python/PythonEngine.kt | 53 ++++++++++--------- .../utbot/python/PythonTestCaseGenerator.kt | 2 +- .../python/framework/api/python/PythonTree.kt | 46 ++++++++++++++-- .../org/utbot/python/fuzzing/PythonApi.kt | 9 ++++ .../inference/baseline/BaselineAlgorithm.kt | 10 ++-- .../inference/baseline/Structures.kt | 1 + .../org/utbot/python/newtyping/utils/Utils.kt | 11 ++++ 7 files changed, 99 insertions(+), 33 deletions(-) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index 871ed4cc0a..8d08fa904e 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -25,11 +25,8 @@ import org.utbot.python.evaluation.PythonWorker import org.utbot.python.evaluation.serialiation.MemoryDump import org.utbot.python.evaluation.serialiation.toPythonTree import org.utbot.python.framework.api.python.PythonTreeModel -import org.utbot.python.fuzzing.PythonFeedback -import org.utbot.python.fuzzing.PythonFuzzedConcreteValue -import org.utbot.python.fuzzing.PythonFuzzedValue -import org.utbot.python.fuzzing.PythonFuzzing -import org.utbot.python.fuzzing.PythonMethodDescription +import org.utbot.python.framework.api.python.PythonTreeWrapper +import org.utbot.python.fuzzing.* import org.utbot.python.newtyping.PythonTypeStorage import org.utbot.python.newtyping.general.Type import org.utbot.python.newtyping.pythonModules @@ -44,12 +41,6 @@ import java.util.concurrent.TimeUnit private val logger = KotlinLogging.logger {} -sealed interface FuzzingExecutionFeedback -class ValidExecution(val utFuzzedExecution: UtFuzzedExecution): FuzzingExecutionFeedback -class InvalidExecution(val utError: UtError): FuzzingExecutionFeedback -class TypeErrorFeedback(val message: String) : FuzzingExecutionFeedback -class ArgumentsTypeErrorFeedback(val message: String) : FuzzingExecutionFeedback - class PythonEngine( private val methodUnderTest: PythonMethod, private val directoriesForSysPath: Set, @@ -195,11 +186,12 @@ class PythonEngine( val codeExecutor = constructEvaluationInput(pythonWorker) logger.info { "Executor was created successfully" } - suspend fun fuzzingResultHandler( + fun fuzzingResultHandler( description: PythonMethodDescription, arguments: List ): PythonFeedback { val argumentValues = arguments.map { PythonTreeModel(it.tree, it.tree.type) } + logger.debug(argumentValues.map { it.tree } .toString()) val argumentModules = argumentValues .flatMap { it.allContainingClassIds } .map { it.moduleName } @@ -225,14 +217,12 @@ class PythonEngine( Throwable(evaluationResult.stackTrace.joinToString("\n")) ) logger.debug(evaluationResult.stackTrace.joinToString("\n")) - emit(InvalidExecution(utError)) - PythonFeedback(control = Control.PASS) + PythonFeedback(control = Control.PASS, executionFeedback = InvalidExecution(utError)) } is PythonEvaluationTimeout -> { val utError = UtError(evaluationResult.message, Throwable()) - emit(InvalidExecution(utError)) - PythonFeedback(control = Control.PASS) + PythonFeedback(control = Control.PASS, executionFeedback = InvalidExecution(utError)) } is PythonEvaluationSuccess -> { @@ -254,16 +244,13 @@ class PythonEngine( )) { is ValidExecution -> { val trieNode: Trie.Node = description.tracer.add(coveredInstructions) - emit(result) - PythonFeedback(control = Control.CONTINUE, result = trieNode) + PythonFeedback(control = Control.CONTINUE, result = trieNode, result) } is ArgumentsTypeErrorFeedback, is TypeErrorFeedback -> { - emit(result) - PythonFeedback(control = Control.PASS) + PythonFeedback(control = Control.PASS, executionFeedback = result) } is InvalidExecution -> { - emit(result) - PythonFeedback(control = Control.CONTINUE) + PythonFeedback(control = Control.CONTINUE, executionFeedback = result) } } } @@ -278,20 +265,36 @@ class PythonEngine( Trie(Instruction::id) ) + val checkedModels = mutableMapOf>, PythonFeedback>() if (parameters.isEmpty()) { fuzzingResultHandler(pmd, emptyList()) } else { PythonFuzzing(pmd.pythonTypeStorage) { description, arguments -> if (isCancelled()) { logger.info { "Fuzzing process was interrupted" } - return@PythonFuzzing PythonFeedback(control = Control.STOP) + return@PythonFuzzing PythonFeedback(control = Control.STOP, executionFeedback = null) } if (System.currentTimeMillis() >= until) { logger.info { "Fuzzing process was interrupted by timeout" } - return@PythonFuzzing PythonFeedback(control = Control.STOP) + return@PythonFuzzing PythonFeedback(control = Control.STOP, executionFeedback = null) } - return@PythonFuzzing fuzzingResultHandler(description, arguments) + val pair = Pair(description, arguments.map { PythonTreeWrapper(it.tree) }) + val mem = checkedModels[pair] + if (mem != null) { + logger.debug("Repeat in fuzzing") + if (mem.executionFeedback != null) + emit(mem.executionFeedback) + + return@PythonFuzzing mem + } + val result = fuzzingResultHandler(description, arguments) + checkedModels[pair] = result + + if (result.executionFeedback != null) + emit(result.executionFeedback) + + return@PythonFuzzing result }.fuzz(pmd) if (codeExecutor is PythonCodeSocketExecutor) { codeExecutor.stop() diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt index d628e43e24..8355e29b25 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt @@ -8,7 +8,7 @@ import org.utbot.framework.plugin.api.UtError import org.utbot.framework.plugin.api.UtExecution import org.utbot.framework.plugin.api.UtExecutionSuccess import org.utbot.python.code.PythonCode -import org.utbot.python.fuzzing.PythonFuzzedConcreteValue +import org.utbot.python.fuzzing.* import org.utbot.python.newtyping.* import org.utbot.python.newtyping.ast.visitor.Visitor import org.utbot.python.newtyping.ast.visitor.constants.ConstantCollector diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt index 8f096e33b3..905284708d 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt @@ -11,6 +11,7 @@ import java.math.BigDecimal import java.math.BigInteger import java.util.* import java.util.concurrent.atomic.AtomicLong +import kotlin.Comparator object PythonTree { @@ -23,6 +24,10 @@ object PythonTree { open val children: List = emptyList() + override fun toString(): String { + return type.name + children.toString() + } + open fun typeEquals(other: Any?): Boolean { return if (other is PythonTreeNode) type == other.type && comparable && other.comparable @@ -34,7 +39,7 @@ object PythonTree { if (other !is PythonTreeNode) { return false } - return type == other.type + return type == other.type && children == other.children && comparable == other.comparable } override fun hashCode(): Int { @@ -43,6 +48,14 @@ object PythonTree { result = 31 * result + children.hashCode() return result } + + open fun softEquals(other: PythonTreeNode): Boolean { // overridden for ReduceNode + return this == other + } + + open fun softHashCode(): Int { // overridden for ReduceNode + return hashCode() + } } class PrimitiveNode( @@ -63,6 +76,10 @@ object PythonTree { result = 31 * result + repr.hashCode() return result } + + override fun toString(): String { + return repr + } } class ListNode( @@ -243,9 +260,20 @@ object PythonTree { } override fun hashCode(): Int { - var result = super.hashCode() + var result = softHashCode() result = 31 * result + id.hashCode() - result = 31 * result + constructor.hashCode() + return result + } + + override fun softEquals(other: PythonTreeNode): Boolean { // like equals(), but skip id check + if (other !is ReduceNode) + return false + return type == other.type && constructor == other.constructor && args == other.args && + state == other.state && listitems == other.listitems && dictitems == other.dictitems + } + + override fun softHashCode(): Int { + var result = constructor.hashCode() result = 31 * result + args.hashCode() result = 31 * result + state.hashCode() result = 31 * result + listitems.hashCode() @@ -346,3 +374,15 @@ object PythonIdGenerator { } } + +class PythonTreeWrapper(val tree: PythonTree.PythonTreeNode) { + override fun equals(other: Any?): Boolean { + if (other !is PythonTreeWrapper) + return false + return tree.softEquals(other.tree) + } + + override fun hashCode(): Int { + return tree.softHashCode() + } +} \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt index 459abe5a24..20aef61ae0 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt @@ -2,9 +2,11 @@ package org.utbot.python.fuzzing import mu.KotlinLogging import org.utbot.framework.plugin.api.Instruction +import org.utbot.framework.plugin.api.UtError import org.utbot.fuzzer.FuzzedContext import org.utbot.fuzzer.IdGenerator import org.utbot.fuzzer.IdentityPreservingIdGenerator +import org.utbot.fuzzer.UtFuzzedExecution import org.utbot.fuzzing.Configuration import org.utbot.fuzzing.Control import org.utbot.fuzzing.Description @@ -40,9 +42,16 @@ class PythonMethodDescription( val tracer: Trie, ) : Description(parameters) +sealed interface FuzzingExecutionFeedback +class ValidExecution(val utFuzzedExecution: UtFuzzedExecution): FuzzingExecutionFeedback +class InvalidExecution(val utError: UtError): FuzzingExecutionFeedback +class TypeErrorFeedback(val message: String) : FuzzingExecutionFeedback +class ArgumentsTypeErrorFeedback(val message: String) : FuzzingExecutionFeedback + data class PythonFeedback( override val control: Control = Control.CONTINUE, val result: Trie.Node = Trie.emptyNode(), + val executionFeedback: FuzzingExecutionFeedback? ) : Feedback class PythonFuzzedValue( diff --git a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt index 2c5ef45432..c192ecc5b4 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt @@ -8,6 +8,7 @@ import org.utbot.python.newtyping.general.FunctionType import org.utbot.python.newtyping.general.Type import org.utbot.python.newtyping.inference.* import org.utbot.python.newtyping.mypy.checkSuggestedSignatureWithDMypy +import org.utbot.python.newtyping.utils.weightedRandom import org.utbot.python.utils.TemporaryFileManager import java.io.File @@ -38,7 +39,7 @@ class BaselineAlgorithm( ) { val generalRating = createGeneralTypeRating(hintCollectorResult, storage) val initialState = getInitialState(hintCollectorResult, generalRating) - val states: MutableSet = mutableSetOf(initialState) + val states: MutableList = mutableListOf(initialState) val fileForMypyRuns = TemporaryFileManager.assignTemporaryFile(tag = "mypy.py") run breaking@ { @@ -57,6 +58,7 @@ class BaselineAlgorithm( when (annotationHandler(newState.signature)) { SuccessFeedback -> { states.add(newState) + state.children += 1 } InvalidTypeFeedback -> {} } @@ -85,9 +87,9 @@ class BaselineAlgorithm( ) } - // TODO: something smarter? - private fun chooseState(states: Set): BaselineAlgorithmState { - return states.random() + private fun chooseState(states: List): BaselineAlgorithmState { + val weights = states.map { 1.0 / (it.children * it.children + 1) } + return weightedRandom(states, weights) } private fun getInitialState( diff --git a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/Structures.kt b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/Structures.kt index f97d2b16b7..dc9552ad68 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/Structures.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/Structures.kt @@ -37,4 +37,5 @@ class BaselineAlgorithmState( get() = nodes.find { it.isRoot }!!.partialType val anyNodes: List = nodes.mapNotNull { it as? AnyTypeNode } val candidateGraph = CandidateGraph(anyNodes, generalRating, typeStorage) + var children: Int = 0 } \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/utils/Utils.kt b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/utils/Utils.kt index f3faa2d296..07bf61550d 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/utils/Utils.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/utils/Utils.kt @@ -1,5 +1,16 @@ package org.utbot.python.newtyping.utils +import kotlin.random.Random + fun getOffsetLine(sourceFileContent: String, offset: Int): Int { return sourceFileContent.take(offset).count { it == '\n' } + 1 } + +fun weightedRandom(elems: List, weights: List): T { + val sum = weights.sum() + val borders = weights.fold(emptyList() to 0.0) { (list, partialSum), cur -> + (list + (partialSum + cur) / sum) to partialSum + cur + }.first + val value = Random.nextDouble() + return elems[borders.indexOfFirst { it >= value }] +} \ No newline at end of file From 80817de9c07be4e512e3e0f291e916a84494ca2a Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Wed, 1 Mar 2023 19:43:50 +0300 Subject: [PATCH 19/32] Fix method import paths, collecting modules to globals, string representation in state dict, move up ADDITIONAL_LIMIT, update utbot_executor --- .../kotlin/org/utbot/python/PythonEngine.kt | 2 +- .../utbot/python/PythonTestCaseGenerator.kt | 5 +++-- .../evaluation/PythonCodeSocketExecutor.kt | 15 +++++++++------ .../serialiation/PythonObjectParser.kt | 18 ++++++++---------- .../python/framework/api/python/PythonTree.kt | 7 ++----- .../framework/api/python/util/PythonUtils.kt | 7 +++++++ .../fuzzing/provider/ReduceValueProvider.kt | 3 ++- .../src/main/resources/requirements.txt | 2 +- 8 files changed, 33 insertions(+), 26 deletions(-) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index 8d08fa904e..8e1781a15f 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -157,7 +157,7 @@ class PythonEngine( fun fuzzing(parameters: List, isCancelled: () -> Boolean, until: Long): Flow = flow { val additionalModules = parameters.flatMap { it.pythonModules() } val coveredLines = initialCoveredLines.toMutableSet() - val logfile = TemporaryFileManager.createTemporaryFile("","utbot_executor", "log", true) + val logfile = TemporaryFileManager.createTemporaryFile("","utbot_executor.log", "log", true) ServerSocket(0).use { serverSocket -> logger.info { "Server port: ${serverSocket.localPort}" } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt index 8355e29b25..5355e19fd7 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt @@ -163,8 +163,10 @@ class PythonTestCaseGenerator( var missingLines: Set? = null val coveredLines = mutableSetOf() var generated = 0 + + var additionalLimit = ADDITIONAL_LIMIT val typeInferenceCancellation = - { isCancelled() || System.currentTimeMillis() >= until || missingLines?.size == 0 } + { isCancelled() || System.currentTimeMillis() >= until || additionalLimit <= 0 } logger.info("Start test generation for ${method.name}") substituteTypeParameters(method, typeStorage).forEach { newMethod -> @@ -194,7 +196,6 @@ class PythonTestCaseGenerator( var invalidExecutionLimit = INVALID_EXECUTION_LIMIT var coverageLimit = COVERAGE_LIMIT - var additionalLimit = ADDITIONAL_LIMIT var coveredBefore = coveredLines.size var feedback: InferredTypeFeedback = SuccessFeedback diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt index 5e321aff1c..8bf3b0e930 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt @@ -12,6 +12,7 @@ import org.utbot.python.evaluation.serialiation.PythonExecutionResult import org.utbot.python.evaluation.serialiation.SuccessExecution import org.utbot.python.evaluation.serialiation.serializeObjects import org.utbot.python.framework.api.python.util.pythonAnyClassId +import org.utbot.python.newtyping.pythonTypeName import org.utbot.python.newtyping.pythonTypeRepresentation @@ -51,8 +52,10 @@ class PythonCodeSocketExecutor( val functionTextName = if (containingClass == null) method.name - else - "${containingClass.pythonTypeRepresentation()}.${method.name}" + else { + val fullname = "${containingClass.pythonTypeName()}.${method.name}" + fullname.drop(moduleToImport.length).removePrefix(".") + } val request = ExecutionRequest( functionTextName, @@ -72,7 +75,7 @@ class PythonCodeSocketExecutor( private fun parseExecutionResult(executionResult: PythonExecutionResult): PythonEvaluationResult { val parsingException = PythonEvaluationError( - 0, + -1, "Incorrect format of output", emptyList() ) @@ -90,9 +93,9 @@ class PythonCodeSocketExecutor( ) } is FailExecution -> PythonEvaluationError( - 0, - executionResult.exception, - emptyList(), + -2, + "Fail Execution", + executionResult.exception.split(System.lineSeparator()), ) } } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/PythonObjectParser.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/PythonObjectParser.kt index 1bc7b376ac..48ce151a80 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/PythonObjectParser.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/PythonObjectParser.kt @@ -37,10 +37,6 @@ class MemoryDump( return objects[id]!! } - fun getByValue(value: MemoryObject): String { - return objects.filter { it.value == value }.keys.first() - } - fun addObject(value: MemoryObject) { objects[value.id] = value } @@ -107,7 +103,9 @@ fun PythonTree.PythonTreeNode.toMemoryObject(memoryDump: MemoryDump): String { } is PythonTree.DictNode -> { val items = this.items.entries - .associate { it.key.toMemoryObject(memoryDump) to it.value.toMemoryObject(memoryDump) } + .associate { + it.key.toMemoryObject(memoryDump) to it.value.toMemoryObject(memoryDump) + } DictMemoryObject(this.id.toString(), this.type.name, this.comparable, items) } is PythonTree.ReduceNode -> { @@ -137,11 +135,11 @@ fun PythonTree.PythonTreeNode.toMemoryObject(memoryDump: MemoryDump): String { fun MemoryObject.toPythonTree(memoryDump: MemoryDump): PythonTree.PythonTreeNode { val obj = when(this) { is ReprMemoryObject -> { - PythonTree.PrimitiveNode( - this.id.toLong(), - PythonClassId(this.kind), - this.value - ) + PythonTree.PrimitiveNode( + this.id.toLong(), + PythonClassId(this.kind), + this.value + ) } is DictMemoryObject -> { PythonTree.DictNode( diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt index 905284708d..d2def022b2 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt @@ -5,13 +5,13 @@ import org.utbot.python.framework.api.python.util.pythonFloatClassId import org.utbot.python.framework.api.python.util.pythonIntClassId import org.utbot.python.framework.api.python.util.pythonNoneClassId import org.utbot.python.framework.api.python.util.pythonStrClassId +import org.utbot.python.framework.api.python.util.toPythonRepr import org.utbot.python.newtyping.general.Type import org.utbot.python.newtyping.pythonTypeName import java.math.BigDecimal import java.math.BigInteger import java.util.* import java.util.concurrent.atomic.AtomicLong -import kotlin.Comparator object PythonTree { @@ -315,12 +315,9 @@ object PythonTree { } fun fromString(value: String): PrimitiveNode { - val repr = value - .replace("\"", "\\\"") - .replace("\\\\\"", "\\\"") return PrimitiveNode( pythonStrClassId, - "\"$repr\"" + value.toPythonRepr() ) } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonUtils.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonUtils.kt index ea0ca0b6fe..f5aaefbd37 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonUtils.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonUtils.kt @@ -15,3 +15,10 @@ fun String.toSnakeCase(): String { } else c }.joinToString("") } + +fun String.toPythonRepr(): String { + val repr = this + .replace("\"", "\\\"") + .replace("\\\\\"", "\\\"") + return "\"$repr\"" +} diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ReduceValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ReduceValueProvider.kt index 637cf993cb..cb5ef0c9c4 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ReduceValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ReduceValueProvider.kt @@ -5,6 +5,7 @@ import org.utbot.fuzzing.Seed import org.utbot.fuzzing.ValueProvider import org.utbot.python.framework.api.python.PythonClassId import org.utbot.python.framework.api.python.PythonTree +import org.utbot.python.framework.api.python.util.toPythonRepr import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription import org.utbot.python.newtyping.* @@ -70,7 +71,7 @@ object ReduceValueProvider : ValueProvider Routine.Call(listOf(field.type)) { instance, arguments -> val obj = instance.tree as PythonTree.ReduceNode - obj.state[field.meta.name] = arguments.first().tree + obj.state[field.meta.name.toPythonRepr()] = arguments.first().tree } }) yieldAll(callConstructors(type, it, modifications.asSequence())) diff --git a/utbot-python/src/main/resources/requirements.txt b/utbot-python/src/main/resources/requirements.txt index cddb74e56f..a8dd4498fb 100644 --- a/utbot-python/src/main/resources/requirements.txt +++ b/utbot-python/src/main/resources/requirements.txt @@ -1,4 +1,4 @@ mypy==1.0.0 coverage -utbot-executor==1.1.2 +utbot-executor==1.1.11 utbot-mypy-runner==0.2.8 From 99c16cbbfd549dd4b31e599b08bb468ee597ce9e Mon Sep 17 00:00:00 2001 From: Ekaterina Tochilina Date: Thu, 2 Mar 2023 10:52:03 +0300 Subject: [PATCH 20/32] Set cache size limit + fixed PythonFeedback --- .../samples/easy_samples/annotation_tests.py | 2 +- .../kotlin/org/utbot/python/PythonEngine.kt | 48 +++++++++++-------- .../python/framework/api/python/PythonTree.kt | 4 ++ .../org/utbot/python/fuzzing/PythonApi.kt | 6 ++- 4 files changed, 39 insertions(+), 21 deletions(-) diff --git a/utbot-python/samples/easy_samples/annotation_tests.py b/utbot-python/samples/easy_samples/annotation_tests.py index 55492d5053..d9412617e6 100644 --- a/utbot-python/samples/easy_samples/annotation_tests.py +++ b/utbot-python/samples/easy_samples/annotation_tests.py @@ -10,7 +10,7 @@ class A(Generic[XXX]): self_: XXX - def f(self, a, b: A[int]): + def f(self, a, b: 'A'[int]): self.y = b self.self_.x = b pass diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index 8e1781a15f..aa668586c6 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -40,6 +40,7 @@ import java.net.SocketTimeoutException import java.util.concurrent.TimeUnit private val logger = KotlinLogging.logger {} +private const val MAX_CACHE_SIZE = 200 class PythonEngine( private val methodUnderTest: PythonMethod, @@ -52,6 +53,18 @@ class PythonEngine( private val pythonTypeStorage: PythonTypeStorage, ) { + private val cache = mutableMapOf>, PythonExecutionResult>() + + private fun addExecutionToCache(key: Pair>, result: PythonExecutionResult) { + cache[key] = result + if (cache.size > MAX_CACHE_SIZE) { + val elemToDelete = cache.keys.minBy { (_, args) -> + args.fold(0) { acc, arg -> acc + arg.tree.diversity() } + } + cache.remove(elemToDelete) + } + } + private fun suggestExecutionName( description: PythonMethodDescription, executionResult: UtExecutionResult @@ -154,6 +167,8 @@ class PythonEngine( ) } + + fun fuzzing(parameters: List, isCancelled: () -> Boolean, until: Long): Flow = flow { val additionalModules = parameters.flatMap { it.pythonModules() } val coveredLines = initialCoveredLines.toMutableSet() @@ -189,7 +204,7 @@ class PythonEngine( fun fuzzingResultHandler( description: PythonMethodDescription, arguments: List - ): PythonFeedback { + ): PythonExecutionResult { val argumentValues = arguments.map { PythonTreeModel(it.tree, it.tree.type) } logger.debug(argumentValues.map { it.tree } .toString()) val argumentModules = argumentValues @@ -217,12 +232,12 @@ class PythonEngine( Throwable(evaluationResult.stackTrace.joinToString("\n")) ) logger.debug(evaluationResult.stackTrace.joinToString("\n")) - PythonFeedback(control = Control.PASS, executionFeedback = InvalidExecution(utError)) + PythonExecutionResult(InvalidExecution(utError), PythonFeedback(control = Control.PASS)) } is PythonEvaluationTimeout -> { val utError = UtError(evaluationResult.message, Throwable()) - PythonFeedback(control = Control.PASS, executionFeedback = InvalidExecution(utError)) + PythonExecutionResult(InvalidExecution(utError), PythonFeedback(control = Control.PASS)) } is PythonEvaluationSuccess -> { @@ -244,13 +259,13 @@ class PythonEngine( )) { is ValidExecution -> { val trieNode: Trie.Node = description.tracer.add(coveredInstructions) - PythonFeedback(control = Control.CONTINUE, result = trieNode, result) + PythonExecutionResult(result, PythonFeedback(control = Control.CONTINUE, result = trieNode)) } is ArgumentsTypeErrorFeedback, is TypeErrorFeedback -> { - PythonFeedback(control = Control.PASS, executionFeedback = result) + PythonExecutionResult(result, PythonFeedback(control = Control.PASS)) } is InvalidExecution -> { - PythonFeedback(control = Control.CONTINUE, executionFeedback = result) + PythonExecutionResult(result, PythonFeedback(control = Control.CONTINUE)) } } } @@ -265,36 +280,31 @@ class PythonEngine( Trie(Instruction::id) ) - val checkedModels = mutableMapOf>, PythonFeedback>() if (parameters.isEmpty()) { fuzzingResultHandler(pmd, emptyList()) } else { PythonFuzzing(pmd.pythonTypeStorage) { description, arguments -> if (isCancelled()) { logger.info { "Fuzzing process was interrupted" } - return@PythonFuzzing PythonFeedback(control = Control.STOP, executionFeedback = null) + return@PythonFuzzing PythonFeedback(control = Control.STOP) } if (System.currentTimeMillis() >= until) { logger.info { "Fuzzing process was interrupted by timeout" } - return@PythonFuzzing PythonFeedback(control = Control.STOP, executionFeedback = null) + return@PythonFuzzing PythonFeedback(control = Control.STOP) } val pair = Pair(description, arguments.map { PythonTreeWrapper(it.tree) }) - val mem = checkedModels[pair] + val mem = cache[pair] if (mem != null) { logger.debug("Repeat in fuzzing") - if (mem.executionFeedback != null) - emit(mem.executionFeedback) - - return@PythonFuzzing mem + emit(mem.fuzzingExecutionFeedback) + return@PythonFuzzing mem.fuzzingPlatformFeedback } val result = fuzzingResultHandler(description, arguments) - checkedModels[pair] = result - - if (result.executionFeedback != null) - emit(result.executionFeedback) + addExecutionToCache(pair, result) - return@PythonFuzzing result + emit(result.fuzzingExecutionFeedback) + return@PythonFuzzing result.fuzzingPlatformFeedback }.fuzz(pmd) if (codeExecutor is PythonCodeSocketExecutor) { codeExecutor.stop() diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt index d2def022b2..76a488aea5 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt @@ -56,6 +56,8 @@ object PythonTree { open fun softHashCode(): Int { // overridden for ReduceNode return hashCode() } + + open fun diversity(): Int = 1 + children.fold(0) { acc, child -> acc + child.diversity() } } class PrimitiveNode( @@ -80,6 +82,8 @@ object PythonTree { override fun toString(): String { return repr } + + override fun diversity(): Int = 2 } class ListNode( diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt index 20aef61ae0..6a7b27d176 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt @@ -48,10 +48,14 @@ class InvalidExecution(val utError: UtError): FuzzingExecutionFeedback class TypeErrorFeedback(val message: String) : FuzzingExecutionFeedback class ArgumentsTypeErrorFeedback(val message: String) : FuzzingExecutionFeedback +data class PythonExecutionResult( + val fuzzingExecutionFeedback: FuzzingExecutionFeedback, + val fuzzingPlatformFeedback: PythonFeedback +) + data class PythonFeedback( override val control: Control = Control.CONTINUE, val result: Trie.Node = Trie.emptyNode(), - val executionFeedback: FuzzingExecutionFeedback? ) : Feedback class PythonFuzzedValue( From e866d2cb050dbc7bb154b751074043d5cbf20b31 Mon Sep 17 00:00:00 2001 From: Ekaterina Tochilina Date: Thu, 2 Mar 2023 10:57:14 +0300 Subject: [PATCH 21/32] Fixed cache selector --- utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index aa668586c6..f8f02494af 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -58,7 +58,7 @@ class PythonEngine( private fun addExecutionToCache(key: Pair>, result: PythonExecutionResult) { cache[key] = result if (cache.size > MAX_CACHE_SIZE) { - val elemToDelete = cache.keys.minBy { (_, args) -> + val elemToDelete = cache.keys.maxBy { (_, args) -> args.fold(0) { acc, arg -> acc + arg.tree.diversity() } } cache.remove(elemToDelete) From 323d88a6a6b966c9458d5381a5536b1af3e8bd88 Mon Sep 17 00:00:00 2001 From: Ekaterina Tochilina Date: Thu, 2 Mar 2023 11:33:50 +0300 Subject: [PATCH 22/32] Use chooseOne function from utbot-fuzzing --- .../main/kotlin/org/utbot/python/newtyping/utils/Utils.kt | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/utils/Utils.kt b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/utils/Utils.kt index 07bf61550d..0172127142 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/utils/Utils.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/utils/Utils.kt @@ -1,5 +1,6 @@ package org.utbot.python.newtyping.utils +import org.utbot.fuzzing.utils.chooseOne import kotlin.random.Random fun getOffsetLine(sourceFileContent: String, offset: Int): Int { @@ -8,9 +9,6 @@ fun getOffsetLine(sourceFileContent: String, offset: Int): Int { fun weightedRandom(elems: List, weights: List): T { val sum = weights.sum() - val borders = weights.fold(emptyList() to 0.0) { (list, partialSum), cur -> - (list + (partialSum + cur) / sum) to partialSum + cur - }.first - val value = Random.nextDouble() - return elems[borders.indexOfFirst { it >= value }] + val index = Random.chooseOne(weights.map { it / sum }.toDoubleArray()) + return elems[index] } \ No newline at end of file From 323cc37d0bc2e0bf18b9ac9d8efa3fa983a93347 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Fri, 3 Mar 2023 09:41:27 +0300 Subject: [PATCH 23/32] Added PythonWorkerManager to handle socket exceptions --- .../kotlin/org/utbot/python/PythonEngine.kt | 60 +++++-------- .../utbot/python/PythonTestCaseGenerator.kt | 6 +- .../python/evaluation/CodeEvaluationApi.kt | 2 + .../evaluation/PythonCodeExecutorImpl.kt | 2 + .../evaluation/PythonCodeSocketExecutor.kt | 20 ++++- .../utbot/python/evaluation/PythonWorker.kt | 8 +- .../python/evaluation/PythonWorkerManager.kt | 86 +++++++++++++++++++ .../src/main/resources/requirements.txt | 2 +- 8 files changed, 138 insertions(+), 48 deletions(-) create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index f8f02494af..9da335d5c1 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -6,7 +6,6 @@ import mu.KotlinLogging import org.utbot.framework.plugin.api.DocRegularStmt import org.utbot.framework.plugin.api.EnvironmentModels import org.utbot.framework.plugin.api.Instruction -import org.utbot.framework.plugin.api.TimeoutException import org.utbot.framework.plugin.api.UtError import org.utbot.framework.plugin.api.UtExecutionResult import org.utbot.framework.plugin.api.UtExecutionSuccess @@ -22,6 +21,7 @@ import org.utbot.python.evaluation.PythonEvaluationError import org.utbot.python.evaluation.PythonEvaluationSuccess import org.utbot.python.evaluation.PythonEvaluationTimeout import org.utbot.python.evaluation.PythonWorker +import org.utbot.python.evaluation.PythonWorkerManager import org.utbot.python.evaluation.serialiation.MemoryDump import org.utbot.python.evaluation.serialiation.toPythonTree import org.utbot.python.framework.api.python.PythonTreeModel @@ -33,11 +33,8 @@ import org.utbot.python.newtyping.pythonModules import org.utbot.python.newtyping.pythonTypeRepresentation import org.utbot.python.utils.TemporaryFileManager import org.utbot.python.utils.camelToSnakeCase -import org.utbot.python.utils.startProcess import org.utbot.summary.fuzzer.names.TestSuggestedInfo import java.net.ServerSocket -import java.net.SocketTimeoutException -import java.util.concurrent.TimeUnit private val logger = KotlinLogging.logger {} private const val MAX_CACHE_SIZE = 200 @@ -156,7 +153,7 @@ class PythonEngine( return ValidExecution(utFuzzedExecution) } - private fun constructEvaluationInput(pythonWorker: PythonWorker): PythonCodeExecutor { + fun constructEvaluationInput(pythonWorker: PythonWorker): PythonCodeExecutor { return PythonCodeSocketExecutor( methodUnderTest, moduleToImport, @@ -167,39 +164,19 @@ class PythonEngine( ) } - - fun fuzzing(parameters: List, isCancelled: () -> Boolean, until: Long): Flow = flow { val additionalModules = parameters.flatMap { it.pythonModules() } val coveredLines = initialCoveredLines.toMutableSet() - val logfile = TemporaryFileManager.createTemporaryFile("","utbot_executor.log", "log", true) ServerSocket(0).use { serverSocket -> logger.info { "Server port: ${serverSocket.localPort}" } - val processStartTime = System.currentTimeMillis() - val process = startProcess(listOf( + val manager = PythonWorkerManager( + serverSocket, pythonPath, - "-m", "utbot_executor", - "localhost", - serverSocket.localPort.toString(), - "--logfile", logfile.absolutePath, - "--loglevel", "DEBUG", - )) - val timeout = until - processStartTime - val workerSocket = try { - serverSocket.soTimeout = timeout.toInt() - serverSocket.accept() - } catch (e: SocketTimeoutException) { - val processHasExited = process.waitFor(timeout, TimeUnit.MILLISECONDS) - if (!processHasExited) { - process.destroy() - } - throw TimeoutException("Worker not connected") - } - logger.info { "Worker connected successfully" } - val pythonWorker = PythonWorker(workerSocket) - val codeExecutor = constructEvaluationInput(pythonWorker) - logger.info { "Executor was created successfully" } + until, + { constructEvaluationInput(it) } + ) + logger.info { "Executor manager was created successfully" } fun fuzzingResultHandler( description: PythonMethodDescription, @@ -224,8 +201,8 @@ class PythonEngine( modelList, methodUnderTest.argumentsNames ) - - return when (val evaluationResult = codeExecutor.run(functionArguments, localAdditionalModules)) { + val evaluationResult = manager.run(functionArguments, localAdditionalModules) + return when (evaluationResult) { is PythonEvaluationError -> { val utError = UtError( "Error evaluation: ${evaluationResult.status}, ${evaluationResult.message}", @@ -248,7 +225,7 @@ class PythonEngine( .zip(methodUnderTest.arguments) .mapNotNull { it.first.summary?.replace("%var%", it.second.name) } - val hasThisObject = codeExecutor.method.hasThisArgument + val hasThisObject = methodUnderTest.hasThisArgument when (val result = handleSuccessResult( parameters, @@ -259,11 +236,16 @@ class PythonEngine( )) { is ValidExecution -> { val trieNode: Trie.Node = description.tracer.add(coveredInstructions) - PythonExecutionResult(result, PythonFeedback(control = Control.CONTINUE, result = trieNode)) + PythonExecutionResult( + result, + PythonFeedback(control = Control.CONTINUE, result = trieNode) + ) } + is ArgumentsTypeErrorFeedback, is TypeErrorFeedback -> { PythonExecutionResult(result, PythonFeedback(control = Control.PASS)) } + is InvalidExecution -> { PythonExecutionResult(result, PythonFeedback(control = Control.CONTINUE)) } @@ -282,14 +264,17 @@ class PythonEngine( if (parameters.isEmpty()) { fuzzingResultHandler(pmd, emptyList()) + manager.disconnect() } else { PythonFuzzing(pmd.pythonTypeStorage) { description, arguments -> if (isCancelled()) { logger.info { "Fuzzing process was interrupted" } + manager.disconnect() return@PythonFuzzing PythonFeedback(control = Control.STOP) } if (System.currentTimeMillis() >= until) { logger.info { "Fuzzing process was interrupted by timeout" } + manager.disconnect() return@PythonFuzzing PythonFeedback(control = Control.STOP) } @@ -302,13 +287,10 @@ class PythonEngine( } val result = fuzzingResultHandler(description, arguments) addExecutionToCache(pair, result) - emit(result.fuzzingExecutionFeedback) return@PythonFuzzing result.fuzzingPlatformFeedback + }.fuzz(pmd) - if (codeExecutor is PythonCodeSocketExecutor) { - codeExecutor.stop() - } } } } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt index 5355e19fd7..267a0b2979 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt @@ -236,7 +236,7 @@ class PythonTestCaseGenerator( if (coveredAfter == coveredBefore) { coverageLimit-- } - logger.info { "${System.currentTimeMillis() - startTime}: $generated, $missingLines" } + logger.info { "Time ${System.currentTimeMillis() - startTime}: $generated, $missingLines" } coveredBefore = coveredAfter } feedback @@ -310,12 +310,12 @@ class PythonTestCaseGenerator( return@breaking } + algo.run(hintCollector.result, isCancelled, annotationHandler) + val existsAnnotation = method.definition.type if (existsAnnotation.arguments.all { it.pythonTypeName() != "typing.Any" }) { annotationHandler(existsAnnotation) } - - algo.run(hintCollector.result, isCancelled, annotationHandler) } } } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt index 4e3534cefa..faa5f516a9 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt @@ -16,6 +16,8 @@ interface PythonCodeExecutor { fuzzedValues: FunctionArguments, additionalModulesToImport: Set ): PythonEvaluationResult + + fun stop() } sealed class PythonEvaluationResult diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeExecutorImpl.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeExecutorImpl.kt index bd5f59c16b..1137bb1b15 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeExecutorImpl.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeExecutorImpl.kt @@ -40,6 +40,8 @@ class PythonCodeExecutorImpl( return getEvaluationResult(evaluationFiles) } + override fun stop() {} + private fun generateExecutionCode( additionalModulesToImport: Set, methodArguments: List, diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt index 8bf3b0e930..dc7bd8219f 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt @@ -1,5 +1,6 @@ package org.utbot.python.evaluation +import mu.KotlinLogging import org.utbot.framework.plugin.api.Coverage import org.utbot.framework.plugin.api.Instruction import org.utbot.python.FunctionArguments @@ -14,7 +15,9 @@ import org.utbot.python.evaluation.serialiation.serializeObjects import org.utbot.python.framework.api.python.util.pythonAnyClassId import org.utbot.python.newtyping.pythonTypeName import org.utbot.python.newtyping.pythonTypeRepresentation +import java.net.SocketException +private val logger = KotlinLogging.logger {} class PythonCodeSocketExecutor( override val method: PythonMethod, @@ -67,9 +70,20 @@ class PythonCodeSocketExecutor( method.moduleFilename, ) val message = ExecutionRequestSerializer.serializeRequest(request) ?: error("Cannot serialize request to python executor") - pythonWorker.sendData(message) + try { + pythonWorker.sendData(message) + } catch (_: SocketException) { + logger.info { "Send data error" } + return parseExecutionResult(FailExecution("Send data error")) + } val response = pythonWorker.receiveMessage() - val executionResult = ExecutionResultDeserializer.parseExecutionResult(response) ?: error("Cannot parse execution result: $response") + val executionResult = if (response == null) { + logger.info { "Response error" } + FailExecution("Execution result error") + } else { + ExecutionResultDeserializer.parseExecutionResult(response) + ?: error("Cannot parse execution result: $response") + } return parseExecutionResult(executionResult) } @@ -123,7 +137,7 @@ class PythonCodeSocketExecutor( ) } - fun stop() { + override fun stop() { pythonWorker.stopServer() } } \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorker.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorker.kt index 6dff965b9d..5c7911d9d9 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorker.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorker.kt @@ -28,8 +28,12 @@ class PythonWorker( outStream.flush() } - fun receiveMessage(): String { - val length = inStream.readLine().toInt() + fun receiveMessage(): String? { + val lengthLine = inStream.readLine() + if (lengthLine == null) { + return null + } + val length = lengthLine.toInt() val buffer = CharArray(length) inStream.read(buffer) return String(buffer) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt new file mode 100644 index 0000000000..877ba1fa6f --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt @@ -0,0 +1,86 @@ +package org.utbot.python.evaluation + +import mu.KotlinLogging +import org.utbot.framework.plugin.api.TimeoutException +import org.utbot.python.FunctionArguments +import org.utbot.python.utils.TemporaryFileManager +import org.utbot.python.utils.startProcess +import java.net.ServerSocket +import java.net.Socket +import java.net.SocketTimeoutException +import java.util.concurrent.TimeUnit + +private val logger = KotlinLogging.logger {} + +class PythonWorkerManager( + val serverSocket: ServerSocket, + val pythonPath: String, + val until: Long, + val pythonCodeExecutorConstructor: (PythonWorker) -> PythonCodeExecutor +) { + val logfile = TemporaryFileManager.createTemporaryFile("","utbot_executor.log", "log", true) + + var timeout: Long = 0 + lateinit var process: Process + lateinit var workerSocket: Socket + lateinit var codeExecutor: PythonCodeExecutor + + init { + connect() + } + + fun connect() { + val processStartTime = System.currentTimeMillis() + process = startProcess(listOf( + pythonPath, + "-m", "utbot_executor", + "localhost", + serverSocket.localPort.toString(), + "--logfile", logfile.absolutePath, + "--loglevel", "DEBUG", + )) + timeout = until - processStartTime + workerSocket = try { + serverSocket.soTimeout = timeout.toInt() + serverSocket.accept() + } catch (e: SocketTimeoutException) { + val processHasExited = process.waitFor(timeout, TimeUnit.MILLISECONDS) + if (!processHasExited) { + process.destroy() + } + throw TimeoutException("Worker not connected") + } + logger.debug { "Worker connected successfully" } + + workerSocket.soTimeout = 2000 + val pythonWorker = PythonWorker(workerSocket) + codeExecutor = pythonCodeExecutorConstructor(pythonWorker) + } + + fun disconnect() { + workerSocket.close() + process.destroy() + } + + fun reconnect() { + disconnect() + connect() + } + + fun run( + fuzzedValues: FunctionArguments, + additionalModulesToImport: Set + ): PythonEvaluationResult { + val evaluationResult = try { + codeExecutor.run(fuzzedValues, additionalModulesToImport) + } catch (_: SocketTimeoutException) { + logger.debug { "Socket timeout" } + reconnect() + PythonEvaluationTimeout() + } + if (evaluationResult is PythonEvaluationError) { + reconnect() + } + return evaluationResult + } +} \ No newline at end of file diff --git a/utbot-python/src/main/resources/requirements.txt b/utbot-python/src/main/resources/requirements.txt index a8dd4498fb..f760678e76 100644 --- a/utbot-python/src/main/resources/requirements.txt +++ b/utbot-python/src/main/resources/requirements.txt @@ -1,4 +1,4 @@ mypy==1.0.0 coverage -utbot-executor==1.1.11 +utbot-executor==1.1.24 utbot-mypy-runner==0.2.8 From a5bde7576e74b1499a2f43874efd8ca4b39680da Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Fri, 3 Mar 2023 10:06:00 +0300 Subject: [PATCH 24/32] Clear preprocessed_values.json --- .../main/resources/preprocessed_values.json | 450 ------------------ 1 file changed, 450 deletions(-) diff --git a/utbot-python/src/main/resources/preprocessed_values.json b/utbot-python/src/main/resources/preprocessed_values.json index 03f768cfd9..19f47bdf4d 100644 --- a/utbot-python/src/main/resources/preprocessed_values.json +++ b/utbot-python/src/main/resources/preprocessed_values.json @@ -1,474 +1,24 @@ [ - { - "name": "builtins.int", - "instances": [ - "-1", - "-3", - "0", - "1", - "10", - "100", - "1144132807", - "123", - "1267650600228229401496703205376", - "1431655765", - "23", - "291", - "3", - "314", - "4", - "4294967296", - "4294967297", - "4886718345", - "83", - "9" - ] - }, - { - "name": "builtins.bool", - "instances": [ - "True", - "False" - ] - }, - { - "name": "builtins.str", - "instances": [ - "''", - "'1001'", - "'3'", - "'3j'", - "'500'", - "'abcdefghijklmnopqrst'", - "'foo'", - "'global'", - "'inf'", - "'pythön.org.'", - "'strings are converted to unicode'", - "'unicode remains unicode'", - "'€'" - ] - }, { "name": "builtins.float", "instances": [ - "-1.0", - "-2.1", - "-2100.0", - "-2147483647.0", - "-2147483648.0", - "-7.3", - "-9.223372036854776e+18", "float('-inf')", - "0.0", - "0.5", - "1.0", - "1.23e+300", - "1.4", - "1.8446744073709552e+19", - "17179869184.0", - "1970.0", - "1e+23", - "2.1", - "2.5", - "2147483647.0", - "2147483648.0", - "3.14", - "3000.0", - "314.0", - "3200.0", - "4294967296.0", - "7.3", - "9.223372036854776e+18", "float('inf')", "float('nan')" ] }, - { - "name": "builtins.range", - "instances": [ - "range(127, 256)", - "range(24)", - "range(0, 3)", - "range(-10, 10)", - "range(10, -11, -1)", - "range(240)", - "range(200)", - "range(50, 400)", - "range(150)", - "range(9, -1, -2)", - "range(4, 16)", - "range(0, 55296)", - "range(101)", - "range(5000)", - "range(65536, 1114112)", - "range(1500)", - "range(1, 9)", - "range(512)", - "range(0, -20, -1)", - "range(32, 127)", - "range(52, 64)", - "range(70)" - ] - }, - { - "name": "builtins.complex", - "instances": [ - "complex(1.0, float('inf'))", - "complex(0, 1)", - "complex(1.0, 10.0)", - "complex(0.0, 3.14)", - "complex(1, 2)", - "complex(float('inf'), float('inf'))", - "complex(float('inf'), -1)", - "complex(0.0, float('nan'))", - "complex(10.0)", - "complex(float('nan'), 1)", - "complex(0, 0)", - "complex(1.3, 2.2)", - "complex(3.14, 0.0)", - "complex(float('nan'), -1)", - "complex(0.0, -float('inf'))", - "complex(-1e500, 1.8e308')", - "complex(float('inf'), 0)", - "complex(0, float('nan'))", - "complex(1e500, 0)", - "complex(0, -1e-500j)", - "complex(10, 0)", - "complex(1, 10)", - "complex(3.14, 0.0)", - "complex(1, float('inf'))", - "complex(314, 0)", - "complex(-1e-500, 1e-500')", - "complex(3.14, 0.0)", - "complex(-float('inf'), float('inf'))", - "complex(5.3, 9.8)", - "complex(1.0, -float('inf'))", - "complex(0, -1e500)", - "complex(0.0, 3.14)", - "complex(0.0, -1.0)", - "complex(1e-200, 1e-200)", - "complex(314)", - "complex(-0.0, -1.0)", - "complex(float('inf'), 0.0)", - "complex(0, -float('inf'))", - "complex()", - "complex(0.0, 1.0)", - "complex(1, 10)", - "complex(0, float('inf'))", - "complex(float('nan'), float('nan'))", - "complex(real=17, imag=23)", - "complex(1e+200, 1e+200)", - "complex(0.0, 3.0)", - "complex(1.0, 10)", - "complex(-0.0, 2.0)", - "complex(float('inf'), 1)", - "complex(real=1, imag=3)", - "complex(1, 10.0)", - "complex(1, float('nan'))", - "complex(0.0, 3.14)", - "complex(0.0, 3.14)", - "complex(1.0, -0.0)" - ] - }, { "name": "builtins.BaseException", "instances": [ "BaseException()" ] }, - { - "name": "types.NoneType", - "instances": [ - "None" - ] - }, - { - "name": "builtins.bytearray", - "instances": [ - "bytearray(b'a')", - "bytearray(100)", - "bytearray(b'\\x00' * 100)", - "bytearray(range(1, 10))", - "bytearray(b'\\x07\\x7f\\x7f')", - "bytearray(b'mutable')", - "bytearray([1, 2])", - "bytearray(range(256))", - "bytearray(b'hell')", - "bytearray([5, 6, 7, 8, 9])", - "bytearray(b'memoryview')", - "bytearray(b'a:b::c:::d')", - "bytearray(b'b')", - "bytearray(b'cd')", - "bytearray(b'world')", - "bytearray(b'[emoryvie]')", - "bytearray(b'x' * 5)", - "bytearray([0, 1, 254, 255])", - "bytearray(b'*$')", - "bytearray(b'abc\\xe9\\x00')", - "bytearray(b'a\\xffb')", - "bytearray(range(100))", - "bytearray(b'[abracadabra]')", - "bytearray([0, 1, 2, 100, 101, 7, 8, 9])", - "bytearray(b'Mary')", - "bytearray(b'baz')", - "bytearray(b'\\xff')", - "bytearray(128 * 1024)", - "bytearray([100, 101])", - "bytearray(b'1')", - "bytearray([1, 2, 3])", - "bytearray(b'')", - "bytearray(b'one')", - "bytearray(b'nul:\\x00')", - "bytearray(1024)", - "bytearray(10)", - "bytearray(b'abcdefgh')", - "bytearray(b'123')", - "bytearray(b'\\x80')", - "bytearray(b'01 had a 9')", - "bytearray(2)", - "bytearray(range(10))", - "bytearray(b'a\\x80b')", - "bytearray(b':a:b::c')", - "bytearray(b'hash this!')", - "bytearray(b'bar')", - "bytearray(b'----')", - "bytearray(b'bytearray')", - "bytearray(b'spam')", - "bytearray(b'abcdefghijk')", - "bytearray(b'msssspp')", - "bytearray(b'no error')", - "bytearray(b'YWJj\\n')", - "bytearray(b'foobaz')", - "bytearray(b'little lamb---')", - "bytearray(b'abc\\xe9\\x00xxx')", - "bytearray(b'def')", - "bytearray(b'eggs\\n')", - "bytearray(b'foo')", - "bytearray(b'foobar')", - "bytearray(128)", - "bytearray(b'key')", - "bytearray(16)", - "bytearray(b'file.py')", - "bytearray(b'ab')", - "bytearray(b'this is a random bytearray object')", - "bytearray(b' world\\n\\n\\n')", - "bytearray(5)", - "bytearray(b'\\x00python\\x00test\\x00')", - "bytearray(9)", - "bytearray(b'abcde')", - "bytearray(b'x')", - "bytearray(b'0123456789')", - "bytearray(b'python')", - "bytearray(8192)", - "bytearray(b'\\xff\\x00\\x00')", - "bytearray(b'hello1')", - "bytearray(b'xyz')", - "bytearray(b'\\xaaU\\xaaU')", - "bytearray(b'Z')", - "bytearray(8)", - "bytearray(b'ghi')", - "bytearray(b'[ytearra]')", - "bytearray(b'abc')", - "bytearray(b'this is a test')", - "bytearray(b'xxx')", - "bytearray()", - "bytearray(b'abcdefghijklmnopqrstuvwxyz')", - "bytearray(b'my dog has fleas')", - "bytearray(b'g\\xfcrk')", - "bytearray(b'hello world')", - "bytearray(b'0102abcdef')", - "bytearray(1)", - "bytearray(b'--------------')", - "bytearray(20)", - "bytearray(b'hello')" - ] - }, - { - "name": "builtins.bytes", - "instances": [ - "b'ab'", - "b'def'", - "b'abc'", - "b'\\x80\\x81'", - "b'Hello world\\n\\x80\\x81\\xfe\\xff'", - "b'3'", - "b'2'" - ] - }, - { - "name": "builtins.dict", - "instances": [ - "dict()" - ] - }, - { - "name": "builtins.frozenset", - "instances": [ - "frozenset()" - ] - }, - { - "name": "builtins.list", - "instances": [ - "list()" - ] - }, - { - "name": "builtins.memoryview", - "instances": [ - "memoryview(b'a')", - "memoryview(b'1234')", - "memoryview(b'ax = 123')", - "memoryview(b'$23$')", - "memoryview(bytes(range(256)))", - "memoryview(b'hash this!')", - "memoryview(b'12.3')", - "memoryview(b'bytes')", - "memoryview(b'a:b::c:::d')", - "memoryview(b'ab')", - "memoryview(b'123')", - "memoryview(b'ac')", - "memoryview(b'YWJj\\n')", - "memoryview(b'')", - "memoryview(b'*$')", - "memoryview(b'spam\\n')", - "memoryview(b'\\xff\\x00\\x00')", - "memoryview(b' ')", - "memoryview(b'abc')", - "memoryview(b'12.3A')", - "memoryview(b':a:b::c')", - "memoryview(b'foo')", - "memoryview(b'\\x124Vx')", - "memoryview(b'[abracadabra]')", - "memoryview(b'123 ')", - "memoryview(b'spam')", - "memoryview(b'text')", - "memoryview(b'memoryview')", - "memoryview(b'123\\x00')", - "memoryview(b'12.3\\x00')", - "memoryview(b'character buffers are decoded to unicode')", - "memoryview(b'xyz')", - "memoryview(b'12.3 ')", - "memoryview(b'cd')", - "memoryview(b'baz')", - "memoryview(b'0102abcdef')", - "memoryview(b'123A')", - "memoryview(b'\\x07\\x7f\\x7f')", - "memoryview(b'file.py')", - "memoryview(b'\\x1a+0')", - "memoryview(b'12.34')" - ] - }, { "name": "builtins.object", "instances": [ "object()" ] }, - { - "name": "builtins.set", - "instances": [ - "set()" - ] - }, - { - "name": "builtins.tuple", - "instances": [ - "tuple()" - ] - }, - { - "name": "builtins.slice", - "instances": [ - "slice(0, 2, 1)", - "slice(2)", - "slice(1, 3)", - "slice(4)", - "slice(0, 10)", - "slice(0, 1, 1)", - "slice(0, 1, 2)", - "slice(0, 2)", - "slice(1, 1)", - "slice(0)", - "slice(0, 1, 2)", - "slice(0, 2, 1)", - "slice(3, 5, 1)", - "slice(0, 1, 1)", - "slice(0, 1, 0)", - "slice(0, 8, 1)", - "slice(0, 2, 0)", - "slice(1, 18, 2)", - "slice(1)", - "slice(0, 10, 1)", - "slice(None, 10, -1)", - "slice(None, -10)", - "slice(None, -11, -1)", - "slice(None, 9)", - "slice(100, -100, -1)", - "slice(None, 10)", - "slice(None)", - "slice(-100, 100)", - "slice(None, None, -1)", - "slice(None, -9)", - "slice(None, 9, -1)", - "slice(0.0, 10, 1)", - "slice(0, 10, 0)", - "slice(10, 20, 3)", - "slice(0, 10, 1.0)", - "slice(1, 2, 4)", - "slice(1, 2)", - "slice(None, None, -2)", - "slice(-100, 100, 2)", - "slice(0, 10.0, 1)", - "slice(3, None, -2)", - "slice(1, None, 2)", - "slice(None, -10, -1)", - "slice(None, 11)", - "slice(1, 2, 3)", - "slice(None, -12, -1)", - "slice(5)", - "slice(None, 8, -1)", - "slice(None, -11)", - "slice(None, None, 2)", - "slice(0, 10, 2)", - "slice(2, 3)", - "slice(0, 10)", - "slice(0, 1, 5)", - "slice(0, 10, 0)", - "slice(2, 10, 3)", - "slice(0, 2)", - "slice(1, 3)", - "slice(1, 2)", - "slice(2, 2)", - "slice(0, 1)", - "slice(0, 0)", - "slice(2, 1)", - "slice(2000, 1000)", - "slice(0, 1000)", - "slice(0, 3)", - "slice(1000, 1000)", - "slice(2, 4)", - "slice(1, 2)", - "slice(1, 2, 3)", - "slice(0, 1)", - "slice(0, 0)", - "slice(2, 3)", - "slice(None, 42)", - "slice(None, 24, None)", - "slice(2, 1024, 10)", - "slice(None, 42, None)", - "slice(0, 2)", - "slice(1, 2)", - "slice(0, 1)", - "slice(3, 5)", - "slice(0, 10, 0)", - "slice(0, 3)", - "slice(1, 10, 2)", - "slice(2, 2, 2)", - "slice(1, 1, 1)" - ] - }, { "name": "builtins.type", "instances": [ From 0da909deb98794d01f735b6ef19e811cb2f3be22 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Fri, 3 Mar 2023 10:10:18 +0300 Subject: [PATCH 25/32] Update string constants --- .../org/utbot/python/fuzzing/provider/StrValueProvider.kt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/StrValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/StrValueProvider.kt index 4769aef332..04c39449ca 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/StrValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/StrValueProvider.kt @@ -29,7 +29,9 @@ object StrValueProvider : ValueProvider Date: Fri, 3 Mar 2023 11:32:40 +0300 Subject: [PATCH 26/32] Update requirements --- utbot-python/src/main/resources/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utbot-python/src/main/resources/requirements.txt b/utbot-python/src/main/resources/requirements.txt index f760678e76..4296e8db6a 100644 --- a/utbot-python/src/main/resources/requirements.txt +++ b/utbot-python/src/main/resources/requirements.txt @@ -1,4 +1,4 @@ mypy==1.0.0 coverage -utbot-executor==1.1.24 +utbot-executor==1.1.25 utbot-mypy-runner==0.2.8 From 573733d80bf841c10447edf87a490ed7a12ddcaa Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Fri, 3 Mar 2023 11:39:02 +0300 Subject: [PATCH 27/32] Update requirements --- utbot-python/src/main/resources/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utbot-python/src/main/resources/requirements.txt b/utbot-python/src/main/resources/requirements.txt index 4296e8db6a..c2fb069681 100644 --- a/utbot-python/src/main/resources/requirements.txt +++ b/utbot-python/src/main/resources/requirements.txt @@ -1,4 +1,4 @@ mypy==1.0.0 coverage -utbot-executor==1.1.25 +utbot-executor==1.1.26 utbot-mypy-runner==0.2.8 From 2f576b654b514e27054572c8e90faa5ad7a6a88e Mon Sep 17 00:00:00 2001 From: Ekaterina Tochilina Date: Fri, 3 Mar 2023 11:08:57 +0300 Subject: [PATCH 28/32] Better handling of utbot_executor exceptions --- utbot-python/samples/easy_samples/boruvka.py | 2 +- .../kotlin/org/utbot/python/PythonEngine.kt | 5 +-- .../utbot/python/PythonTestCaseGenerator.kt | 31 +------------ .../python/PythonTestGenerationProcessor.kt | 44 +++++++++++++++++-- .../python/evaluation/PythonWorkerManager.kt | 10 ++--- 5 files changed, 50 insertions(+), 42 deletions(-) diff --git a/utbot-python/samples/easy_samples/boruvka.py b/utbot-python/samples/easy_samples/boruvka.py index cb6f34205b..8f406b6012 100644 --- a/utbot-python/samples/easy_samples/boruvka.py +++ b/utbot-python/samples/easy_samples/boruvka.py @@ -58,7 +58,7 @@ def boruvka(self) -> None: component_size = [] mst_weight = 0 - minimum_weight_edge: list[Any] = [-1] * self.m_num_of_nodes + minimum_weight_edge: list = [-1] * self.m_num_of_nodes # A list of components (initialized to all of the nodes) for node in range(self.m_num_of_nodes): diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index 9da335d5c1..493d7a4717 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -173,9 +173,8 @@ class PythonEngine( val manager = PythonWorkerManager( serverSocket, pythonPath, - until, - { constructEvaluationInput(it) } - ) + until + ) { constructEvaluationInput(it) } logger.info { "Executor manager was created successfully" } fun fuzzingResultHandler( diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt index 267a0b2979..9215bff3c2 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt @@ -2,12 +2,10 @@ package org.utbot.python import kotlinx.coroutines.runBlocking import mu.KotlinLogging -import org.parsers.python.PythonParser import org.utbot.framework.minimization.minimizeExecutions import org.utbot.framework.plugin.api.UtError import org.utbot.framework.plugin.api.UtExecution import org.utbot.framework.plugin.api.UtExecutionSuccess -import org.utbot.python.code.PythonCode import org.utbot.python.fuzzing.* import org.utbot.python.newtyping.* import org.utbot.python.newtyping.ast.visitor.Visitor @@ -49,32 +47,6 @@ class PythonTestCaseGenerator( private val storageForMypyMessages: MutableList = mutableListOf() - private fun findMethodByDescription(mypyStorage: MypyAnnotationStorage, method: PythonMethodHeader): PythonMethod { - var containingClass: CompositeType? = null - val containingClassName = method.containingPythonClassId?.simpleName - val functionDef = if (containingClassName == null) { - mypyStorage.definitions[curModule]!![method.name]!!.getUtBotDefinition()!! - } else { - containingClass = - mypyStorage.definitions[curModule]!![containingClassName]!!.getUtBotType() as CompositeType - mypyStorage.definitions[curModule]!![containingClassName]!!.type.asUtBotType.getPythonAttributes().first { - it.meta.name == method.name - } - } as? PythonFunctionDefinition ?: error("Selected method is not a function definition") - - val parsedFile = PythonParser(sourceFileContent).Module() - val funcDef = PythonCode.findFunctionDefinition(parsedFile, method) - - return PythonMethod( - name = method.name, - moduleFilename = method.moduleFilename, - containingPythonClass = containingClass, - codeAsString = funcDef.body.source, - definition = functionDef, - ast = funcDef.body - ) - } - private fun constructCollectors( mypyStorage: MypyAnnotationStorage, typeStorage: PythonTypeStorage, @@ -146,11 +118,10 @@ class PythonTestCaseGenerator( }.take(maxSubstitutions) } - fun generate(methodDescription: PythonMethodHeader, until: Long): PythonTestSet { + fun generate(method: PythonMethod, until: Long): PythonTestSet { storageForMypyMessages.clear() val typeStorage = PythonTypeStorage.get(mypyStorage) - val method = findMethodByDescription(mypyStorage, methodDescription) val (hintCollector, constantCollector) = constructCollectors(mypyStorage, typeStorage, method) val constants = constantCollector.result.map { (type, value) -> diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt index bd9cc7f2c1..4f9a8f1fa6 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt @@ -3,6 +3,7 @@ package org.utbot.python import com.squareup.moshi.Moshi import com.squareup.moshi.kotlin.reflect.KotlinJsonAdapterFactory import mu.KotlinLogging +import org.parsers.python.PythonParser import org.utbot.python.framework.codegen.model.PythonSysPathImport import org.utbot.python.framework.codegen.model.PythonSystemImport import org.utbot.python.framework.codegen.model.PythonUserImport @@ -14,12 +15,17 @@ import org.utbot.framework.plugin.api.UtClusterInfo import org.utbot.framework.plugin.api.UtExecutionSuccess import org.utbot.framework.plugin.api.util.UtContext import org.utbot.framework.plugin.api.util.withUtContext +import org.utbot.python.code.PythonCode import org.utbot.python.framework.api.python.PythonClassId import org.utbot.python.framework.api.python.PythonMethodId import org.utbot.python.framework.api.python.PythonModel import org.utbot.python.framework.api.python.RawPythonAnnotation import org.utbot.python.framework.api.python.util.pythonAnyClassId import org.utbot.python.framework.codegen.model.PythonCodeGenerator +import org.utbot.python.newtyping.PythonFunctionDefinition +import org.utbot.python.newtyping.general.CompositeType +import org.utbot.python.newtyping.getPythonAttributes +import org.utbot.python.newtyping.mypy.MypyAnnotationStorage import org.utbot.python.newtyping.mypy.readMypyAnnotationStorageAndInitialErrors import org.utbot.python.newtyping.mypy.setConfigFile import org.utbot.python.typing.MypyAnnotations @@ -110,16 +116,17 @@ object PythonTestGenerationProcessor { ) val until = startTime + timeout - val tests = pythonMethods.mapIndexed { index, method -> + val tests = pythonMethods.mapIndexed { index, methodHeader -> val methodsLeft = pythonMethods.size - index val localUntil = (until - System.currentTimeMillis()) / methodsLeft + System.currentTimeMillis() + val method = findMethodByHeader(mypyStorage, methodHeader, currentPythonModule, pythonFileContent) try { testCaseGenerator.generate(method, localUntil) } catch (_: TimeoutException) { logger.warn { "Cannot connect to python code executor for method ${method.name}" } - null + PythonTestSet(method, emptyList(), emptyList(), emptyList()) } - }.filterNotNull() + } val (notEmptyTests, emptyTestSets) = tests.partition { it.executions.isNotEmpty() } @@ -238,6 +245,37 @@ object PythonTestGenerationProcessor { } } + private fun findMethodByHeader( + mypyStorage: MypyAnnotationStorage, + method: PythonMethodHeader, + curModule: String, + sourceFileContent: String + ): PythonMethod { + var containingClass: CompositeType? = null + val containingClassName = method.containingPythonClassId?.simpleName + val functionDef = if (containingClassName == null) { + mypyStorage.definitions[curModule]!![method.name]!!.getUtBotDefinition()!! + } else { + containingClass = + mypyStorage.definitions[curModule]!![containingClassName]!!.getUtBotType() as CompositeType + mypyStorage.definitions[curModule]!![containingClassName]!!.type.asUtBotType.getPythonAttributes().first { + it.meta.name == method.name + } + } as? PythonFunctionDefinition ?: error("Selected method is not a function definition") + + val parsedFile = PythonParser(sourceFileContent).Module() + val funcDef = PythonCode.findFunctionDefinition(parsedFile, method) + + return PythonMethod( + name = method.name, + moduleFilename = method.moduleFilename, + containingPythonClass = containingClass, + codeAsString = funcDef.body.source, + definition = functionDef, + ast = funcDef.body + ) + } + enum class MissingRequirementsActionResult { INSTALLED, NOT_INSTALLED } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt index 877ba1fa6f..699babb3da 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt @@ -4,6 +4,7 @@ import mu.KotlinLogging import org.utbot.framework.plugin.api.TimeoutException import org.utbot.python.FunctionArguments import org.utbot.python.utils.TemporaryFileManager +import org.utbot.python.utils.getResult import org.utbot.python.utils.startProcess import java.net.ServerSocket import java.net.Socket @@ -37,17 +38,16 @@ class PythonWorkerManager( "localhost", serverSocket.localPort.toString(), "--logfile", logfile.absolutePath, - "--loglevel", "DEBUG", + //"--loglevel", "DEBUG", )) timeout = until - processStartTime workerSocket = try { serverSocket.soTimeout = timeout.toInt() serverSocket.accept() } catch (e: SocketTimeoutException) { - val processHasExited = process.waitFor(timeout, TimeUnit.MILLISECONDS) - if (!processHasExited) { - process.destroy() - } + val result = getResult(process, 10) + logger.info("utbot_executor exit value: ${result.exitValue}. stderr: ${result.stderr}.") + process.destroy() throw TimeoutException("Worker not connected") } logger.debug { "Worker connected successfully" } From db50188100c8a4f9221848d9c20ad41016ad79bc Mon Sep 17 00:00:00 2001 From: Ekaterina Tochilina Date: Fri, 3 Mar 2023 11:52:32 +0300 Subject: [PATCH 29/32] Memorization of recursive objects --- .../kotlin/org/utbot/python/PythonEngine.kt | 2 +- .../python/evaluation/PythonWorkerManager.kt | 7 +- .../python/framework/api/python/PythonTree.kt | 143 +++++++----------- 3 files changed, 57 insertions(+), 95 deletions(-) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index 493d7a4717..6d1e32a0d1 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -56,7 +56,7 @@ class PythonEngine( cache[key] = result if (cache.size > MAX_CACHE_SIZE) { val elemToDelete = cache.keys.maxBy { (_, args) -> - args.fold(0) { acc, arg -> acc + arg.tree.diversity() } + args.fold(0) { acc, arg -> arg.commonDiversity(acc) } } cache.remove(elemToDelete) } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt index 699babb3da..d77b34df01 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt @@ -6,10 +6,10 @@ import org.utbot.python.FunctionArguments import org.utbot.python.utils.TemporaryFileManager import org.utbot.python.utils.getResult import org.utbot.python.utils.startProcess +import java.lang.Long.max import java.net.ServerSocket import java.net.Socket import java.net.SocketTimeoutException -import java.util.concurrent.TimeUnit private val logger = KotlinLogging.logger {} @@ -40,12 +40,13 @@ class PythonWorkerManager( "--logfile", logfile.absolutePath, //"--loglevel", "DEBUG", )) - timeout = until - processStartTime + timeout = max(until - processStartTime, 0) workerSocket = try { serverSocket.soTimeout = timeout.toInt() serverSocket.accept() } catch (e: SocketTimeoutException) { - val result = getResult(process, 10) + timeout = max(until - processStartTime, 0) + val result = getResult(process, timeout) logger.info("utbot_executor exit value: ${result.exitValue}. stderr: ${result.stderr}.") process.destroy() throw TimeoutException("Worker not connected") diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt index 76a488aea5..e07015daab 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt @@ -15,6 +15,17 @@ import java.util.concurrent.atomic.AtomicLong object PythonTree { + fun isRecursiveObject(tree: PythonTreeNode): Boolean { + return isRecursiveObjectDFS(tree, mutableSetOf()) + } + + private fun isRecursiveObjectDFS(tree: PythonTreeNode, visited: MutableSet): Boolean { + if (visited.contains(tree)) + return true + visited.add(tree) + return tree.children.any { isRecursiveObjectDFS(it, visited) } + } + open class PythonTreeNode( val id: Long, val type: PythonClassId, @@ -39,25 +50,23 @@ object PythonTree { if (other !is PythonTreeNode) { return false } - return type == other.type && children == other.children && comparable == other.comparable + return id == other.id } override fun hashCode(): Int { - var result = type.hashCode() - result = 31 * result + comparable.hashCode() - result = 31 * result + children.hashCode() - return result + return id.hashCode() } - open fun softEquals(other: PythonTreeNode): Boolean { // overridden for ReduceNode - return this == other + open fun softEquals(other: PythonTreeNode): Boolean { // must be called only from PythonTreeWrapper! + return type == other.type && children == other.children } - open fun softHashCode(): Int { // overridden for ReduceNode - return hashCode() + open fun softHashCode(): Int { // must be called only from PythonTreeWrapper! + return type.hashCode() * 31 + children.hashCode() } - open fun diversity(): Int = 1 + children.fold(0) { acc, child -> acc + child.diversity() } + open fun diversity(): Int = // must be called only from PythonTreeWrapper! + 1 + children.fold(0) { acc, child -> acc + child.diversity() } } class PrimitiveNode( @@ -66,21 +75,29 @@ object PythonTree { val repr: String, ) : PythonTreeNode(id, type) { constructor(type: PythonClassId, repr: String) : this(PythonIdGenerator.getOrCreateIdForValue(repr), type, repr) + + override fun toString(): String { + return repr + } + override fun equals(other: Any?): Boolean { - if (other !is PrimitiveNode) { + if (other !is PrimitiveNode) return false - } - return repr == other.repr && type == other.type + return repr == other.repr } override fun hashCode(): Int { - var result = super.hashCode() - result = 31 * result + repr.hashCode() - return result + return repr.hashCode() } - override fun toString(): String { - return repr + override fun softEquals(other: PythonTreeNode): Boolean { + if (other !is PrimitiveNode) + return false + return repr == other.repr + } + + override fun softHashCode(): Int { + return repr.hashCode() } override fun diversity(): Int = 2 @@ -102,19 +119,6 @@ object PythonTree { } else false } - - override fun equals(other: Any?): Boolean { - if (other !is ListNode) { - return false - } - return children == other.children - } - - override fun hashCode(): Int { - var result = super.hashCode() - result = 31 * result + items.hashCode() - return result - } } class DictNode( @@ -135,19 +139,6 @@ object PythonTree { } else false } - override fun equals(other: Any?): Boolean { - if (other !is DictNode) { - return false - } - return children == other.children - } - - override fun hashCode(): Int { - var result = super.hashCode() - result = 31 * result + items.hashCode() - return result - } - } class SetNode( @@ -171,20 +162,6 @@ object PythonTree { false } } - - override fun equals(other: Any?): Boolean { - if (other !is SetNode) { - return false - } - return items == other.items - } - - override fun hashCode(): Int { - var result = super.hashCode() - result = 31 * result + items.hashCode() - return result - } - } class TupleNode( @@ -205,19 +182,6 @@ object PythonTree { false } } - - override fun equals(other: Any?): Boolean { - if (other !is TupleNode) { - return false - } - return items == other.items - } - - override fun hashCode(): Int { - var result = super.hashCode() - result = 31 * result + items.hashCode() - return result - } } class ReduceNode( @@ -250,26 +214,7 @@ object PythonTree { } else false } - override fun equals(other: Any?): Boolean { - if (other !is ReduceNode) { - return false - } - return type == other.type && - id == other.id && - constructor == other.constructor && - args == other.args && - state == other.state && - listitems == other.listitems && - dictitems == other.dictitems - } - - override fun hashCode(): Int { - var result = softHashCode() - result = 31 * result + id.hashCode() - return result - } - - override fun softEquals(other: PythonTreeNode): Boolean { // like equals(), but skip id check + override fun softEquals(other: PythonTreeNode): Boolean { if (other !is ReduceNode) return false return type == other.type && constructor == other.constructor && args == other.args && @@ -380,10 +325,26 @@ class PythonTreeWrapper(val tree: PythonTree.PythonTreeNode) { override fun equals(other: Any?): Boolean { if (other !is PythonTreeWrapper) return false + if (PythonTree.isRecursiveObject(tree) || PythonTree.isRecursiveObject(other.tree)) + return tree == other.tree return tree.softEquals(other.tree) } override fun hashCode(): Int { + if (PythonTree.isRecursiveObject(tree)) + return tree.hashCode() return tree.softHashCode() } + + private val INF = 1000_000_000 + + private fun diversity(): Int { + if (PythonTree.isRecursiveObject(tree)) + return INF + return tree.diversity() + } + + fun commonDiversity(other: Int): Int { + return listOf(diversity() + other, INF).min() + } } \ No newline at end of file From 52cd8e9a0d6fe2a45b4e27c447d8e8a1f3da070e Mon Sep 17 00:00:00 2001 From: Ekaterina Tochilina Date: Fri, 3 Mar 2023 12:24:36 +0300 Subject: [PATCH 30/32] More accurate timeout handling --- .../kotlin/org/utbot/python/PythonEngine.kt | 106 +++++++++--------- .../python/PythonTestGenerationProcessor.kt | 7 +- .../utbot/python/evaluation/PythonWorker.kt | 9 +- 3 files changed, 59 insertions(+), 63 deletions(-) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index 6d1e32a0d1..4e4b1a0ab3 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -3,14 +3,7 @@ package org.utbot.python import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.flow import mu.KotlinLogging -import org.utbot.framework.plugin.api.DocRegularStmt -import org.utbot.framework.plugin.api.EnvironmentModels -import org.utbot.framework.plugin.api.Instruction -import org.utbot.framework.plugin.api.UtError -import org.utbot.framework.plugin.api.UtExecutionResult -import org.utbot.framework.plugin.api.UtExecutionSuccess -import org.utbot.framework.plugin.api.UtExplicitlyThrownException -import org.utbot.framework.plugin.api.UtModel +import org.utbot.framework.plugin.api.* import org.utbot.fuzzer.UtFuzzedExecution import org.utbot.fuzzing.Control import org.utbot.fuzzing.fuzz @@ -180,7 +173,7 @@ class PythonEngine( fun fuzzingResultHandler( description: PythonMethodDescription, arguments: List - ): PythonExecutionResult { + ): PythonExecutionResult? { val argumentValues = arguments.map { PythonTreeModel(it.tree, it.tree.type) } logger.debug(argumentValues.map { it.tree } .toString()) val argumentModules = argumentValues @@ -200,56 +193,59 @@ class PythonEngine( modelList, methodUnderTest.argumentsNames ) - val evaluationResult = manager.run(functionArguments, localAdditionalModules) - return when (evaluationResult) { - is PythonEvaluationError -> { - val utError = UtError( - "Error evaluation: ${evaluationResult.status}, ${evaluationResult.message}", - Throwable(evaluationResult.stackTrace.joinToString("\n")) - ) - logger.debug(evaluationResult.stackTrace.joinToString("\n")) - PythonExecutionResult(InvalidExecution(utError), PythonFeedback(control = Control.PASS)) - } + try { + return when (val evaluationResult = manager.run(functionArguments, localAdditionalModules)) { + is PythonEvaluationError -> { + val utError = UtError( + "Error evaluation: ${evaluationResult.status}, ${evaluationResult.message}", + Throwable(evaluationResult.stackTrace.joinToString("\n")) + ) + logger.debug(evaluationResult.stackTrace.joinToString("\n")) + PythonExecutionResult(InvalidExecution(utError), PythonFeedback(control = Control.PASS)) + } - is PythonEvaluationTimeout -> { - val utError = UtError(evaluationResult.message, Throwable()) - PythonExecutionResult(InvalidExecution(utError), PythonFeedback(control = Control.PASS)) - } + is PythonEvaluationTimeout -> { + val utError = UtError(evaluationResult.message, Throwable()) + PythonExecutionResult(InvalidExecution(utError), PythonFeedback(control = Control.PASS)) + } - is PythonEvaluationSuccess -> { - val coveredInstructions = evaluationResult.coverage.coveredInstructions - coveredInstructions.forEach { coveredLines.add(it.lineNumber) } - - val summary = arguments - .zip(methodUnderTest.arguments) - .mapNotNull { it.first.summary?.replace("%var%", it.second.name) } - - val hasThisObject = methodUnderTest.hasThisArgument - - when (val result = handleSuccessResult( - parameters, - evaluationResult, - description, - hasThisObject, - summary - )) { - is ValidExecution -> { - val trieNode: Trie.Node = description.tracer.add(coveredInstructions) - PythonExecutionResult( - result, - PythonFeedback(control = Control.CONTINUE, result = trieNode) - ) - } + is PythonEvaluationSuccess -> { + val coveredInstructions = evaluationResult.coverage.coveredInstructions + coveredInstructions.forEach { coveredLines.add(it.lineNumber) } - is ArgumentsTypeErrorFeedback, is TypeErrorFeedback -> { - PythonExecutionResult(result, PythonFeedback(control = Control.PASS)) - } + val summary = arguments + .zip(methodUnderTest.arguments) + .mapNotNull { it.first.summary?.replace("%var%", it.second.name) } + + val hasThisObject = methodUnderTest.hasThisArgument - is InvalidExecution -> { - PythonExecutionResult(result, PythonFeedback(control = Control.CONTINUE)) + when (val result = handleSuccessResult( + parameters, + evaluationResult, + description, + hasThisObject, + summary + )) { + is ValidExecution -> { + val trieNode: Trie.Node = description.tracer.add(coveredInstructions) + PythonExecutionResult( + result, + PythonFeedback(control = Control.CONTINUE, result = trieNode) + ) + } + + is ArgumentsTypeErrorFeedback, is TypeErrorFeedback -> { + PythonExecutionResult(result, PythonFeedback(control = Control.PASS)) + } + + is InvalidExecution -> { + PythonExecutionResult(result, PythonFeedback(control = Control.CONTINUE)) + } } } } + } catch (_: TimeoutException) { + return null } } @@ -285,6 +281,12 @@ class PythonEngine( return@PythonFuzzing mem.fuzzingPlatformFeedback } val result = fuzzingResultHandler(description, arguments) + if (result == null) { // timeout + logger.info { "Fuzzing process was interrupted by timeout" } + manager.disconnect() + return@PythonFuzzing PythonFeedback(control = Control.STOP) + } + addExecutionToCache(pair, result) emit(result.fuzzingExecutionFeedback) return@PythonFuzzing result.fuzzingPlatformFeedback diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt index 4f9a8f1fa6..b35d36a809 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt @@ -120,12 +120,7 @@ object PythonTestGenerationProcessor { val methodsLeft = pythonMethods.size - index val localUntil = (until - System.currentTimeMillis()) / methodsLeft + System.currentTimeMillis() val method = findMethodByHeader(mypyStorage, methodHeader, currentPythonModule, pythonFileContent) - try { - testCaseGenerator.generate(method, localUntil) - } catch (_: TimeoutException) { - logger.warn { "Cannot connect to python code executor for method ${method.name}" } - PythonTestSet(method, emptyList(), emptyList(), emptyList()) - } + testCaseGenerator.generate(method, localUntil) } val (notEmptyTests, emptyTestSets) = tests.partition { it.executions.isNotEmpty() } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorker.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorker.kt index 5c7911d9d9..a38d3f2acc 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorker.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorker.kt @@ -29,13 +29,12 @@ class PythonWorker( } fun receiveMessage(): String? { - val lengthLine = inStream.readLine() - if (lengthLine == null) { - return null - } + val lengthLine = inStream.readLine() ?: return null val length = lengthLine.toInt() val buffer = CharArray(length) - inStream.read(buffer) + val bytesRead = inStream.read(buffer) + if (bytesRead < length) // TODO: maybe we should add more time for reading? + return null return String(buffer) } From ec7c7543b0fc53842f3d56681ea4a0c9cff82c20 Mon Sep 17 00:00:00 2001 From: Ekaterina Tochilina Date: Fri, 3 Mar 2023 12:43:02 +0300 Subject: [PATCH 31/32] Moved evaluation cache to separate class --- .../kotlin/org/utbot/python/PythonEngine.kt | 28 ++++--------------- .../python/evaluation/EvaluationCache.kt | 26 +++++++++++++++++ 2 files changed, 31 insertions(+), 23 deletions(-) create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/evaluation/EvaluationCache.kt diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index 4e4b1a0ab3..fd5390ad67 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -8,13 +8,7 @@ import org.utbot.fuzzer.UtFuzzedExecution import org.utbot.fuzzing.Control import org.utbot.fuzzing.fuzz import org.utbot.fuzzing.utils.Trie -import org.utbot.python.evaluation.PythonCodeExecutor -import org.utbot.python.evaluation.PythonCodeSocketExecutor -import org.utbot.python.evaluation.PythonEvaluationError -import org.utbot.python.evaluation.PythonEvaluationSuccess -import org.utbot.python.evaluation.PythonEvaluationTimeout -import org.utbot.python.evaluation.PythonWorker -import org.utbot.python.evaluation.PythonWorkerManager +import org.utbot.python.evaluation.* import org.utbot.python.evaluation.serialiation.MemoryDump import org.utbot.python.evaluation.serialiation.toPythonTree import org.utbot.python.framework.api.python.PythonTreeModel @@ -24,13 +18,11 @@ import org.utbot.python.newtyping.PythonTypeStorage import org.utbot.python.newtyping.general.Type import org.utbot.python.newtyping.pythonModules import org.utbot.python.newtyping.pythonTypeRepresentation -import org.utbot.python.utils.TemporaryFileManager import org.utbot.python.utils.camelToSnakeCase import org.utbot.summary.fuzzer.names.TestSuggestedInfo import java.net.ServerSocket private val logger = KotlinLogging.logger {} -private const val MAX_CACHE_SIZE = 200 class PythonEngine( private val methodUnderTest: PythonMethod, @@ -43,17 +35,7 @@ class PythonEngine( private val pythonTypeStorage: PythonTypeStorage, ) { - private val cache = mutableMapOf>, PythonExecutionResult>() - - private fun addExecutionToCache(key: Pair>, result: PythonExecutionResult) { - cache[key] = result - if (cache.size > MAX_CACHE_SIZE) { - val elemToDelete = cache.keys.maxBy { (_, args) -> - args.fold(0) { acc, arg -> arg.commonDiversity(acc) } - } - cache.remove(elemToDelete) - } - } + private val cache = EvaluationCache() private fun suggestExecutionName( description: PythonMethodDescription, @@ -146,7 +128,7 @@ class PythonEngine( return ValidExecution(utFuzzedExecution) } - fun constructEvaluationInput(pythonWorker: PythonWorker): PythonCodeExecutor { + private fun constructEvaluationInput(pythonWorker: PythonWorker): PythonCodeExecutor { return PythonCodeSocketExecutor( methodUnderTest, moduleToImport, @@ -274,7 +256,7 @@ class PythonEngine( } val pair = Pair(description, arguments.map { PythonTreeWrapper(it.tree) }) - val mem = cache[pair] + val mem = cache.get(pair) if (mem != null) { logger.debug("Repeat in fuzzing") emit(mem.fuzzingExecutionFeedback) @@ -287,7 +269,7 @@ class PythonEngine( return@PythonFuzzing PythonFeedback(control = Control.STOP) } - addExecutionToCache(pair, result) + cache.add(pair, result) emit(result.fuzzingExecutionFeedback) return@PythonFuzzing result.fuzzingPlatformFeedback diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/EvaluationCache.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/EvaluationCache.kt new file mode 100644 index 0000000000..8890757b06 --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/EvaluationCache.kt @@ -0,0 +1,26 @@ +package org.utbot.python.evaluation + +import mu.KotlinLogging +import org.utbot.python.framework.api.python.PythonTreeWrapper +import org.utbot.python.fuzzing.PythonExecutionResult +import org.utbot.python.fuzzing.PythonMethodDescription + +private const val MAX_CACHE_SIZE = 200 + +class EvaluationCache { + private val cache = mutableMapOf>, PythonExecutionResult>() + + fun add(key: Pair>, result: PythonExecutionResult) { + cache[key] = result + if (cache.size > MAX_CACHE_SIZE) { + val elemToDelete = cache.keys.maxBy { (_, args) -> + args.fold(0) { acc, arg -> arg.commonDiversity(acc) } + } + cache.remove(elemToDelete) + } + } + + fun get(key: Pair>): PythonExecutionResult? { + return cache[key] + } +} \ No newline at end of file From a1a66380de91e082b0be7f33ccfaab57e7325700 Mon Sep 17 00:00:00 2001 From: Ekaterina Tochilina Date: Fri, 3 Mar 2023 13:01:19 +0300 Subject: [PATCH 32/32] Added seed in type inference; get timeoutForRun from settings --- utbot-python/build.gradle.kts | 2 +- .../src/main/kotlin/org/utbot/python/PythonEngine.kt | 6 ++++-- .../org/utbot/python/evaluation/PythonWorkerManager.kt | 9 +++++---- .../newtyping/inference/baseline/BaselineAlgorithm.kt | 5 ++++- .../kotlin/org/utbot/python/newtyping/utils/Utils.kt | 5 ++--- 5 files changed, 16 insertions(+), 11 deletions(-) diff --git a/utbot-python/build.gradle.kts b/utbot-python/build.gradle.kts index 00de176540..a297f3e9fb 100644 --- a/utbot-python/build.gradle.kts +++ b/utbot-python/build.gradle.kts @@ -21,7 +21,7 @@ tasks { } dependencies { - api(project(":utbot-fuzzers")) + api(project(":utbot-fuzzing")) api(project(":utbot-framework")) api(project(":utbot-python-parser")) testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.8.1") diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index fd5390ad67..211cda6387 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -148,8 +148,10 @@ class PythonEngine( val manager = PythonWorkerManager( serverSocket, pythonPath, - until - ) { constructEvaluationInput(it) } + until, + { constructEvaluationInput(it) }, + timeoutForRun.toInt() + ) logger.info { "Executor manager was created successfully" } fun fuzzingResultHandler( diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt index d77b34df01..fd98864bf8 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt @@ -14,12 +14,13 @@ import java.net.SocketTimeoutException private val logger = KotlinLogging.logger {} class PythonWorkerManager( - val serverSocket: ServerSocket, + private val serverSocket: ServerSocket, val pythonPath: String, val until: Long, - val pythonCodeExecutorConstructor: (PythonWorker) -> PythonCodeExecutor + val pythonCodeExecutorConstructor: (PythonWorker) -> PythonCodeExecutor, + private val timeoutForRun: Int ) { - val logfile = TemporaryFileManager.createTemporaryFile("","utbot_executor.log", "log", true) + private val logfile = TemporaryFileManager.createTemporaryFile("","utbot_executor.log", "log", true) var timeout: Long = 0 lateinit var process: Process @@ -53,7 +54,7 @@ class PythonWorkerManager( } logger.debug { "Worker connected successfully" } - workerSocket.soTimeout = 2000 + workerSocket.soTimeout = timeoutForRun // TODO: maybe +eps for serialization/deserialization? val pythonWorker = PythonWorker(workerSocket) codeExecutor = pythonCodeExecutorConstructor(pythonWorker) } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt index c192ecc5b4..4466a9abb8 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt @@ -11,6 +11,7 @@ import org.utbot.python.newtyping.mypy.checkSuggestedSignatureWithDMypy import org.utbot.python.newtyping.utils.weightedRandom import org.utbot.python.utils.TemporaryFileManager import java.io.File +import kotlin.random.Random private val EDGES_TO_LINK = listOf( EdgeSource.Identification, @@ -32,6 +33,8 @@ class BaselineAlgorithm( private val initialErrorNumber: Int, private val configFile: File ) : TypeInferenceAlgorithm() { + private val random = Random(0) + override suspend fun run( hintCollectorResult: HintCollectorResult, isCancelled: () -> Boolean, @@ -89,7 +92,7 @@ class BaselineAlgorithm( private fun chooseState(states: List): BaselineAlgorithmState { val weights = states.map { 1.0 / (it.children * it.children + 1) } - return weightedRandom(states, weights) + return weightedRandom(states, weights, random) } private fun getInitialState( diff --git a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/utils/Utils.kt b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/utils/Utils.kt index 0172127142..bf6c906b98 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/utils/Utils.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/utils/Utils.kt @@ -7,8 +7,7 @@ fun getOffsetLine(sourceFileContent: String, offset: Int): Int { return sourceFileContent.take(offset).count { it == '\n' } + 1 } -fun weightedRandom(elems: List, weights: List): T { - val sum = weights.sum() - val index = Random.chooseOne(weights.map { it / sum }.toDoubleArray()) +fun weightedRandom(elems: List, weights: List, random: Random): T { + val index = random.chooseOne(weights.toDoubleArray()) return elems[index] } \ No newline at end of file