From 90c82cee7535b039a0ddd7dc40f280a5e1ca75a1 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Thu, 9 Mar 2023 17:13:31 +0300 Subject: [PATCH 1/6] Continue separate --- .../serialiation/PythonObjectParser.kt | 26 ++++++++++++------- .../python/framework/api/python/PythonApi.kt | 19 +++++++------- .../api/python/util/PythonIdUtils.kt | 5 ---- .../tree/PythonCgVariableConstructor.kt | 14 +++++++--- 4 files changed, 36 insertions(+), 28 deletions(-) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/PythonObjectParser.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/PythonObjectParser.kt index 48ce151a80..27fbbf0099 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/PythonObjectParser.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/PythonObjectParser.kt @@ -45,68 +45,73 @@ class MemoryDump( sealed class MemoryObject( val id: String, val kind: String, + val module: String, val comparable: Boolean, ) class ReprMemoryObject( id: String, kind: String, + module: String, comparable: Boolean, val value: String, -): MemoryObject(id, kind, comparable) +): MemoryObject(id, kind, module, comparable) class ListMemoryObject( id: String, kind: String, + module: String, comparable: Boolean, val items: List, -): MemoryObject(id, kind, comparable) +): MemoryObject(id, kind, module, comparable) class DictMemoryObject( id: String, kind: String, + module: String, comparable: Boolean, val items: Map, -): MemoryObject(id, kind, comparable) +): MemoryObject(id, kind, module, comparable) class ReduceMemoryObject( id: String, kind: String, + module: String, comparable: Boolean, val constructor: String, val args: String, val state: String, val listitems: String, val dictitems: String -): MemoryObject(id, kind, comparable) +): MemoryObject(id, kind, module, comparable) fun PythonTree.PythonTreeNode.toMemoryObject(memoryDump: MemoryDump): String { val obj = when(this) { is PythonTree.PrimitiveNode -> { - ReprMemoryObject(this.id.toString(), this.type.name, this.comparable, this.repr) + ReprMemoryObject(this.id.toString(), this.type.name, this.type.moduleName, this.comparable, this.repr) } is PythonTree.ListNode -> { val items = this.items.entries .sortedBy { it.key } .map { it.value.toMemoryObject(memoryDump) } - ListMemoryObject(this.id.toString(), this.type.name, this.comparable, items) + ListMemoryObject(this.id.toString(), this.type.name, this.type.moduleName, this.comparable, items) } is PythonTree.TupleNode -> { val items = this.items.entries .sortedBy { it.key } .map { it.value.toMemoryObject(memoryDump) } - ListMemoryObject(this.id.toString(), this.type.name, this.comparable, items) + ListMemoryObject(this.id.toString(), this.type.name, this.type.moduleName, this.comparable, items) } is PythonTree.SetNode -> { val items = this.items.map { it.toMemoryObject(memoryDump) } - ListMemoryObject(this.id.toString(), this.type.name, this.comparable, items) + ListMemoryObject(this.id.toString(), this.type.name, this.type.moduleName, this.comparable, items) } is PythonTree.DictNode -> { val items = this.items.entries .associate { it.key.toMemoryObject(memoryDump) to it.value.toMemoryObject(memoryDump) } - DictMemoryObject(this.id.toString(), this.type.name, this.comparable, items) + DictMemoryObject(this.id.toString(), this.type.name, this.type.moduleName, this.comparable, items) } is PythonTree.ReduceNode -> { val stateObjId = PythonTree.DictNode(this.state.entries.associate { PythonTree.PrimitiveNode(pythonStrClassId, it.key) to it.value }.toMutableMap()) @@ -116,6 +121,7 @@ fun PythonTree.PythonTreeNode.toMemoryObject(memoryDump: MemoryDump): String { ReduceMemoryObject( this.id.toString(), this.type.name, + this.type.moduleName, this.comparable, this.constructor.name, argsIds.toMemoryObject(memoryDump), @@ -137,7 +143,7 @@ fun MemoryObject.toPythonTree(memoryDump: MemoryDump): PythonTree.PythonTreeNode is ReprMemoryObject -> { PythonTree.PrimitiveNode( this.id.toLong(), - PythonClassId(this.kind), + PythonClassId(this.module, this.kind), this.value ) } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonApi.kt index 9355a5b449..2ccb2f9795 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonApi.kt @@ -17,17 +17,18 @@ import org.utbot.python.framework.api.python.util.moduleOfType const val pythonBuiltinsModuleName = "builtins" class PythonClassId( - name: String // includes module (like "_ast.Assign") + val moduleName: String, + name: String, ) : ClassId(name) { - override fun toString(): String = name - val rootModuleName: String = this.toString().split(".")[0] - override val simpleName: String = name.split(".").last() - val moduleName: String - get() { - return moduleOfType(name) ?: pythonBuiltinsModuleName - } + constructor(fullName: String) : this( + moduleOfType(fullName) ?: pythonBuiltinsModuleName, + fullName.removePrefix(moduleOfType(fullName) ?: pythonBuiltinsModuleName).removePrefix(".") + ) + override fun toString(): String = canonicalName + val rootModuleName: String = moduleName.split(".").first() + override val simpleName: String = name + override val canonicalName = "$moduleName.$name" override val packageName = moduleName - override val canonicalName = name } open class RawPythonAnnotation( diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonIdUtils.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonIdUtils.kt index 9a15f99dd4..7e84734e8d 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonIdUtils.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonIdUtils.kt @@ -2,11 +2,6 @@ package org.utbot.python.framework.api.python.util import org.utbot.python.framework.api.python.PythonClassId import org.utbot.python.framework.api.python.NormalizedPythonAnnotation -import org.utbot.python.framework.api.python.PythonBoolModel -import org.utbot.python.framework.api.python.PythonListModel -import org.utbot.python.framework.api.python.PythonTupleModel -import org.utbot.python.framework.api.python.PythonDictModel -import org.utbot.python.framework.api.python.PythonSetModel // none annotation can be used in code only since Python 3.10 val pythonNoneClassId = PythonClassId("types.NoneType") diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonCgVariableConstructor.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonCgVariableConstructor.kt index 8f4b6cc479..e23d5159bc 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonCgVariableConstructor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonCgVariableConstructor.kt @@ -17,20 +17,26 @@ import org.utbot.python.framework.api.python.util.pythonNoneClassId import org.utbot.python.framework.codegen.PythonCgLanguageAssistant import org.utbot.python.framework.codegen.model.tree.* -class PythonCgVariableConstructor(context_: CgContext) : CgVariableConstructor(context_) { +class PythonCgVariableConstructor(cgContext: CgContext) : CgVariableConstructor(cgContext) { private val nameGenerator = CgComponents.getNameGeneratorBy(context) override fun getOrCreateVariable(model: UtModel, name: String?): CgValue { val baseName = name ?: nameGenerator.nameFrom(model.classId) return valueByModel.getOrPut(model) { when (model) { - is PythonBoolModel -> CgLiteral(model.classId, model.value) - is PythonPrimitiveModel -> CgLiteral(model.classId, model.value) + is PythonBoolModel -> { + CgLiteral(model.classId, model.value) + } + is PythonPrimitiveModel -> { + CgLiteral(model.classId, model.value) + } is PythonTreeModel -> { val (value, arguments) = pythonBuildObject(model.tree) CgPythonTree(model.classId, model.tree, value, arguments) } - is PythonInitObjectModel -> constructInitObjectModel(model, baseName) + is PythonInitObjectModel -> { + constructInitObjectModel(model, baseName) + } is PythonDictModel -> CgPythonDict(model.stores.map { getOrCreateVariable(it.key) to getOrCreateVariable( it.value From 333fe5cb8830a799925a6759cec5076c7e61b1e4 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Mon, 13 Mar 2023 18:26:44 +0300 Subject: [PATCH 2/6] continue refactor --- .../language/python/PythonDialogProcessor.kt | 2 +- .../python/PythonTestGenerationProcessor.kt | 4 +- .../serialiation/PythonObjectParser.kt | 8 +- .../python/framework/api/python/PythonApi.kt | 109 +----------------- .../python/framework/api/python/PythonTree.kt | 15 ++- .../codegen/model/PythonCodeGenerator.kt | 3 +- .../tree/PythonCgVariableConstructor.kt | 33 ------ .../fuzzing/provider/BoolValueProvider.kt | 3 +- .../provider/BytearrayValueProvider.kt | 2 +- .../fuzzing/provider/BytesValueProvider.kt | 2 +- .../fuzzing/provider/ComplexValueProvider.kt | 2 +- .../fuzzing/provider/DictValueProvider.kt | 3 +- .../fuzzing/provider/FloatValueProvider.kt | 6 +- .../fuzzing/provider/IntValueProvider.kt | 3 +- .../fuzzing/provider/ListValueProvider.kt | 3 +- .../fuzzing/provider/ReduceValueProvider.kt | 28 ++--- .../fuzzing/provider/SetValueProvider.kt | 3 +- .../fuzzing/provider/StrValueProvider.kt | 3 +- .../fuzzing/provider/TupleValueProvider.kt | 3 +- .../org/utbot/python/newtyping/PythonType.kt | 6 + .../utbot/python/newtyping/PythonTypeAPI.kt | 8 ++ .../utbot/python/utils/RequirementsUtils.kt | 12 +- 22 files changed, 79 insertions(+), 182 deletions(-) diff --git a/utbot-intellij-python/src/main/kotlin/org/utbot/intellij/plugin/language/python/PythonDialogProcessor.kt b/utbot-intellij-python/src/main/kotlin/org/utbot/intellij/plugin/language/python/PythonDialogProcessor.kt index 1bcd0d3d68..08311194ed 100644 --- a/utbot-intellij-python/src/main/kotlin/org/utbot/intellij/plugin/language/python/PythonDialogProcessor.kt +++ b/utbot-intellij-python/src/main/kotlin/org/utbot/intellij/plugin/language/python/PythonDialogProcessor.kt @@ -137,7 +137,7 @@ object PythonDialogProcessor { .mapNotNull { val functionName = it.name ?: return@mapNotNull null val moduleFilename = it.containingFile.virtualFile?.canonicalPath ?: "" - val containingClassId = it.containingClass?.name?.let{ PythonClassId(it) } + val containingClassId = it.containingClass?.name?.let{ cls -> PythonClassId(cls) } return@mapNotNull PythonMethodHeader( functionName, moduleFilename, diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt index 1f8c3d3e3b..f904e49c30 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt @@ -124,9 +124,9 @@ object PythonTestGenerationProcessor { val classId = if (containingClassName == null) - PythonClassId("$currentPythonModule.TopLevelFunctions") + PythonClassId(currentPythonModule, "TopLevelFunctions") else - PythonClassId("$currentPythonModule.$containingClassName") + PythonClassId(currentPythonModule, containingClassName) val methodIds = notEmptyTests.associate { it.method to PythonMethodId( diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/PythonObjectParser.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/PythonObjectParser.kt index 27fbbf0099..07e113458a 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/PythonObjectParser.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/PythonObjectParser.kt @@ -47,7 +47,9 @@ sealed class MemoryObject( val kind: String, val module: String, val comparable: Boolean, -) +) { + val qualname: String = if (module.isEmpty()) kind else "$module.$kind" +} class ReprMemoryObject( id: String, @@ -159,7 +161,7 @@ fun MemoryObject.toPythonTree(memoryDump: MemoryDump): PythonTree.PythonTreeNode val elementsMap = items.withIndex().associate { it.index to memoryDump.getById(it.value).toPythonTree(memoryDump) }.toMutableMap() - when (this.kind) { + when (this.qualname) { "builtins.tuple" -> { PythonTree.TupleNode(this.id.toLong(), elementsMap) } @@ -178,7 +180,7 @@ fun MemoryObject.toPythonTree(memoryDump: MemoryDump): PythonTree.PythonTreeNode val dictitemsObjs = memoryDump.getById(dictitems) as DictMemoryObject PythonTree.ReduceNode( this.id.toLong(), - PythonClassId(this.kind), + PythonClassId(this.kind, this.module), PythonClassId(this.constructor), arguments.items.map { memoryDump.getById(it).toPythonTree(memoryDump) }, stateObjs.items.entries.associate { diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonApi.kt index 2ccb2f9795..c5555c83f8 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonApi.kt @@ -18,16 +18,16 @@ const val pythonBuiltinsModuleName = "builtins" class PythonClassId( val moduleName: String, - name: String, -) : ClassId(name) { + val typeName: String, +) : ClassId("$moduleName.$typeName") { constructor(fullName: String) : this( moduleOfType(fullName) ?: pythonBuiltinsModuleName, fullName.removePrefix(moduleOfType(fullName) ?: pythonBuiltinsModuleName).removePrefix(".") ) override fun toString(): String = canonicalName val rootModuleName: String = moduleName.split(".").first() - override val simpleName: String = name - override val canonicalName = "$moduleName.$name" + override val simpleName: String = typeName + override val canonicalName = name override val packageName = moduleName } @@ -84,104 +84,3 @@ class PythonTreeModel( return tree.hashCode() } } - -class PythonDefaultModel( - val repr: String, - classId: PythonClassId -): PythonModel(classId) { - override fun toString() = repr -} - -class PythonPrimitiveModel( - val value: Any, - classId: PythonClassId -): PythonModel(classId) { - override fun toString() = "$value" -} - -class PythonBoolModel(val value: Boolean): PythonModel(classId) { - override fun toString() = - if (value) "True" else "False" - companion object { - val classId = PythonClassId("builtins.bool") - } -} - -class PythonInitObjectModel( - val type: String, - val initValues: List -): PythonModel(PythonClassId(type)) { - override fun toString(): String { - val params = initValues.joinToString(separator = ", ") { it.toString() } - return "$type($params)" - } - - override val allContainingClassIds: Set - get() = super.allContainingClassIds + initValues.flatMap { it.allContainingClassIds } -} - -class PythonListModel( - val length: Int = 0, - val stores: List -) : PythonModel(classId) { - override fun toString() = - (0 until length).joinToString(", ", "[", "]") { stores[it].toString() } - - override val allContainingClassIds: Set - get() = super.allContainingClassIds + stores.flatMap { it.allContainingClassIds } - - companion object { - val classId = PythonClassId("builtins.list") - } -} - -class PythonTupleModel( - val length: Int = 0, - val stores: List -) : PythonModel(classId) { - override fun toString() = - (0 until length).joinToString(", ", "(", ")") { stores[it].toString() } - - override val allContainingClassIds: Set - get() = super.allContainingClassIds + stores.flatMap { it.allContainingClassIds } - - companion object { - val classId = PythonClassId("builtins.tuple") - } -} - -class PythonDictModel( - val length: Int = 0, - val stores: Map -) : PythonModel(classId) { - override fun toString() = withToStringThreadLocalReentrancyGuard { - stores.entries.joinToString(", ", "{", "}") { "${it.key}: ${it.value}" } - } - - override val allContainingClassIds: Set - get() = super.allContainingClassIds + - stores.entries.flatMap { it.key.allContainingClassIds + it.value.allContainingClassIds } - - companion object { - val classId = PythonClassId("builtins.dict") - } -} - -class PythonSetModel( - val length: Int = 0, - val stores: Set -) : PythonModel(classId) { - override fun toString() = withToStringThreadLocalReentrancyGuard { - if (stores.isEmpty()) - "set()" - else - stores.joinToString(", ", "{", "}") { it.toString() } - } - - override val allContainingClassIds: Set - get() = super.allContainingClassIds + stores.flatMap { it.allContainingClassIds } - - companion object { - val classId = PythonClassId("builtins.set") - } -} diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt index 245d8c2c32..ecaec2eb62 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt @@ -1,10 +1,15 @@ package org.utbot.python.framework.api.python import org.utbot.python.framework.api.python.util.pythonBoolClassId +import org.utbot.python.framework.api.python.util.pythonDictClassId import org.utbot.python.framework.api.python.util.pythonFloatClassId import org.utbot.python.framework.api.python.util.pythonIntClassId +import org.utbot.python.framework.api.python.util.pythonListClassId import org.utbot.python.framework.api.python.util.pythonNoneClassId +import org.utbot.python.framework.api.python.util.pythonObjectClassId +import org.utbot.python.framework.api.python.util.pythonSetClassId import org.utbot.python.framework.api.python.util.pythonStrClassId +import org.utbot.python.framework.api.python.util.pythonTupleClassId import org.utbot.python.framework.api.python.util.toPythonRepr import org.utbot.python.newtyping.general.Type import org.utbot.python.newtyping.pythonTypeName @@ -121,7 +126,7 @@ object PythonTree { class ListNode( id: Long, val items: MutableMap - ) : PythonTreeNode(id, PythonClassId("builtins.list")) { + ) : PythonTreeNode(id, pythonListClassId) { constructor(items: MutableMap) : this(PythonIdGenerator.createId(), items) override val children: List @@ -139,7 +144,7 @@ object PythonTree { class DictNode( id: Long, val items: MutableMap - ) : PythonTreeNode(id, PythonClassId("builtins.dict")) { + ) : PythonTreeNode(id, pythonDictClassId) { constructor(items: MutableMap) : this(PythonIdGenerator.createId(), items) override val children: List @@ -159,7 +164,7 @@ object PythonTree { class SetNode( id: Long, val items: MutableSet - ) : PythonTreeNode(id, PythonClassId("builtins.set")) { + ) : PythonTreeNode(id, pythonSetClassId) { constructor(items: MutableSet) : this(PythonIdGenerator.createId(), items) override val children: List @@ -182,7 +187,7 @@ object PythonTree { class TupleNode( id: Long, val items: MutableMap - ) : PythonTreeNode(id, PythonClassId("builtins.tuple")) { + ) : PythonTreeNode(id, pythonTupleClassId) { constructor(items: MutableMap) : this(PythonIdGenerator.createId(), items) override val children: List @@ -259,7 +264,7 @@ object PythonTree { fun fromObject(): PrimitiveNode { return PrimitiveNode( - PythonClassId("builtins.object"), + pythonObjectClassId, "object()" ) } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/PythonCodeGenerator.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/PythonCodeGenerator.kt index e991aaaf0b..74f52a6795 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/PythonCodeGenerator.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/PythonCodeGenerator.kt @@ -43,6 +43,7 @@ import org.utbot.python.newtyping.pythonModules import org.utbot.python.newtyping.pythonTypeRepresentation import org.utbot.python.framework.codegen.toPythonRawString import org.utbot.python.newtyping.pythonDescription +import org.utbot.python.newtyping.pythonName class PythonCodeGenerator( classUnderTest: ClassId, @@ -139,7 +140,7 @@ class PythonCodeGenerator( if (containingClass == null) method.name else - "${containingClass.pythonDescription().name.name}.${method.name}" + "${containingClass.pythonName()}.${method.name}" if (functionModule.isNotEmpty()) { functionTextName = "$functionModule.$functionTextName" } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonCgVariableConstructor.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonCgVariableConstructor.kt index e23d5159bc..734c3f7056 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonCgVariableConstructor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonCgVariableConstructor.kt @@ -21,49 +21,18 @@ class PythonCgVariableConstructor(cgContext: CgContext) : CgVariableConstructor( private val nameGenerator = CgComponents.getNameGeneratorBy(context) override fun getOrCreateVariable(model: UtModel, name: String?): CgValue { - val baseName = name ?: nameGenerator.nameFrom(model.classId) return valueByModel.getOrPut(model) { when (model) { - is PythonBoolModel -> { - CgLiteral(model.classId, model.value) - } - is PythonPrimitiveModel -> { - CgLiteral(model.classId, model.value) - } is PythonTreeModel -> { val (value, arguments) = pythonBuildObject(model.tree) CgPythonTree(model.classId, model.tree, value, arguments) } - is PythonInitObjectModel -> { - constructInitObjectModel(model, baseName) - } - is PythonDictModel -> CgPythonDict(model.stores.map { - getOrCreateVariable(it.key) to getOrCreateVariable( - it.value - ) - }.toMap()) - - is PythonListModel -> CgPythonList(model.stores.map { getOrCreateVariable(it) }) - is PythonSetModel -> CgPythonSet(model.stores.map { getOrCreateVariable(it) }.toSet()) - is PythonTupleModel -> CgPythonTuple(model.stores.map { getOrCreateVariable(it) }) - is PythonDefaultModel -> CgPythonRepr(model.classId, model.repr) is PythonModel -> error("Unexpected PythonModel: ${model::class}") else -> super.getOrCreateVariable(model, name) } } } - private fun declareOrGet(model: UtModel): CgValue = valueByModel[model] ?: getOrCreateVariable(model) - - private fun constructInitObjectModel(model: PythonInitObjectModel, baseName: String): CgVariable { - return newVar(model.classId, baseName) { - CgConstructorCall( - ConstructorId(model.classId, model.initValues.map { it.classId }), - model.initValues.map { getOrCreateVariable(it) } - ) - } - } - private fun pythonBuildObject(objectNode: PythonTree.PythonTreeNode): Pair> { return when (objectNode) { is PythonTree.PrimitiveNode -> { @@ -120,7 +89,6 @@ class PythonCgVariableConstructor(cgContext: CgContext) : CgVariableConstructor( val obj = newVar(objectNode.type) { constructorCall } -// obj `=` constructorCall (context.cgLanguageAssistant as PythonCgLanguageAssistant).memoryObjects[id] = obj (context.cgLanguageAssistant as PythonCgLanguageAssistant).memoryObjectsModels[id] = objectNode @@ -138,7 +106,6 @@ class PythonCgVariableConstructor(cgContext: CgContext) : CgVariableConstructor( } state.forEach { (key, value) -> -// val fieldAccess = CgFieldAccess(obj, FieldId(objectNode.type, key)) obj[FieldId(objectNode.type, key)] `=` value } listitems.forEach { diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/BoolValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/BoolValueProvider.kt index ab6a30050c..865100d58d 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/BoolValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/BoolValueProvider.kt @@ -5,6 +5,7 @@ import org.utbot.fuzzing.ValueProvider import org.utbot.fuzzing.seeds.Bool import org.utbot.fuzzing.seeds.KnownValue import org.utbot.python.framework.api.python.PythonTree +import org.utbot.python.framework.api.python.util.pythonBoolClassId import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription import org.utbot.python.fuzzing.provider.utils.generateSummary @@ -14,7 +15,7 @@ import org.utbot.python.newtyping.pythonTypeName object BoolValueProvider : ValueProvider{ override fun accept(type: Type): Boolean { - return type.pythonTypeName() == "builtins.bool" || type.isAny() + return type.pythonTypeName() == pythonBoolClassId.canonicalName || type.isAny() } override fun generate(description: PythonMethodDescription, type: Type) = sequence { diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/BytearrayValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/BytearrayValueProvider.kt index 22b43229e9..3da69ae420 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/BytearrayValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/BytearrayValueProvider.kt @@ -13,7 +13,7 @@ import org.utbot.python.newtyping.pythonTypeRepresentation object BytearrayValueProvider : ValueProvider { override fun accept(type: Type): Boolean { - return type.pythonTypeName() == "builtins.bytearray" + return type.pythonTypeName() == pythonBytearrayClassId.canonicalName } override fun generate(description: PythonMethodDescription, type: Type) = sequence { diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/BytesValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/BytesValueProvider.kt index f81b916be3..4f9d793387 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/BytesValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/BytesValueProvider.kt @@ -13,7 +13,7 @@ import org.utbot.python.newtyping.pythonTypeRepresentation object BytesValueProvider : ValueProvider { override fun accept(type: Type): Boolean { - return type.pythonTypeName() == "builtins.bytes" + return type.pythonTypeName() == pythonBytesClassId.canonicalName } override fun generate(description: PythonMethodDescription, type: Type) = sequence { diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ComplexValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ComplexValueProvider.kt index b7b1aadbf7..824d646849 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ComplexValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ComplexValueProvider.kt @@ -14,7 +14,7 @@ import org.utbot.python.newtyping.pythonTypeRepresentation object ComplexValueProvider : ValueProvider { override fun accept(type: Type): Boolean { - return type.pythonTypeName() == "builtins.complex" || type.isAny() + return type.pythonTypeName() == pythonComplexClassId.canonicalName || type.isAny() } override fun generate(description: PythonMethodDescription, type: Type) = sequence { diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/DictValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/DictValueProvider.kt index 6b692c007c..2cc51ada41 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/DictValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/DictValueProvider.kt @@ -4,6 +4,7 @@ import org.utbot.fuzzing.Routine import org.utbot.fuzzing.Seed import org.utbot.fuzzing.ValueProvider import org.utbot.python.framework.api.python.PythonTree +import org.utbot.python.framework.api.python.util.pythonDictClassId import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription import org.utbot.python.newtyping.general.Type @@ -13,7 +14,7 @@ import org.utbot.python.newtyping.pythonTypeRepresentation object DictValueProvider : ValueProvider { override fun accept(type: Type): Boolean { - return type.pythonTypeName() == "builtins.dict" + return type.pythonTypeName() == pythonDictClassId.canonicalName } override fun generate(description: PythonMethodDescription, type: Type) = sequence { diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/FloatValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/FloatValueProvider.kt index 2afc307738..49b2cd0eed 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/FloatValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/FloatValueProvider.kt @@ -4,6 +4,8 @@ import org.utbot.fuzzing.Seed import org.utbot.fuzzing.ValueProvider import org.utbot.fuzzing.seeds.IEEE754Value import org.utbot.python.framework.api.python.PythonTree +import org.utbot.python.framework.api.python.util.pythonFloatClassId +import org.utbot.python.framework.api.python.util.pythonIntClassId import org.utbot.python.fuzzing.PythonFuzzedConcreteValue import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription @@ -16,7 +18,7 @@ import java.math.BigInteger object FloatValueProvider : ValueProvider { override fun accept(type: Type): Boolean { - return type.pythonTypeName() == "builtins.float" || type.isAny() + return type.pythonTypeName() == pythonFloatClassId.canonicalName || type.isAny() } private fun getFloatConstants(concreteValues: Collection): List { @@ -31,7 +33,7 @@ object FloatValueProvider : ValueProvider): List { return concreteValues - .filter { it.type.pythonTypeName() == "builtins.int" } + .filter { it.type.pythonTypeName() == pythonIntClassId.canonicalName } .map { fuzzedValue -> (fuzzedValue.value as BigInteger).let { IEEE754Value.fromValue(it.toDouble()) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/IntValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/IntValueProvider.kt index 748a143f92..97b634e92d 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/IntValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/IntValueProvider.kt @@ -8,6 +8,7 @@ import org.utbot.fuzzing.seeds.BitVectorValue import org.utbot.fuzzing.seeds.KnownValue import org.utbot.fuzzing.seeds.Signed import org.utbot.python.framework.api.python.PythonTree +import org.utbot.python.framework.api.python.util.pythonIntClassId import org.utbot.python.fuzzing.PythonFuzzedConcreteValue import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription @@ -23,7 +24,7 @@ object IntValueProvider : ValueProvider Unit): BitVectorValue { diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ListValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ListValueProvider.kt index fa31e72fc3..9d8f748950 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ListValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ListValueProvider.kt @@ -4,6 +4,7 @@ import org.utbot.fuzzing.Routine import org.utbot.fuzzing.Seed import org.utbot.fuzzing.ValueProvider import org.utbot.python.framework.api.python.PythonTree +import org.utbot.python.framework.api.python.util.pythonListClassId import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription import org.utbot.python.newtyping.general.Type @@ -13,7 +14,7 @@ import org.utbot.python.newtyping.pythonTypeRepresentation object ListValueProvider : ValueProvider { override fun accept(type: Type): Boolean { - return type.pythonTypeName() == "builtins.list" + return type.pythonTypeName() == pythonListClassId.canonicalName } override fun generate(description: PythonMethodDescription, type: Type) = sequence { diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ReduceValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ReduceValueProvider.kt index 9aec8b9599..4025dad6a1 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ReduceValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ReduceValueProvider.kt @@ -5,7 +5,7 @@ import org.utbot.fuzzing.Seed import org.utbot.fuzzing.ValueProvider import org.utbot.python.framework.api.python.PythonClassId import org.utbot.python.framework.api.python.PythonTree -import org.utbot.python.framework.api.python.util.toPythonRepr +import org.utbot.python.framework.api.python.util.* import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription import org.utbot.python.fuzzing.provider.utils.isConcreteType @@ -15,17 +15,17 @@ import org.utbot.python.newtyping.general.Type object ReduceValueProvider : ValueProvider { private val unsupportedTypes = listOf( - "builtins.list", - "builtins.set", - "builtins.tuple", - "builtins.dict", - "builtins.bytes", - "builtins.bytearray", - "builtins.complex", - "builtins.int", - "builtins.float", - "builtins.str", - "builtins.bool", + pythonListClassId.canonicalName, + pythonSetClassId.canonicalName, + pythonTupleClassId.canonicalName, + pythonDictClassId.canonicalName, + pythonBytesClassId.canonicalName, + pythonBytearrayClassId.canonicalName, + pythonComplexClassId.canonicalName, + pythonIntClassId.canonicalName, + pythonFloatClassId.canonicalName, + pythonStrClassId.canonicalName, + pythonBoolClassId.canonicalName, ) override fun accept(type: Type): Boolean { @@ -93,8 +93,8 @@ object ReduceValueProvider : ValueProvider PythonFuzzedValue( PythonTree.ReduceNode( - PythonClassId(type.pythonTypeName()), - PythonClassId(type.pythonTypeName()), + PythonClassId(type.pythonModuleName(), type.pythonName()), + PythonClassId(type.pythonModuleName(), type.pythonName()), v.map { it.tree }, ), "%var% = ${type.pythonTypeRepresentation()}" diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/SetValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/SetValueProvider.kt index ef58b85cd0..465a8d59ac 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/SetValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/SetValueProvider.kt @@ -4,6 +4,7 @@ import org.utbot.fuzzing.Routine import org.utbot.fuzzing.Seed import org.utbot.fuzzing.ValueProvider import org.utbot.python.framework.api.python.PythonTree +import org.utbot.python.framework.api.python.util.pythonSetClassId import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription import org.utbot.python.newtyping.general.Type @@ -13,7 +14,7 @@ import org.utbot.python.newtyping.pythonTypeRepresentation object SetValueProvider : ValueProvider { override fun accept(type: Type): Boolean { - return type.pythonTypeName() == "builtins.set" + return type.pythonTypeName() == pythonSetClassId.canonicalName } override fun generate(description: PythonMethodDescription, type: Type) = sequence { diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/StrValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/StrValueProvider.kt index 04c39449ca..5175cb005d 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/StrValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/StrValueProvider.kt @@ -5,6 +5,7 @@ import org.utbot.fuzzing.ValueProvider import org.utbot.fuzzing.seeds.KnownValue import org.utbot.fuzzing.seeds.StringValue import org.utbot.python.framework.api.python.PythonTree +import org.utbot.python.framework.api.python.util.pythonStrClassId import org.utbot.python.fuzzing.PythonFuzzedConcreteValue import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription @@ -15,7 +16,7 @@ import org.utbot.python.newtyping.pythonTypeName object StrValueProvider : ValueProvider { override fun accept(type: Type): Boolean { - return type.pythonTypeName() == "builtins.str" + return type.pythonTypeName() == pythonStrClassId.canonicalName } private fun getStrConstants(concreteValues: Collection): List { diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/TupleValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/TupleValueProvider.kt index d53def76ed..e39505d400 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/TupleValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/TupleValueProvider.kt @@ -4,6 +4,7 @@ import org.utbot.fuzzing.Routine import org.utbot.fuzzing.Seed import org.utbot.fuzzing.ValueProvider import org.utbot.python.framework.api.python.PythonTree +import org.utbot.python.framework.api.python.util.pythonTupleClassId import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription import org.utbot.python.fuzzing.provider.utils.getSuitableConstantsFromCode @@ -12,7 +13,7 @@ import org.utbot.python.newtyping.general.Type object TupleValueProvider : ValueProvider { override fun accept(type: Type): Boolean { - return type.pythonTypeName() == "builtins.tuple" + return type.pythonTypeName() == pythonTupleClassId.canonicalName } override fun generate(description: PythonMethodDescription, type: Type) = sequence { diff --git a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/PythonType.kt b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/PythonType.kt index a5ea8a0eb9..8d7d936f4c 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/PythonType.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/PythonType.kt @@ -28,6 +28,12 @@ sealed class PythonTypeDescription(name: Name) : TypeMetaDataWithName(name) { else name.prefix.joinToString(".") + "." + name.name } + fun getModuleName(): String { + return name.prefix.joinToString(".") + } + fun getName(): String { + return name.name + } fun getModules(type: Type): Set { val cur = if (name.prefix.isNotEmpty()) setOf(name.prefix.joinToString(separator = ".")) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/PythonTypeAPI.kt b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/PythonTypeAPI.kt index d171456ff5..622ab7848a 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/PythonTypeAPI.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/PythonTypeAPI.kt @@ -41,6 +41,14 @@ fun Type.pythonTypeName(): String { return pythonDescription().getTypeName() } +fun Type.pythonModuleName(): String { + return pythonDescription().getModuleName() +} + +fun Type.pythonName(): String { + return pythonDescription().getName() +} + val pythonAnyName = Name(listOf("typing"), "Any") val pythonUnionName = Name(listOf("typing"), "Union") val pythonNoneName = Name(emptyList(), "None") diff --git a/utbot-python/src/main/kotlin/org/utbot/python/utils/RequirementsUtils.kt b/utbot-python/src/main/kotlin/org/utbot/python/utils/RequirementsUtils.kt index 1100150eb0..e2fe93e2ae 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/utils/RequirementsUtils.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/utils/RequirementsUtils.kt @@ -1,12 +1,12 @@ package org.utbot.python.utils object RequirementsUtils { - val requirements: List = - RequirementsUtils::class.java.getResource("/requirements.txt") - ?.readText() - ?.split('\n') - ?.filter { it.isNotEmpty() } - ?: error("Didn't find /requirements.txt") + val requirements: List = listOf( + "mypy==1.0.0", + "coverage==6.5.0", + "utbot-executor==1.1.37", + "utbot-mypy-runner==0.2.8", + ) private val requirementsScriptContent: String = RequirementsUtils::class.java.getResource("/check_requirements.py") From a23267f01ea9396eff2b6ed03622685f57947d57 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Wed, 15 Mar 2023 16:37:45 +0300 Subject: [PATCH 3/6] Fix bugs with memory and module names --- .../python/evaluation/serialiation/PythonObjectParser.kt | 2 +- .../org/utbot/python/framework/api/python/PythonApi.kt | 3 +-- .../model/constructor/tree/PythonCgVariableConstructor.kt | 6 +----- .../main/kotlin/org/utbot/python/utils/RequirementsUtils.kt | 2 +- 4 files changed, 4 insertions(+), 9 deletions(-) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/PythonObjectParser.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/PythonObjectParser.kt index 07e113458a..b4edda2a64 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/PythonObjectParser.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/PythonObjectParser.kt @@ -180,7 +180,7 @@ fun MemoryObject.toPythonTree(memoryDump: MemoryDump): PythonTree.PythonTreeNode val dictitemsObjs = memoryDump.getById(dictitems) as DictMemoryObject PythonTree.ReduceNode( this.id.toLong(), - PythonClassId(this.kind, this.module), + PythonClassId(this.module, this.kind), PythonClassId(this.constructor), arguments.items.map { memoryDump.getById(it).toPythonTree(memoryDump) }, stateObjs.items.entries.associate { diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonApi.kt index c5555c83f8..1981a5838d 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonApi.kt @@ -1,6 +1,5 @@ package org.utbot.python.framework.api.python -import org.utbot.common.withToStringThreadLocalReentrancyGuard import org.utbot.framework.plugin.api.ClassId import org.utbot.framework.plugin.api.MethodId import org.utbot.framework.plugin.api.UtModel @@ -75,7 +74,7 @@ class PythonTreeModel( override fun equals(other: Any?): Boolean { if (other is PythonTreeModel) { - return tree == other.tree + return tree.softEquals(other.tree) } return false } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonCgVariableConstructor.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonCgVariableConstructor.kt index 734c3f7056..283c392466 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonCgVariableConstructor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonCgVariableConstructor.kt @@ -6,8 +6,6 @@ import org.utbot.framework.codegen.domain.models.CgLiteral import org.utbot.framework.codegen.domain.models.CgMethodCall import org.utbot.framework.codegen.domain.models.CgStatement import org.utbot.framework.codegen.domain.models.CgValue -import org.utbot.framework.codegen.domain.models.CgVariable -import org.utbot.framework.codegen.tree.CgComponents import org.utbot.framework.codegen.tree.CgVariableConstructor import org.utbot.framework.plugin.api.ConstructorId import org.utbot.framework.plugin.api.FieldId @@ -18,8 +16,6 @@ import org.utbot.python.framework.codegen.PythonCgLanguageAssistant import org.utbot.python.framework.codegen.model.tree.* class PythonCgVariableConstructor(cgContext: CgContext) : CgVariableConstructor(cgContext) { - private val nameGenerator = CgComponents.getNameGeneratorBy(context) - override fun getOrCreateVariable(model: UtModel, name: String?): CgValue { return valueByModel.getOrPut(model) { when (model) { @@ -73,7 +69,7 @@ class PythonCgVariableConstructor(cgContext: CgContext) : CgVariableConstructor( if (assistant.memoryObjects.containsKey(id)) { val tree = assistant.memoryObjectsModels[id] val savedObj = assistant.memoryObjects[id] - if (tree == objectNode && savedObj != null) { + if (tree != null && savedObj != null && tree.softEquals(objectNode)) { return Pair(savedObj, emptyList()) } } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/utils/RequirementsUtils.kt b/utbot-python/src/main/kotlin/org/utbot/python/utils/RequirementsUtils.kt index e2fe93e2ae..f600bf084d 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/utils/RequirementsUtils.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/utils/RequirementsUtils.kt @@ -4,7 +4,7 @@ object RequirementsUtils { val requirements: List = listOf( "mypy==1.0.0", "coverage==6.5.0", - "utbot-executor==1.1.37", + "utbot-executor==1.2.0", "utbot-mypy-runner==0.2.8", ) From 136c72d73b1dd99a0fb116780e6e70f39201b6cb Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Thu, 16 Mar 2023 11:03:28 +0300 Subject: [PATCH 4/6] Add PythonTreeComparator --- .../python/framework/api/python/PythonApi.kt | 7 +- .../api/python/util/PythonTreeComparator.kt | 93 +++++ .../tree/PythonCgMethodConstructor.kt | 5 +- .../python/utils/PythonTreeComparatorTest.kt | 380 ++++++++++++++++++ 4 files changed, 480 insertions(+), 5 deletions(-) create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonTreeComparator.kt create mode 100644 utbot-python/src/test/kotlin/org/utbot/python/framework/api/python/utils/PythonTreeComparatorTest.kt diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonApi.kt index 1981a5838d..5b29dc2520 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonApi.kt @@ -3,6 +3,7 @@ package org.utbot.python.framework.api.python import org.utbot.framework.plugin.api.ClassId import org.utbot.framework.plugin.api.MethodId import org.utbot.framework.plugin.api.UtModel +import org.utbot.python.framework.api.python.util.comparePythonTree import org.utbot.python.framework.api.python.util.moduleOfType /** @@ -73,10 +74,10 @@ class PythonTreeModel( } override fun equals(other: Any?): Boolean { - if (other is PythonTreeModel) { - return tree.softEquals(other.tree) + if (other !is PythonTreeModel) { + return false } - return false + return comparePythonTree(tree, other.tree) } override fun hashCode(): Int { diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonTreeComparator.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonTreeComparator.kt new file mode 100644 index 0000000000..609c63871a --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonTreeComparator.kt @@ -0,0 +1,93 @@ +package org.utbot.python.framework.api.python.util + +import org.utbot.python.framework.api.python.PythonTree + +enum class VisitStatus { + OPENED, CLOSED +} + +fun comparePythonTree( + left: PythonTree.PythonTreeNode, + right: PythonTree.PythonTreeNode, + visitedLeft: MutableMap = emptyMap().toMutableMap(), + visitedRight: MutableMap = emptyMap().toMutableMap(), + equals: MutableMap, Boolean> = emptyMap, Boolean>().toMutableMap(), + ): Boolean { + if (visitedLeft[left.id] != visitedRight[right.id]) { + visitedLeft[left.id] = VisitStatus.CLOSED + visitedRight[right.id] = VisitStatus.CLOSED + equals[left.id to right.id] = false + return false + } + if (visitedLeft[left.id] == VisitStatus.CLOSED) { + return equals[left.id to right.id]!! + } + if (visitedLeft[left.id] == VisitStatus.OPENED) { + return true + } + + visitedLeft[left.id] = VisitStatus.OPENED + visitedRight[right.id] = VisitStatus.OPENED + + val areEquals = if (left.comparable && right.comparable && left.type == right.type) { + when (left) { + is PythonTree.PrimitiveNode -> { + left == right + } + + is PythonTree.ListNode -> { + if (right !is PythonTree.ListNode) false + else if (left.items.keys != right.items.keys) false + else left.items.keys.all { comparePythonTree(left.items[it]!!, right.items[it]!!, visitedLeft, visitedRight, equals) } + } + + is PythonTree.DictNode -> { + if (right !is PythonTree.DictNode) false + else if (left.items.keys != right.items.keys) false + else left.items.keys.all { comparePythonTree(left.items[it]!!, right.items[it]!!, visitedLeft, visitedRight, equals) } + } + + is PythonTree.TupleNode -> { + if (right !is PythonTree.TupleNode) false + else if (left.items.keys != right.items.keys) false + else left.items.keys.all { comparePythonTree(left.items[it]!!, right.items[it]!!, visitedLeft, visitedRight, equals) } + } + + is PythonTree.SetNode -> { + if (right !is PythonTree.SetNode) false + else if (left.items.size != right.items.size) false + else left.items.sortedBy { it.id } + .zip(right.items.sortedBy { it.id }) + .all { comparePythonTree(it.first, it.second, visitedLeft, visitedRight, equals) } + } + + is PythonTree.ReduceNode -> { + if (right !is PythonTree.ReduceNode) false + else { + val state = left.state.size == right.state.size && left.state.keys.all { + comparePythonTree( + left.state[it]!!, + right.state[it]!!, + visitedLeft, + visitedRight, + equals + ) + } + val listitems = left.listitems.size == right.listitems.size && left.listitems.zip(right.listitems) + .all { comparePythonTree(it.first, it.second, visitedLeft, visitedRight, equals) } + val dictitems = left.dictitems.keys == right.dictitems.keys && left.dictitems.keys + .all { comparePythonTree(left.dictitems[it]!!, right.dictitems[it]!!, visitedLeft, visitedRight, equals) } + + state && listitems && dictitems + } + } + + else -> false + } + } else left.id == right.id + + visitedLeft[left.id] = VisitStatus.CLOSED + visitedRight[right.id] = VisitStatus.CLOSED + equals[left.id to right.id] = areEquals + return areEquals +} \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonCgMethodConstructor.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonCgMethodConstructor.kt index b9c143b4b1..8e0c4e29d1 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonCgMethodConstructor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonCgMethodConstructor.kt @@ -11,6 +11,7 @@ import org.utbot.framework.codegen.domain.models.CgVariable import org.utbot.framework.codegen.tree.CgMethodConstructor import org.utbot.framework.plugin.api.* import org.utbot.python.framework.api.python.* +import org.utbot.python.framework.api.python.util.comparePythonTree import org.utbot.python.framework.api.python.util.pythonIntClassId import org.utbot.python.framework.api.python.util.pythonNoneClassId import org.utbot.python.framework.codegen.PythonCgLanguageAssistant @@ -62,7 +63,7 @@ class PythonCgMethodConstructor(context: CgContext) : CgMethodConstructor(contex val afterThisInstance = execution.stateAfter.thisInstance val assertThisObject = emptyList>().toMutableList() if (beforeThisInstance is PythonTreeModel && afterThisInstance is PythonTreeModel) { - if (PythonTreeWrapper(beforeThisInstance.tree) != PythonTreeWrapper(afterThisInstance.tree)) { + if (!comparePythonTree(beforeThisInstance.tree, afterThisInstance.tree)) { thisInstance = thisInstance?.let { val newValue = if (it is CgPythonTree) { @@ -91,7 +92,7 @@ class PythonCgMethodConstructor(context: CgContext) : CgMethodConstructor(contex val afterValue = execution.stateAfter.parameters[index] if (afterValue is PythonTreeModel && param is PythonTreeModel) { - if (PythonTreeWrapper(afterValue.tree) != PythonTreeWrapper(param.tree)) { + if (!comparePythonTree(afterValue.tree, param.tree)) { if (argument !is CgVariable) { argument = newVar(argument.type, name) {argument} } diff --git a/utbot-python/src/test/kotlin/org/utbot/python/framework/api/python/utils/PythonTreeComparatorTest.kt b/utbot-python/src/test/kotlin/org/utbot/python/framework/api/python/utils/PythonTreeComparatorTest.kt new file mode 100644 index 0000000000..77b2177eac --- /dev/null +++ b/utbot-python/src/test/kotlin/org/utbot/python/framework/api/python/utils/PythonTreeComparatorTest.kt @@ -0,0 +1,380 @@ +package org.utbot.python.framework.api.python.utils + +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.Test +import org.utbot.python.framework.api.python.PythonClassId +import org.utbot.python.framework.api.python.PythonTree +import org.utbot.python.framework.api.python.util.comparePythonTree +import org.utbot.python.framework.api.python.util.pythonIntClassId +import org.utbot.python.framework.api.python.util.pythonStrClassId + +internal class PythonTreeComparatorTest { + @Test + fun testEqualPrimitive() { + val left = PythonTree.PrimitiveNode(pythonIntClassId, "1") + + assertTrue(comparePythonTree(left, left)) + } + + @Test + fun testEqualList() { + val left = PythonTree.ListNode(mapOf( + 0 to PythonTree.PrimitiveNode(pythonIntClassId, "1"), + 1 to PythonTree.PrimitiveNode(pythonIntClassId, "2"), + ).toMutableMap()) + + assertTrue(comparePythonTree(left, left)) + } + + @Test + fun testEqualTuple() { + val left = PythonTree.TupleNode(mapOf( + 0 to PythonTree.PrimitiveNode(pythonIntClassId, "1"), + 1 to PythonTree.PrimitiveNode(pythonIntClassId, "2"), + ).toMutableMap()) + + assertTrue(comparePythonTree(left, left)) + } + + @Test + fun testEqualDict() { + val left = PythonTree.DictNode(mapOf( + PythonTree.PrimitiveNode(pythonStrClassId, "'a'") to PythonTree.PrimitiveNode(pythonIntClassId, "1"), + PythonTree.PrimitiveNode(pythonStrClassId, "'b'") to PythonTree.PrimitiveNode(pythonIntClassId, "2"), + ).toMutableMap()) + + assertTrue(comparePythonTree(left, left)) + } + + @Test + fun testEqualSet() { + val left = PythonTree.SetNode(setOf( + PythonTree.PrimitiveNode(pythonIntClassId, "1"), + PythonTree.PrimitiveNode(pythonIntClassId, "2"), + ).toMutableSet()) + + assertTrue(comparePythonTree(left, left)) + } + + @Test + fun testEqualEmptyReduce() { + val left = PythonTree.ReduceNode( + PythonClassId("my_module", "MyClass"), + PythonClassId("my_module.MyClass"), + listOf(PythonTree.PrimitiveNode(pythonIntClassId, "2")), + ) + + assertTrue(comparePythonTree(left, left)) + } + + @Test + fun testEqualNotEmptyReduce() { + val left = PythonTree.ReduceNode( + PythonClassId("my_module", "MyClass"), + PythonClassId("my_module.MyClass"), + listOf(PythonTree.PrimitiveNode(pythonIntClassId, "2")), + ) + left.state["my_field"] = PythonTree.PrimitiveNode(pythonIntClassId, "2") + left.state["my_field_1"] = PythonTree.PrimitiveNode(pythonIntClassId, "1") + + assertTrue(comparePythonTree(left, left)) + } + + @Test + fun testEqualReduceWithListItems() { + val left = PythonTree.ReduceNode( + PythonClassId("my_module", "MyClass"), + PythonClassId("my_module.MyClass"), + listOf(PythonTree.PrimitiveNode(pythonIntClassId, "2")), + ) + left.listitems = listOf( + PythonTree.PrimitiveNode(pythonIntClassId, "1"), + PythonTree.PrimitiveNode(pythonIntClassId, "2"), + ) + + assertTrue(comparePythonTree(left, left)) + } + + @Test + fun testEqualReduceWithDictItems() { + val left = PythonTree.ReduceNode( + PythonClassId("my_module", "MyClass"), + PythonClassId("my_module.MyClass"), + listOf(PythonTree.PrimitiveNode(pythonIntClassId, "2")), + ) + left.dictitems = mapOf( + PythonTree.PrimitiveNode(pythonStrClassId, "'a'") to PythonTree.PrimitiveNode(pythonIntClassId, "1"), + PythonTree.PrimitiveNode(pythonStrClassId, "'b'") to PythonTree.PrimitiveNode(pythonIntClassId, "2"), + ) + + assertTrue(comparePythonTree(left, left)) + } + + @Test + fun testEqualRecursiveReduce() { + val left = PythonTree.ReduceNode( + PythonClassId("my_module", "MyClass"), + PythonClassId("my_module.MyClass"), + listOf(), + ) + val child = PythonTree.ReduceNode( + PythonClassId("my_module", "MyClass"), + PythonClassId("my_module.MyClass"), + listOf(), + ) + val child2 = PythonTree.ReduceNode( + PythonClassId("my_module", "MyClass"), + PythonClassId("my_module.MyClass"), + listOf(), + ) + left.state["children"] = PythonTree.ListNode( + mapOf(0 to child).toMutableMap() + ) + child.state["children"] = PythonTree.ListNode( + mapOf(0 to child2).toMutableMap() + ) + child2.state["children"] = PythonTree.ListNode( + mapOf(0 to left).toMutableMap() + ) + + assertTrue(comparePythonTree(left, left)) + } + + @Test + fun testEqualHardRecursiveReduce() { + val left = PythonTree.ReduceNode( + PythonClassId("my_module", "MyClass"), + PythonClassId("my_module.MyClass"), + listOf(), + ) + val child = PythonTree.ReduceNode( + PythonClassId("my_module", "MyClass"), + PythonClassId("my_module.MyClass"), + listOf(), + ) + val child2 = PythonTree.ReduceNode( + PythonClassId("my_module", "MyClass"), + PythonClassId("my_module.MyClass"), + listOf(), + ) + val child3 = PythonTree.ReduceNode( + PythonClassId("my_module", "MyClass"), + PythonClassId("my_module.MyClass"), + listOf(), + ) + val child4 = PythonTree.ReduceNode( + PythonClassId("my_module", "MyClass"), + PythonClassId("my_module.MyClass"), + listOf(), + ) + val child5 = PythonTree.ReduceNode( + PythonClassId("my_module", "MyClass"), + PythonClassId("my_module.MyClass"), + listOf(), + ) + val child6 = PythonTree.ReduceNode( + PythonClassId("my_module", "MyClass"), + PythonClassId("my_module.MyClass"), + listOf(), + ) + left.state["children"] = PythonTree.ListNode(mapOf( + 0 to child, + 1 to child2, + 2 to child3, + ).toMutableMap()) + child.state["children"] = PythonTree.ListNode(mapOf( + 0 to child2, + 1 to child3, + ).toMutableMap()) + child2.state["children"] = PythonTree.ListNode(mapOf( + 0 to child2, + 1 to child3, + 2 to left, + 3 to child4, + ).toMutableMap()) + child3.state["children"] = PythonTree.ListNode(mapOf( + 0 to left, + ).toMutableMap()) + child4.state["children"] = PythonTree.ListNode(mapOf( + 0 to left, + 1 to child5, + ).toMutableMap()) + child5.state["children"] = PythonTree.ListNode(mapOf( + 0 to child2, + ).toMutableMap()) + child6.state["children"] = PythonTree.ListNode(mapOf( + 0 to child2, + ).toMutableMap()) + + assertTrue(comparePythonTree(left, left)) + } + + @Test + fun testNotEqualPrimitive() { + val left = PythonTree.PrimitiveNode(pythonIntClassId, "1") + val right = PythonTree.PrimitiveNode(pythonIntClassId, "2") + + assertFalse(comparePythonTree(left, right)) + } + + @Test + fun testNotEqualList() { + val left = PythonTree.ListNode(mapOf( + 0 to PythonTree.PrimitiveNode(pythonIntClassId, "1"), + 1 to PythonTree.PrimitiveNode(pythonIntClassId, "2"), + ).toMutableMap()) + val right = PythonTree.ListNode(mapOf( + 0 to PythonTree.PrimitiveNode(pythonIntClassId, "0"), + 1 to PythonTree.PrimitiveNode(pythonIntClassId, "2"), + ).toMutableMap()) + + assertFalse(comparePythonTree(left, right)) + } + + @Test + fun testNotEqualTuple() { + val left = PythonTree.TupleNode(mapOf( + 0 to PythonTree.PrimitiveNode(pythonIntClassId, "1"), + 1 to PythonTree.PrimitiveNode(pythonIntClassId, "2"), + ).toMutableMap()) + val right = PythonTree.TupleNode(mapOf( + 0 to PythonTree.PrimitiveNode(pythonIntClassId, "0"), + 1 to PythonTree.PrimitiveNode(pythonIntClassId, "2"), + ).toMutableMap()) + + assertFalse(comparePythonTree(left, right)) + } + + @Test + fun testNotEqualDict() { + val left = PythonTree.DictNode(mapOf( + PythonTree.PrimitiveNode(pythonStrClassId, "'c'") to PythonTree.PrimitiveNode(pythonIntClassId, "1"), + PythonTree.PrimitiveNode(pythonStrClassId, "'b'") to PythonTree.PrimitiveNode(pythonIntClassId, "0"), + ).toMutableMap()) + val right = PythonTree.DictNode(mapOf( + PythonTree.PrimitiveNode(pythonStrClassId, "'a'") to PythonTree.PrimitiveNode(pythonIntClassId, "1"), + PythonTree.PrimitiveNode(pythonStrClassId, "'b'") to PythonTree.PrimitiveNode(pythonIntClassId, "2"), + ).toMutableMap()) + + assertFalse(comparePythonTree(left, right)) + } + + @Test + fun testNotEqualSet() { + val left = PythonTree.SetNode(setOf( + PythonTree.PrimitiveNode(pythonIntClassId, "1"), + PythonTree.PrimitiveNode(pythonIntClassId, "2"), + ).toMutableSet()) + val right = PythonTree.SetNode(setOf( + PythonTree.PrimitiveNode(pythonIntClassId, "0"), + PythonTree.PrimitiveNode(pythonIntClassId, "2"), + ).toMutableSet()) + + assertFalse(comparePythonTree(left, right)) + } + + @Test + fun testNotEqualListDiffSize() { + val left = PythonTree.ListNode(mapOf( + 0 to PythonTree.PrimitiveNode(pythonIntClassId, "1"), + 1 to PythonTree.PrimitiveNode(pythonIntClassId, "2"), + ).toMutableMap()) + val right = PythonTree.ListNode(mapOf( + 1 to PythonTree.PrimitiveNode(pythonIntClassId, "2"), + ).toMutableMap()) + + assertFalse(comparePythonTree(left, right)) + } + + @Test + fun testNotEqualTupleDiffSize() { + val left = PythonTree.TupleNode(mapOf( + 0 to PythonTree.PrimitiveNode(pythonIntClassId, "1"), + 1 to PythonTree.PrimitiveNode(pythonIntClassId, "2"), + ).toMutableMap()) + val right = PythonTree.TupleNode(mapOf( + 1 to PythonTree.PrimitiveNode(pythonIntClassId, "2"), + ).toMutableMap()) + + assertFalse(comparePythonTree(left, right)) + } + + @Test + fun testNotEqualDictDiffSize() { + val left = PythonTree.DictNode(mapOf( + PythonTree.PrimitiveNode(pythonStrClassId, "'c'") to PythonTree.PrimitiveNode(pythonIntClassId, "1"), + PythonTree.PrimitiveNode(pythonStrClassId, "'b'") to PythonTree.PrimitiveNode(pythonIntClassId, "2"), + ).toMutableMap()) + val right = PythonTree.DictNode(mapOf( + PythonTree.PrimitiveNode(pythonStrClassId, "'b'") to PythonTree.PrimitiveNode(pythonIntClassId, "2"), + ).toMutableMap()) + + assertFalse(comparePythonTree(left, right)) + } + + @Test + fun testNotEqualSetDiffSize() { + val left = PythonTree.SetNode(setOf( + PythonTree.PrimitiveNode(pythonIntClassId, "1"), + PythonTree.PrimitiveNode(pythonIntClassId, "2"), + ).toMutableSet()) + val right = PythonTree.SetNode(setOf( + PythonTree.PrimitiveNode(pythonIntClassId, "2"), + ).toMutableSet()) + + assertFalse(comparePythonTree(left, right)) + } + + @Test + fun testNotEqualRecursiveReduce() { + val left = PythonTree.ReduceNode( + PythonClassId("my_module", "MyClass"), + PythonClassId("my_module.MyClass"), + listOf(), + ) + val leftChild1 = PythonTree.ReduceNode( + PythonClassId("my_module", "MyClass"), + PythonClassId("my_module.MyClass"), + listOf(), + ) + val leftChild2 = PythonTree.ReduceNode( + PythonClassId("my_module", "MyClass"), + PythonClassId("my_module.MyClass"), + listOf(), + ) + left.state["children"] = PythonTree.ListNode( + mapOf(0 to leftChild1).toMutableMap() + ) + leftChild1.state["children"] = PythonTree.ListNode( + mapOf(0 to leftChild2).toMutableMap() + ) + leftChild2.state["children"] = PythonTree.ListNode( + mapOf(0 to left).toMutableMap() + ) + + val right = PythonTree.ReduceNode( + PythonClassId("my_module", "MyClass"), + PythonClassId("my_module.MyClass"), + listOf(), + ) + val rightChild1 = PythonTree.ReduceNode( + PythonClassId("my_module", "MyClass"), + PythonClassId("my_module.MyClass"), + listOf(), + ) + val rightChild2 = PythonTree.ReduceNode( + PythonClassId("my_module", "MyClass"), + PythonClassId("my_module.MyClass"), + listOf(), + ) + right.state["children"] = PythonTree.ListNode(mapOf( + 0 to rightChild1, + 1 to rightChild2, + ).toMutableMap()) + rightChild1.state["children"] = PythonTree.ListNode(mapOf( + 0 to rightChild2, + ).toMutableMap()) + + assertFalse(comparePythonTree(left, right)) + } +} \ No newline at end of file From 15ce6198810fce6648358cd435b72835194eea81 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Thu, 16 Mar 2023 11:05:42 +0300 Subject: [PATCH 5/6] Remove files --- .../normalize_annotation_from_project.py | 50 --- .../main/resources/python_tree_serializer.py | 206 ----------- .../src/main/resources/requirements.txt | 4 - .../src/main/resources/typeshed_stub.py | 322 ------------------ 4 files changed, 582 deletions(-) delete mode 100644 utbot-python/src/main/resources/normalize_annotation_from_project.py delete mode 100644 utbot-python/src/main/resources/python_tree_serializer.py delete mode 100644 utbot-python/src/main/resources/requirements.txt delete mode 100644 utbot-python/src/main/resources/typeshed_stub.py diff --git a/utbot-python/src/main/resources/normalize_annotation_from_project.py b/utbot-python/src/main/resources/normalize_annotation_from_project.py deleted file mode 100644 index 5d3e29d5f7..0000000000 --- a/utbot-python/src/main/resources/normalize_annotation_from_project.py +++ /dev/null @@ -1,50 +0,0 @@ -import importlib.machinery -import inspect -import sys -import types -import mypy.fastparse - - -def main(annotation: str, cur_module: str, path: str): - def walk_mypy_type(mypy_type) -> str: - try: - source = inspect.getfile(eval(mypy_type.name)) - # in_project = source.startswith(project_root) - except: - None - - modname = eval(mypy_type.name).__module__ - simple_name = mypy_type.name.split('.')[-1] - fullname = f'{modname}.{simple_name}' - - result = fullname - if len(mypy_type.args) != 0: - arg_strs = [ - walk_mypy_type(arg) - for arg in mypy_type.args - ] - result += f"[{', '.join(arg_strs)}]" - return result - - loader = importlib.machinery.SourceFileLoader(cur_module, path) - mod = types.ModuleType(loader.name) - loader.exec_module(mod) - - for name in dir(mod): - globals()[name] = getattr(mod, name) - - mypy_type_ = mypy.fastparse.parse_type_string(annotation, annotation, -1, -1) - print(walk_mypy_type(mypy_type_), end='') - - -def get_args(): - annotation = sys.argv[1] - cur_module = sys.argv[2] - path = sys.argv[3] - for extra_path in sys.argv[4:]: - sys.path.append(extra_path) - return annotation, cur_module, path - - -if __name__ == '__main__': - main(*get_args()) diff --git a/utbot-python/src/main/resources/python_tree_serializer.py b/utbot-python/src/main/resources/python_tree_serializer.py deleted file mode 100644 index 557cb56954..0000000000 --- a/utbot-python/src/main/resources/python_tree_serializer.py +++ /dev/null @@ -1,206 +0,0 @@ -import copy -import pickle -import types -from itertools import zip_longest -import copyreg -import importlib - - -class _PythonTreeSerializer: - class MemoryObj: - def __init__(self, json): - self.json = json - self.deserialized_obj = None - self.comparable = False - self.is_draft = True - - def __init__(self): - self.memory = {} - - def memory_view(self): - return ' | '.join(f'{id_}: {obj.deserialized_obj}' for id_, obj in self.memory.items()) - - @staticmethod - def get_type(py_object): - if py_object is None: - return 'types.NoneType' - module = type(py_object).__module__ - return '{module}.{name}'.format( - module=module, - name=type(py_object).__name__, - ) - - @staticmethod - def get_type_name(type_): - if type_ is None: - return 'types.NoneType' - return '{module}.{name}'.format( - module=type_.__module__, - name=type_.__name__, - ) - - @staticmethod - def has_reduce(py_object) -> bool: - if getattr(py_object, '__reduce__', None) is None: - return False - else: - try: - py_object.__reduce__() - return True - except TypeError: - return False - - def save_to_memory(self, id_, py_json, deserialized_obj): - mem_obj = _PythonTreeSerializer.MemoryObj(py_json) - mem_obj.deserialized_obj = deserialized_obj - self.memory[id_] = mem_obj - return mem_obj - - def get_reduce(self, py_object): - id_ = id(py_object) - - py_object_reduce = py_object.__reduce__() - reduce_value = [ - default if obj is None else obj - for obj, default in zip_longest( - py_object_reduce, - [None, [], {}, [], []], - fillvalue=None - ) - ] - - constructor = _PythonTreeSerializer.get_type_name(reduce_value[0]) - args, deserialized_args = _PythonTreeSerializer.unzip_list([ - self.serialize(arg) - for arg in reduce_value[1] - ]) - json_obj = { - 'id': id_, - 'type': _PythonTreeSerializer.get_type(py_object), - 'constructor': constructor, - 'args': args, - 'state': [], - 'listitems': [], - 'dictitems': [], - } - deserialized_obj = reduce_value[0](*deserialized_args) - memory_obj = self.save_to_memory(id_, json_obj, deserialized_obj) - - state, deserialized_state = self.unzip_dict([ - (attr, self.serialize(value)) - for attr, value in reduce_value[2].items() - ], skip_first=True) - listitems, deserialized_listitems = self.unzip_list([ - self.serialize(item) - for item in reduce_value[3] - ]) - dictitems, deserialized_dictitems = self.unzip_dict([ - (self.serialize(key), self.serialize(value)) - for key, value in reduce_value[4] - ]) - - memory_obj.json['state'] = state - memory_obj.json['listitems'] = listitems - memory_obj.json['dictitems'] = dictitems - - for key, value in deserialized_state.items(): - setattr(deserialized_obj, key, value) - for item in deserialized_listitems: - deserialized_obj.append(item) - for key, value in deserialized_dictitems.items(): - deserialized_obj[key] = value - - memory_obj.deserialized_obj = deserialized_obj - memory_obj.is_draft = False - - return id_, deserialized_obj - - def serialize(self, py_object): - type_ = _PythonTreeSerializer.get_type(py_object) - id_ = id(py_object) - skip_comparable = False - comparable = True - - if id_ in self.memory: - value = id_ - strategy = 'memory' - skip_comparable = True - comparable = False - deserialized_obj = self.memory[id_].deserialized_obj - if not self.memory[id_].is_draft: - self.memory[id_].comparable = py_object == deserialized_obj - skip_comparable = False - elif isinstance(py_object, type): - value = _PythonTreeSerializer.get_type_name(py_object) - strategy = 'repr' - deserialized_obj = py_object - elif any(type(py_object) == t for t in (list, set, tuple)): - elements = [ - self.serialize(element) for element in py_object - ] - value, deserialized_obj = _PythonTreeSerializer.unzip_list(elements, type(py_object)) - comparable = all([element['comparable'] for element in value]) - strategy = 'generic' - elif type(py_object) == dict: - elements = [ - [self.serialize(key), self.serialize(value)] - for key, value in py_object.items() - ] - value, deserialized_obj = _PythonTreeSerializer.unzip_dict(elements) - comparable = all([element[1]['comparable'] for element in value]) - strategy = 'generic' - elif _PythonTreeSerializer.has_reduce(py_object): - value, deserialized_obj = self.get_reduce(py_object) - strategy = 'memory' - else: - value = repr(py_object) - try: - deserialized_obj = pickle.loads(pickle.dumps(py_object)) - except Exception: - deserialized_obj = py_object - skip_comparable = True - comparable = False - strategy = 'repr' - - if not skip_comparable: - try: - comparable = comparable and (py_object == deserialized_obj) - except Exception: - comparable = False - - return { - 'type': type_, - 'value': value, - 'strategy': strategy, - 'comparable': comparable, - }, deserialized_obj - - @staticmethod - def unzip_list(elements, cast_second=list): - if len(elements) == 0: - first, second = [], [] - else: - first, second = list(zip(*elements)) - return first, cast_second(second) - - @staticmethod - def unzip_dict(elements, cast_second=dict, skip_first=False): - if len(elements) == 0: - first, second = [], [] - else: - if skip_first: - first = [[element[0], element[1][0]] for element in elements] - second = [[element[0], element[1][1]] for element in elements] - else: - first = [[element[0][0], element[1][0]] for element in elements] - second = [[element[0][1], element[1][1]] for element in elements] - return first, cast_second(second) - - def dumps(self, obj): - return { - 'json': self.serialize(obj)[0], - 'memory': { - key: value.json - for key, value in self.memory.items() - } - } diff --git a/utbot-python/src/main/resources/requirements.txt b/utbot-python/src/main/resources/requirements.txt deleted file mode 100644 index 5c2469fb5e..0000000000 --- a/utbot-python/src/main/resources/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -mypy==1.0.0 -coverage -utbot-executor==1.1.34 -utbot-mypy-runner==0.2.8 diff --git a/utbot-python/src/main/resources/typeshed_stub.py b/utbot-python/src/main/resources/typeshed_stub.py deleted file mode 100644 index 7d77a113a2..0000000000 --- a/utbot-python/src/main/resources/typeshed_stub.py +++ /dev/null @@ -1,322 +0,0 @@ -import ast -import importlib -import json -import sys -import os - -import mypy.fastparse - -from contextlib import contextmanager -from collections import defaultdict - -import astor -from typeshed_client import get_stub_names, get_search_context, OverloadedName - - -def normalize_annotation(annotation, module_of_annotation): - def walk_mypy_type(mypy_type): - try: - prefix = f'{module_of_annotation}.' if len(module_of_annotation) > 0 else '' - - if mypy_type.name[:len(prefix)] == prefix: - name = mypy_type.name[len(prefix):] - else: - name = mypy_type.name - - if eval(name) is None: - result = "types.NoneType" - else: - modname = eval(name).__module__ - result = f'{modname}.{name}' - - except Exception as e: - result = 'typing.Any' - - if hasattr(mypy_type, 'args') and len(mypy_type.args) != 0: - arg_strs = [ - walk_mypy_type(arg) - for arg in mypy_type.args - ] - result += f"[{', '.join(arg_strs)}]" - - return result - - mod = importlib.import_module(module_of_annotation) - - for name in dir(mod): - globals()[name] = getattr(mod, name) - - mypy_type_ = mypy.fastparse.parse_type_string(annotation, annotation, -1, -1) - return walk_mypy_type(mypy_type_) - - -class AstClassEncoder(json.JSONEncoder): - def default(self, o): - if isinstance(o, ast.ClassDef): - json_dump = { - 'className': o.name, - 'methods': [], - 'fields': [], - } - - def _function_statements_handler(_statement): - if isinstance(_statement, ast.FunctionDef): - method = AstFunctionDefEncoder().default(_statement) - is_property = method['is_property'] - del method['is_property'] - if is_property: - del method['args'] - del method['kwonlyargs'] - - method['annotation'] = method['returns'] - del method['returns'] - - json_dump['fields'].append(method) - else: - json_dump['methods'].append(method) - if isinstance(_statement, ast.AnnAssign): - field = AstAnnAssignEncoder().default(_statement) - json_dump['fields'].append(field) - - for statement in o.body: - _function_statements_handler(statement) - - return json_dump - return json.JSONEncoder.default(self, o) - - -class AstAnnAssignEncoder(json.JSONEncoder): - def default(self, o): - if isinstance(o, ast.AnnAssign): - json_dump = { - 'name': '...' if isinstance(o.target, type(Ellipsis)) else o.target.id, - 'annotation': transform_annotation(o.annotation), - } - return json_dump - return json.JSONEncoder.default(self, o) - - -def find_init_method(function_ast): - for statement in function_ast.body: - if isinstance(statement, ast.FunctionDef) and statement.name == '__init__': - return statement - return None - - -class AstFunctionDefEncoder(json.JSONEncoder): - def default(self, o): - if isinstance(o, (ast.FunctionDef, ast.AsyncFunctionDef)): - json_dump = { - 'name': o.name, - 'returns': transform_annotation(o.returns), - 'args': [ - AstArgEncoder().default(arg) - for arg in o.args.args - ], - 'kwonlyargs': [ - AstArgEncoder().default(arg) - for arg in o.args.kwonlyargs - ], - 'is_property': function_is_property(o), - } - return json_dump - - -def function_is_property(function): - return bool(any([ - 'property' == astor.code_gen.to_source(decorator).strip() - for decorator in function.decorator_list - ])) - - -class AstArgEncoder(json.JSONEncoder): - def default(self, o): - if isinstance(o, ast.arg): - json_dump = { - 'arg': o.arg, - 'annotation': transform_annotation(o.annotation) - } - return json_dump - - -class AstConstantEncoder(json.JSONEncoder): - def default(self, o): - if isinstance(o, ast.Constant): - json_dump = '...' if isinstance(o.value, type(Ellipsis)) else o.value - - return json_dump - if isinstance(o, type(Ellipsis)): - return '...' - if o is None: - return None - - -def transform_annotation(annotation): - return '' if annotation is None else astor.code_gen.to_source(annotation).strip() - - -def recursive_normalize_annotations(json_data, module_name): - if 'annotation' in json_data: - json_data['annotation'] = normalize_annotation( - annotation=json_data['annotation'], - module_of_annotation=module_name - ) - elif 'returns' in json_data: - json_data['returns'] = normalize_annotation( - annotation=json_data['returns'], - module_of_annotation=module_name - ) - json_data['args'] = [ - recursive_normalize_annotations(arg, module_name) - for arg in json_data['args'] - ] - json_data['kwonlyargs'] = [ - recursive_normalize_annotations(arg, module_name) - for arg in json_data['kwonlyargs'] - ] - elif 'className' in json_data: - for key, value in json_data.items(): - if key in {'methods', 'fields'}: - json_data[key] = [ - recursive_normalize_annotations(elem, module_name) - for elem in value - ] - else: - for key, value in json_data.items(): - json_data[key] = [ - recursive_normalize_annotations(elem, module_name) - for elem in value - ] - - return json_data - - -class StubFileCollector: - def __init__(self, python_version): - self.methods_dataset = defaultdict(list) - self.fields_dataset = defaultdict(list) - self.functions_dataset = defaultdict(list) - self.classes_dataset = [] - self.assigns_dataset = defaultdict(list) - self.ann_assigns_dataset = defaultdict(list) - self.python_version = python_version - self.visited_modules = [] - - def create_module_table(self, module_name): - self.visited_modules.append(module_name) - - stub = get_stub_names( - module_name, - search_context=get_search_context(version=self.python_version) - ) - - def _ast_handler(ast_): - if isinstance(ast_, OverloadedName): - for definition in ast_.definitions: - _ast_handler(definition) - else: - if isinstance(ast_, ast.ClassDef): - json_data = AstClassEncoder().default(ast_) - recursive_normalize_annotations(json_data, module_name) - - if not ast_.name.startswith('_'): - class_name = f'{module_name}.{ast_.name}' - json_data['className'] = class_name - self.classes_dataset.append(json_data) - - for method in json_data['methods']: - method['className'] = class_name - self.methods_dataset[method['name']].append(method) - - for field in json_data['fields']: - field['className'] = class_name - self.fields_dataset[field['name']].append(field) - - elif isinstance(ast_, (ast.FunctionDef, ast.AsyncFunctionDef)): - json_data = AstFunctionDefEncoder().default(ast_) - recursive_normalize_annotations(json_data, module_name) - - function_name = f'{module_name}.{ast_.name}' - json_data['name'] = function_name - json_data['className'] = None - self.functions_dataset[ast_.name].append(json_data) - - else: - pass - - ast_nodes = set() - - if stub is None: - return - - for name, name_info in stub.items(): - ast_nodes.add(name_info.ast.__class__.__name__) - _ast_handler(name_info.ast) - - def save_method_annotations(self): - return json.dumps({ - 'classAnnotations': self.classes_dataset, - 'fieldAnnotations': defaultdict_to_array(self.fields_dataset), - 'functionAnnotations': defaultdict_to_array(self.functions_dataset), - 'methodAnnotations': defaultdict_to_array(self.methods_dataset), - }) - - -def defaultdict_to_array(dataset): - return [ - { - 'name': name, - 'definitions': types, - } - for name, types in dataset.items() - ] - - -def parse_submodule(module_name, collector_): - collector_.create_module_table(module_name) - try: - submodules = [ - f'{module_name}.{submodule}' if module_name != 'builtins' else submodule - for submodule in importlib.import_module(module_name).__dir__() - ] - for submodule in submodules: - if type(eval(submodule)) == 'module' and submodule not in collector_.visited_modules: - parse_submodule(submodule, collector_) - except ModuleNotFoundError: - pass - except ImportError: - pass - except NameError: - pass - except AttributeError: - pass - - -@contextmanager -def suppress_stdout(): - with open(os.devnull, "w") as devnull: - old_stdout = sys.stdout - old_stderr = sys.stderr - sys.stdout = devnull - sys.stderr = devnull - try: - yield - finally: - sys.stdout = old_stdout - sys.stderr = old_stderr - - -def main(): - python_version = sys.version_info - modules = sys.argv[1:] - with suppress_stdout(): - collector = StubFileCollector((python_version.major, python_version.minor)) - for module in modules: - parse_submodule(module, collector) - result = collector.save_method_annotations() - sys.stdout.write(result) - - -if __name__ == '__main__': - main() - sys.exit(0) From 437370290d6fee7c67fd11d3636e0139ec4a6655 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Thu, 16 Mar 2023 12:08:50 +0300 Subject: [PATCH 6/6] Add docs comment and replace softEquals by comparePythonTree --- .../framework/api/python/util/PythonTreeComparator.kt | 6 ++++++ .../model/constructor/tree/PythonCgVariableConstructor.kt | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonTreeComparator.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonTreeComparator.kt index 609c63871a..af34cdb541 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonTreeComparator.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonTreeComparator.kt @@ -6,6 +6,12 @@ enum class VisitStatus { OPENED, CLOSED } +/* + * Compare python tree by structure. Returns false if: + * - objects have different types + * - incomparable and have different ids + * - have the same type but structures aren't equal recursively + */ fun comparePythonTree( left: PythonTree.PythonTreeNode, right: PythonTree.PythonTreeNode, diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonCgVariableConstructor.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonCgVariableConstructor.kt index 283c392466..dea85ddadb 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonCgVariableConstructor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonCgVariableConstructor.kt @@ -11,6 +11,7 @@ import org.utbot.framework.plugin.api.ConstructorId import org.utbot.framework.plugin.api.FieldId import org.utbot.framework.plugin.api.UtModel import org.utbot.python.framework.api.python.* +import org.utbot.python.framework.api.python.util.comparePythonTree import org.utbot.python.framework.api.python.util.pythonNoneClassId import org.utbot.python.framework.codegen.PythonCgLanguageAssistant import org.utbot.python.framework.codegen.model.tree.* @@ -69,7 +70,7 @@ class PythonCgVariableConstructor(cgContext: CgContext) : CgVariableConstructor( if (assistant.memoryObjects.containsKey(id)) { val tree = assistant.memoryObjectsModels[id] val savedObj = assistant.memoryObjects[id] - if (tree != null && savedObj != null && tree.softEquals(objectNode)) { + if (tree != null && savedObj != null && comparePythonTree(tree, objectNode)) { return Pair(savedObj, emptyList()) } }