Update of module utbot-python-types

utbot-python-types/build.gradle.kts

@@ -1,9 +1,9 @@
 val kotlinLoggingVersion: String? by rootProject
 
 dependencies {
-    implementation("com.squareup.moshi:moshi:1.11.0")
-    implementation("com.squareup.moshi:moshi-kotlin:1.11.0")
-    implementation("com.squareup.moshi:moshi-adapters:1.11.0")
+    implementation("com.squareup.moshi:moshi:1.14.0")
+    implementation("com.squareup.moshi:moshi-kotlin:1.14.0")
+    implementation("com.squareup.moshi:moshi-adapters:1.14.0")
     implementation(group = "io.github.microutils", name = "kotlin-logging", version = kotlinLoggingVersion)
 }
 
@@ -72,6 +72,29 @@ if (pythonInterpreter != null && pypiToken != null) {
     }
 }
 
+val uninstallMypyRunner =
+    if (pythonInterpreter != null) {
+        tasks.register<Exec>("uninstallMypyRunner") {
+            group = "python"
+            commandLine(
+                pythonInterpreter,
+                "-m",
+                "pip",
+                "uninstall",
+                "utbot_mypy_runner"
+            )
+            commandLine(
+                pythonInterpreter,
+                "-m",
+                "pip",
+                "cache",
+                "purge"
+            )
+        }
+    } else {
+        null
+    }
+
 val installMypyRunner =
     if (pythonInterpreter != null) {
         tasks.register<Exec>("installUtbotMypyRunner") {
@@ -96,6 +119,7 @@ val jsonDir = File(project.projectDir, "src/test/resources")
 
 fun getParamsForSample(sampleName: String): List<String> =
     listOf(
+        sampleName,
         File(samplesDir, "$sampleName.py").canonicalPath,
         sampleName,
         buildDir.canonicalPath,
@@ -110,12 +134,12 @@ val samplesInfo = listOf(
 )
 
 if (pythonInterpreter != null) {
-    val subtasks = samplesInfo.mapIndexed { index, params ->
-        tasks.register<JavaExec>("regenerateJsonForTests_$index") {
+    val subtasks = samplesInfo.map { params ->
+        tasks.register<JavaExec>("regenerateJsonForTests_${params.first()}") {
             dependsOn(installMypyRunner!!)
             group = "python_subtasks"
             classpath = sourceSets.test.get().runtimeClasspath
-            args = listOf(pythonInterpreter) + params
+            args = listOf(pythonInterpreter) + params.drop(1)
             mainClass.set("org.utbot.python.newtyping.samples.GenerateMypyInfoBuildKt")
         }
     }

utbot-python-types/src/main/kotlin/org/utbot/python/newtyping/mypy/MypyAnnotations.kt

@@ -1,8 +1,8 @@
 package org.utbot.python.newtyping.mypy
 
-import com.squareup.moshi.adapters.PolymorphicJsonAdapterFactory
 import org.utbot.python.newtyping.*
 import org.utbot.python.newtyping.general.*
+import org.utbot.python.utils.CustomPolymorphicJsonAdapterFactory
 
 class MypyAnnotation(
     val nodeId: String,
@@ -11,14 +11,24 @@ class MypyAnnotation(
     var initialized = false
     @Transient lateinit var storage: MypyInfoBuild
     val node: MypyAnnotationNode
-        get() = storage.nodeStorage[nodeId]!!
+        get() {
+            val result = storage.nodeStorage[nodeId]
+            require(result != null) {
+                "Required node is absent in storage: $nodeId"
+            }
+            return result
+        }
     val asUtBotType: UtType
         get() {
-            assert(initialized)
+            require(initialized)
             val origin = storage.getUtBotTypeOfNode(node)
+            if (origin.pythonDescription() is PythonAnyTypeDescription)
+                return origin
             if (args != null) {
-                assert(origin.parameters.size == args.size)
-                assert(origin.parameters.all { it is TypeParameter })
+                require(origin.parameters.size == args.size) {
+                    "Bad arguments on ${origin.pythonTypeRepresentation()}. Expected ${origin.parameters.size} parameters but got ${args.size}"
+                }
+                require(origin.parameters.all { it is TypeParameter })
                 val argTypes = args.map { it.asUtBotType }
                 return DefaultSubstitutionProvider.substitute(
                     origin,
@@ -47,7 +57,10 @@ sealed class CompositeAnnotationNode(
     fun getInitData(self: CompositeTypeCreator.Original): CompositeTypeCreator.InitializationData {
         storage.nodeToUtBotType[this] = self
         (typeVars zip self.parameters).forEach { (node, typeParam) ->
-            val typeVar = node.node as TypeVarNode
+            val typeVar = node.node as? TypeVarNode
+            require(typeVar != null) {
+                "Did not construct type variable"
+            }
             storage.nodeToUtBotType[typeVar] = typeParam
             typeParam.meta = PythonTypeVarDescription(Name(emptyList(), typeVar.varName), typeVar.variance, typeVar.kind)
             typeParam.constraints = typeVar.constraints
@@ -70,7 +83,7 @@ class ConcreteAnnotation(
     val isAbstract: Boolean
 ): CompositeAnnotationNode(module, simpleName, members, typeVars, bases) {
     override fun initializeType(): UtType {
-        assert(storage.nodeToUtBotType[this] == null)
+        require(storage.nodeToUtBotType[this] == null)
         return createPythonConcreteCompositeType(
             Name(module.split('.'), simpleName),
             typeVars.size,
@@ -117,7 +130,10 @@ class FunctionNode(
         ) { self ->
             storage.nodeToUtBotType[this] = self
             (typeVars zip self.parameters).forEach { (nodeId, typeParam) ->
-                val typeVar = storage.nodeStorage[nodeId] as TypeVarNode
+                val typeVar = storage.nodeStorage[nodeId] as? TypeVarNode
+                require(typeVar != null) {
+                    "Did not construct type variable"
+                }
                 storage.nodeToUtBotType[typeVar] = typeParam
                 typeParam.meta = PythonTypeVarDescription(Name(emptyList(), typeVar.varName), typeVar.variance, typeVar.kind)
                 typeParam.constraints = typeVar.constraints
@@ -140,9 +156,16 @@ class TypeVarNode(
 ): MypyAnnotationNode() {
     override val children: List<MypyAnnotation>
         get() = super.children + values + (upperBound?.let { listOf(it) } ?: emptyList())
-    override fun initializeType() =
-        error("Initialization of TypeVar must be done in defining class or function." +
-                " TypeVar name: $varName, def_id: $def")
+    override fun initializeType(): UtType {
+        /*error(
+            "Initialization of TypeVar must be done in defining class or function." +
+                    " TypeVar name: $varName, def_id: $def"
+        )*/
+        // this a rare and bad case:
+        // https://github.com/sqlalchemy/sqlalchemy/blob/rel_2_0_20/lib/sqlalchemy/sql/sqltypes.py#L2091C5-L2091C23
+        storage.nodeStorage[def]!!.initializeType()
+        return storage.nodeToUtBotType[this] ?: error("Error while initializing TypeVar name: $varName, def_id: $def")
+    }
     val constraints: Set<TypeParameterConstraint> by lazy {
         val upperBoundConstraint: Set<TypeParameterConstraint> =
             upperBound?.let { setOf(TypeParameterConstraint(upperBoundRelation, it.asUtBotType)) } ?: emptySet()
@@ -236,28 +259,22 @@ enum class AnnotationType {
     Unknown
 }
 
-val annotationAdapter: PolymorphicJsonAdapterFactory<MypyAnnotationNode> =
-    PolymorphicJsonAdapterFactory.of(MypyAnnotationNode::class.java, "type")
-        .withSubtype(ConcreteAnnotation::class.java, AnnotationType.Concrete.name)
-        .withSubtype(Protocol::class.java, AnnotationType.Protocol.name)
-        .withSubtype(TypeVarNode::class.java, AnnotationType.TypeVar.name)
-        .withSubtype(OverloadedFunction::class.java, AnnotationType.Overloaded.name)
-        .withSubtype(FunctionNode::class.java, AnnotationType.Function.name)
-        .withSubtype(PythonAny::class.java, AnnotationType.Any.name)
-        //.withSubtype(PythonLiteral::class.java, AnnotationType.Literal.name)
-        .withSubtype(PythonUnion::class.java, AnnotationType.Union.name)
-        .withSubtype(PythonTuple::class.java, AnnotationType.Tuple.name)
-        .withSubtype(PythonNoneType::class.java, AnnotationType.NoneType.name)
-        .withSubtype(TypeAliasNode::class.java, AnnotationType.TypeAlias.name)
-        .withSubtype(UnknownAnnotationNode::class.java, AnnotationType.Unknown.name)
-
-object MypyAnnotations {
-
-    data class MypyReportLine(
-        val line: Int,
-        val type: String,
-        val message: String,
-        val file: String
+val annotationAdapter = CustomPolymorphicJsonAdapterFactory(
+    MypyAnnotationNode::class.java,
+    contentLabel = "content",
+    keyLabel = "type",
+    mapOf(
+        AnnotationType.Concrete.name to ConcreteAnnotation::class.java,
+        AnnotationType.Protocol.name to Protocol::class.java,
+        AnnotationType.TypeVar.name to TypeVarNode::class.java,
+        AnnotationType.Overloaded.name to OverloadedFunction::class.java,
+        AnnotationType.Function.name to FunctionNode::class.java,
+        AnnotationType.Any.name to PythonAny::class.java,
+        // .withSubtype(PythonLiteral::class.java, AnnotationType.Literal.name)
+        AnnotationType.Union.name to PythonUnion::class.java,
+        AnnotationType.Tuple.name to PythonTuple::class.java,
+        AnnotationType.NoneType.name to PythonNoneType::class.java,
+        AnnotationType.TypeAlias.name to TypeAliasNode::class.java,
+        AnnotationType.Unknown.name to UnknownAnnotationNode::class.java
     )
-
-}
+)

utbot-python-types/src/main/kotlin/org/utbot/python/newtyping/mypy/MypyBuild.kt

@@ -33,7 +33,7 @@ class ExpressionTypeFromMypy(
 class MypyInfoBuild(
     val nodeStorage: Map<String, MypyAnnotationNode>,
     val definitions: Map<String, Map<String, MypyDefinition>>,
-    val types: Map<String, List<ExpressionTypeFromMypy>>,
+    val exprTypes: Map<String, List<ExpressionTypeFromMypy>>,
     val names: Map<String, List<Name>>
 ) {
     @Transient lateinit var buildRoot: MypyBuildDirectory
@@ -77,7 +77,7 @@ class MypyInfoBuild(
                 fillArgNames(it.value)
             }
         }
-        types.values.flatten().forEach {
+        exprTypes.values.flatten().forEach {
             initAnnotation(it.type)
         }
         nodeStorage.values.forEach { node ->

utbot-python-types/src/main/kotlin/org/utbot/python/newtyping/mypy/MypyDefinitions.kt

@@ -1,9 +1,9 @@
 package org.utbot.python.newtyping.mypy
 
-import com.squareup.moshi.adapters.PolymorphicJsonAdapterFactory
 import org.utbot.python.newtyping.*
 import org.utbot.python.newtyping.general.FunctionType
 import org.utbot.python.newtyping.general.UtType
+import org.utbot.python.utils.CustomPolymorphicJsonAdapterFactory
 
 sealed class MypyDefinition(val type: MypyAnnotation) {
     fun getUtBotType(): UtType = type.asUtBotType
@@ -69,9 +69,14 @@ enum class MypyDefinitionLabel {
     OverloadedFuncDef
 }
 
-val definitionAdapter: PolymorphicJsonAdapterFactory<MypyDefinition> =
-    PolymorphicJsonAdapterFactory.of(MypyDefinition::class.java, "kind")
-        .withSubtype(Variable::class.java, MypyDefinitionLabel.Variable.name)
-        .withSubtype(ClassDef::class.java, MypyDefinitionLabel.ClassDef.name)
-        .withSubtype(FuncDef::class.java, MypyDefinitionLabel.FuncDef.name)
-        .withSubtype(OverloadedFuncDef::class.java, MypyDefinitionLabel.OverloadedFuncDef.name)
+val definitionAdapter = CustomPolymorphicJsonAdapterFactory(
+    MypyDefinition::class.java,
+    contentLabel = "content",
+    keyLabel = "kind",
+    mapOf(
+        MypyDefinitionLabel.Variable.name to Variable::class.java,
+        MypyDefinitionLabel.ClassDef.name to ClassDef::class.java,
+        MypyDefinitionLabel.FuncDef.name to FuncDef::class.java,
+        MypyDefinitionLabel.OverloadedFuncDef.name to OverloadedFuncDef::class.java
+    )
+)

utbot-python-types/src/main/kotlin/org/utbot/python/newtyping/mypy/RunMypy.kt

@@ -76,9 +76,10 @@ fun buildMypyInfo(
     val mypyExitStatus = if (mypyBuildDir.fileForMypyExitStatus.exists()) mypyBuildDir.fileForMypyExitStatus.readText() else null
     if (result.exitValue != 0 || mypyExitStatus != "0")
         error("Something went wrong in initial mypy run. " +
-                "\nPython stderr ${result.stderr}" +
-                "\nMypy stderr: $stderr" +
-                "\nMypy stdout: $stdout")
+                "\nPython stdout:\n${result.stdout}" +
+                "\nPython stderr:\n${result.stderr}" +
+                "\nMypy stderr:\n$stderr" +
+                "\nMypy stdout:\n$stdout")
 }
 
 fun readMypyAnnotationStorageAndInitialErrors(

utbot-python-types/src/main/kotlin/org/utbot/python/utils/CustomJsonAdapter.kt

@@ -0,0 +1,57 @@
+package org.utbot.python.utils
+
+import com.squareup.moshi.*
+import com.squareup.moshi.JsonAdapter.Factory
+import java.lang.reflect.Type
+
+class CustomPolymorphicJsonAdapterFactory(
+    private val baseType: Type,
+    contentLabel: String,
+    keyLabel: String,
+    private val elementAdapters: Map<String, Type>
+): Factory {
+    private val contentOption = JsonReader.Options.of(contentLabel)
+    private val keyLabelOption = JsonReader.Options.of(keyLabel)
+    private val labels = elementAdapters.keys.toList()
+    private val labelOptions = JsonReader.Options.of(*labels.toTypedArray<String>())
+
+    class CustomPolymorphicJsonAdapter(
+        private val contentOption: JsonReader.Options,
+        private val keyLabelOption: JsonReader.Options,
+        private val adapters: List<JsonAdapter<Any>>,
+        private val labelOptions: JsonReader.Options
+    ): JsonAdapter<Any>() {
+        override fun fromJson(reader: JsonReader): Any? {
+            reader.beginObject()
+
+            val index = reader.selectName(keyLabelOption)
+            if (index == -1)
+                return null
+
+            val labelIndex = reader.selectString(labelOptions)
+            if (labelIndex == -1)
+                return null
+
+            val contentIndex = reader.selectName(contentOption)
+            if (contentIndex == -1)
+                return null
+
+            val result = adapters[labelIndex].fromJson(reader)
+            reader.endObject()
+
+            return result
+        }
+
+        override fun toJson(writer: JsonWriter, value: Any?) {
+            error("Writing with this Json adapter is not supported")
+        }
+    }
+
+    override fun create(type: Type, annotations: MutableSet<out Annotation>, moshi: Moshi): JsonAdapter<*>? {
+        if (Types.getRawType(type) != baseType || annotations.isNotEmpty()) {
+            return null
+        }
+        val adapters: List<JsonAdapter<Any>> = labels.map { moshi.adapter(elementAdapters[it]!!) }
+        return CustomPolymorphicJsonAdapter(contentOption, keyLabelOption, adapters, labelOptions)
+    }
+}

utbot-python-types/src/main/python/utbot_mypy_runner/.gitignore

@@ -0,0 +1,3 @@
+mypy_config.ini
+out.json
+.vscode/

utbot-python-types/src/main/python/utbot_mypy_runner/pyproject.toml

@@ -1,14 +1,14 @@
 [tool.poetry]
 name = "utbot_mypy_runner"
-version = "0.2.11"
+version = "0.2.15"
 description = ""
 authors = ["Ekaterina Tochilina <katerina_t_n@mail.ru>"]
 readme = "README.md"
 packages = [{include = "utbot_mypy_runner"}]
 
 [tool.poetry.dependencies]
 python = "^3.8"
-mypy = "1.0.0"
+mypy = "1.5.1"
 
 
 [build-system]

utbot-python-types/src/main/python/utbot_mypy_runner/utbot_mypy_runner/__main__.py

@@ -51,4 +51,4 @@
         print("Extracted annotations and wrote to", args.annotations_out)
     else:
         print("For some reason BuildResult is None")
-        exit(11)
+        exit(11)