Generics in Kotlin codegen (#88)

Refactored imports to support kotlin builtins (List, Map, e.t.c).
Added generics propagation using callables.
Added filling generics with Any? and *.
Reworked Kotlin type render.
Fixed assemble model generation in concrete.
Fixed bug with backticks.

Author: nikitavlaev

utbot-core/src/main/kotlin/org/utbot/common/HackUtil.kt

@@ -67,4 +67,9 @@ enum class WorkaroundReason {
      * requires thorough [investigation](https://github.com/UnitTestBot/UTBotJava/issues/716).
      */
     IGNORE_STATICS_FROM_TRUSTED_LIBRARIES,
+
+    /**
+     * Special handling of collection constructors from other collections (for generics processing)
+     */
+    COLLECTION_CONSTRUCTOR_FROM_COLLECTION,
 }

utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt

@@ -8,8 +8,11 @@
 
 package org.utbot.framework.plugin.api
 
+import org.utbot.common.WorkaroundReason
 import org.utbot.common.isDefaultValue
+import org.utbot.common.unreachableBranch
 import org.utbot.common.withToStringThreadLocalReentrancyGuard
+import org.utbot.common.workaround
 import org.utbot.framework.UtSettings
 import org.utbot.framework.plugin.api.MockFramework.MOCKITO
 import org.utbot.framework.plugin.api.impl.FieldIdReflectionStrategy
@@ -19,15 +22,18 @@ import org.utbot.framework.plugin.api.util.byteClassId
 import org.utbot.framework.plugin.api.util.charClassId
 import org.utbot.framework.plugin.api.util.constructor
 import org.utbot.framework.plugin.api.util.doubleClassId
+import org.utbot.framework.plugin.api.util.executable
 import org.utbot.framework.plugin.api.util.executableId
 import org.utbot.framework.plugin.api.util.floatClassId
 import org.utbot.framework.plugin.api.util.id
 import org.utbot.framework.plugin.api.util.intClassId
 import org.utbot.framework.plugin.api.util.isArray
 import org.utbot.framework.plugin.api.util.isPrimitive
+import org.utbot.framework.plugin.api.util.isSubtypeOf
 import org.utbot.framework.plugin.api.util.jClass
 import org.utbot.framework.plugin.api.util.longClassId
 import org.utbot.framework.plugin.api.util.method
+import org.utbot.framework.plugin.api.util.objectClassId
 import org.utbot.framework.plugin.api.util.primitiveTypeJvmNameOrNull
 import org.utbot.framework.plugin.api.util.safeJField
 import org.utbot.framework.plugin.api.util.shortClassId
@@ -48,17 +54,29 @@ import soot.Type
 import soot.VoidType
 import soot.jimple.JimpleBody
 import soot.jimple.Stmt
+import sun.reflect.generics.parser.SignatureParser
+import sun.reflect.generics.tree.ArrayTypeSignature
+import sun.reflect.generics.tree.ClassTypeSignature
+import sun.reflect.generics.tree.SimpleClassTypeSignature
+import sun.reflect.generics.tree.TypeArgument
+import sun.reflect.generics.tree.TypeTree
+import sun.reflect.generics.tree.TypeVariableSignature
 import java.io.File
+import java.lang.reflect.Constructor
+import java.lang.reflect.GenericSignatureFormatError
+import java.lang.reflect.Method
 import java.lang.reflect.Modifier
 import kotlin.contracts.ExperimentalContracts
 import kotlin.contracts.contract
 import kotlin.jvm.internal.CallableReference
 import kotlin.reflect.KCallable
 import kotlin.reflect.KClass
 import kotlin.reflect.KFunction
+import kotlin.reflect.KType
 import kotlin.reflect.full.instanceParameter
 import kotlin.reflect.jvm.javaConstructor
 import kotlin.reflect.jvm.javaType
+import kotlin.reflect.jvm.kotlinFunction
 
 const val SYMBOLIC_NULL_ADDR: Int = 0
 
@@ -264,7 +282,11 @@ data class UtError(
  */
 sealed class UtModel(
     open val classId: ClassId
-)
+) {
+    open fun processGenerics(type: KType) {}
+
+    open fun fillGenericsAsObjects() {}
+}
 
 /**
  * Class representing models for values that might have an address.
@@ -402,6 +424,10 @@ data class UtCompositeModel(
     val fields: MutableMap<FieldId, UtModel> = mutableMapOf(),
     val mocks: MutableMap<ExecutableId, List<UtModel>> = mutableMapOf(),
 ) : UtReferenceModel(id, classId) {
+    override fun processGenerics(type: KType) {
+       classId.typeParameters.fromType(type)
+    }
+
     //TODO: SAT-891 - rewrite toString() method
     override fun toString() = withToStringThreadLocalReentrancyGuard {
         buildString {
@@ -505,6 +531,150 @@ data class UtAssembleModel(
     val finalInstantiationModel
         get() = instantiationChain.lastOrNull()
 
+    private fun TypeTree.cmp(other: TypeTree): Boolean {
+        if (this::class != other::class) return false
+
+        when (this) {
+            is TypeVariableSignature -> return identifier == (other as TypeVariableSignature).identifier
+            is ClassTypeSignature -> {
+                val otherPath = (other as ClassTypeSignature).path
+                return path.foldIndexed(true) { i, prev, it ->
+                       prev && (otherPath.getOrNull(i)?.cmp(it) ?: false)
+                }
+            }
+            is SimpleClassTypeSignature -> {
+                val otherTypeArgs = (other as SimpleClassTypeSignature).typeArguments
+                return typeArguments.foldIndexed(true) { i, prev, it ->
+                    prev && (otherTypeArgs.getOrNull(i)?.cmp(it) ?: false)
+                }
+            }
+            is ArrayTypeSignature -> return componentType.cmp((other as ArrayTypeSignature).componentType)
+            // other cases are trivial and handled by class comparison
+            else -> return true
+        }
+    }
+
+    override fun processGenerics(type: KType) {
+        classId.typeParameters.fromType(type)
+
+        // TODO might cause problems with type params when program synthesis comes
+        // assume that last statement is constructor call
+        instantiationChain.lastOrNull()?.let inst@ { lastStatement ->
+            (lastStatement as? UtExecutableCallModel)?.let {
+                when (val executable = it.executable) {
+                    is ConstructorId -> executable.typeParameters.copyFromClassId(classId)
+                    is MethodId -> executable.typeParameters.copyFromClassId(classId)
+                }
+
+                try {
+                    val function = when (val executable = it.executable.executable) {
+                        is Constructor<*> -> executable.kotlinFunction
+                        is Method -> executable.kotlinFunction
+                        else -> unreachableBranch("this executable does not exist $executable")
+                    }
+
+                    it.params.mapIndexed { i, param ->
+                        function?.parameters?.getOrNull(i)?.type?.let { it1 -> param.processGenerics(it1) }
+                    }
+                } catch (e: Error) {
+                    // KotlinReflectionInternalError can't be imported, but it is assumed here
+                    // it can be thrown here because, i.e., Int(Int) constructor does not exist in Kotlin
+                }
+
+                workaround(WorkaroundReason.COLLECTION_CONSTRUCTOR_FROM_COLLECTION) {
+                    val propagateFromReturnTypeToParameter = { id: Int ->
+                        ((it.params[id] as? UtAssembleModel)?.instantiationChain?.get(0) as? UtExecutableCallModel)
+                            ?.executable?.typeParameters?.copyFromClassId(classId)
+                    }
+
+                    when (val executable = it.executable.executable) {
+                        is Constructor<*> -> {
+                            // Can't parse signature here, since constructors return void
+                            // This part only works for cases like Collection<T>(collection: Collection<T>)
+                            if (it.executable is ConstructorId) {
+                                if (it.executable.classId.isSubtypeOf(Collection::class.id)) {
+                                    if (it.executable.parameters.size == 1 && it.executable.parameters[0].isSubtypeOf(Collection::class.id)) {
+                                        propagateFromReturnTypeToParameter(0)
+                                    }
+                                }
+                            }
+                        }
+                        is Method -> {
+                            try {
+                                val f = Method::class.java.getDeclaredField("signature")
+                                f.isAccessible = true
+                                val signature = f.get(executable) as? String ?: return@inst
+                                val parsedSignature = SignatureParser.make().parseMethodSig(signature)
+
+                                // check if parameter types are equal to return types
+                                // e.g. <T:Ljava/lang/Object;>(Ljava/util/List<TT;>;)Ljava/util/List<TT;>;
+                                parsedSignature.parameterTypes.forEachIndexed { paramId, param ->
+                                    if (param::class != parsedSignature.returnType::class) return@forEachIndexed
+
+                                    param as? TypeArgument ?: error("Only TypeArgument is expected")
+                                    parsedSignature as? TypeArgument ?: error("Only TypeArgument is expected")
+                                    if (param.cmp(parsedSignature)) {
+                                        propagateFromReturnTypeToParameter(paramId)
+                                    }
+                                }
+                            } catch (e: GenericSignatureFormatError) {
+                                // TODO log
+                            }
+                        }
+                        else -> unreachableBranch("this executable does not exist $executable")
+                    }
+                }
+            }
+        }
+
+        for (model in modificationsChain) {
+            if (model is UtExecutableCallModel) {
+                model.params.mapIndexed { i, param ->
+                    type.arguments.getOrNull(i)?.type?.let { it1 -> param.processGenerics(it1) }
+                }
+            }
+        }
+    }
+
+    override fun fillGenericsAsObjects() {
+        // TODO might cause problems with type params when program synthesis comes
+        // assume that last statement is constructor call
+        instantiationChain.lastOrNull()?.let { lastStatement ->
+            (lastStatement as? UtExecutableCallModel)?.let {
+                try {
+                    val function = when (val executable = it.executable.executable) {
+                        is Constructor<*> -> executable.kotlinFunction
+                        is Method -> executable.kotlinFunction
+                        else -> unreachableBranch("this executable does not exist $executable")
+                    }
+                    function?.let { f ->
+                        classId.typeParameters.parameters = List(f.typeParameters.size) { objectClassId }
+                    }
+
+                    it.params.map { param -> param.fillGenericsAsObjects() }
+                } catch (e: Error) {
+                    // KotlinReflectionInternalError can't be imported, but it is assumed here
+                    // it can be thrown here because, i.e., Int(Int) constructor does not exist in Kotlin
+                }
+            }
+        }
+
+        for (model in modificationsChain) {
+            if (model is UtExecutableCallModel) {
+                val function = when (val executable = model.executable.executable) {
+                    is Constructor<*> -> executable.kotlinFunction
+                    is Method -> executable.kotlinFunction
+                    else -> unreachableBranch("this executable does not exist $executable")
+                }
+                function?.let { f ->
+                    model.executable.classId.typeParameters.parameters = List(f.typeParameters.size) { objectClassId }
+                }
+
+                model.params.map { it.fillGenericsAsObjects() }
+            }
+        }
+    }
+
     override fun toString() = withToStringThreadLocalReentrancyGuard {
         buildString {
             append("UtAssembleModel(${classId.simpleName} $modelName) ")
@@ -760,8 +930,7 @@ open class ClassId @JvmOverloads constructor(
     open val allConstructors: Sequence<ConstructorId>
         get() = jClass.declaredConstructors.asSequence().map { it.executableId }
 
-    open val typeParameters: TypeParameters
-        get() = TypeParameters()
+    open val typeParameters: TypeParameters = TypeParameters()
 
     open val outerClass: Class<*>?
         get() = jClass.enclosingClass
@@ -904,6 +1073,19 @@ open class FieldId(val declaringClass: ClassId, val name: String) {
     open val type: ClassId
         get() = strategy.type
 
+    // required to store and update type parameters
+    // TODO check if by lazy works correctly in newer Kotlin (https://stackoverflow.com/questions/47638464/kotlin-lazy-var-throwing-classcastexception-kotlin-uninitialized-value)
+    // val fixedType: ClassId by lazy { type }
+    private var hiddenFixedType: ClassId? = null
+
+    val fixedType: ClassId
+        get() {
+            if (hiddenFixedType == null) {
+                hiddenFixedType = type
+            }
+            return hiddenFixedType!!
+        }
+
     override fun equals(other: Any?): Boolean {
         if (this === other) return true
         if (javaClass != other?.javaClass) return false
@@ -979,6 +1161,8 @@ sealed class ExecutableId : StatementId() {
     abstract val isProtected: Boolean
     abstract val isPrivate: Boolean
 
+    val typeParameters: TypeParameters = TypeParameters()
+
     val isPackagePrivate: Boolean
         get() = !(isPublic || isProtected || isPrivate)
 
@@ -1064,9 +1248,31 @@ class BuiltinMethodId(
     override val isPrivate: Boolean = false
 ) : MethodId(classId, name, returnType, parameters)
 
-open class TypeParameters(val parameters: List<ClassId> = emptyList())
+open class TypeParameters(var parameters: List<ClassId> = emptyList()) {
+    fun copyFromClassId(classId: ClassId) {
+        parameters = classId.typeParameters.parameters
+    }
+
+    fun fromType(type: KType) {
+        if (type.arguments.isEmpty()) return
+
+        parameters = type.arguments.map {
+            when (val clazz = it.type?.classifier) {
+                is KClass<*> -> {
+                    val classId = clazz.id
+                    it.type?.let { t ->
+                        classId.typeParameters.fromType(t)
+                    } ?: error("")
+
+                    classId
+                }
+                else -> objectClassId
+            }
+        }
+    }
+}
 
-class WildcardTypeParameter : TypeParameters(emptyList())
+class WildcardTypeParameter: ClassId("org.utbot.framework.plugin.api.WildcardTypeParameter")
 
 interface CodeGenerationSettingItem {
     val displayName: String

utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt

@@ -146,7 +146,8 @@ private fun pathSelector(graph: InterProceduralUnitGraph, typeRegistry: TypeRegi
 
 class UtBotSymbolicEngine(
     private val controller: EngineController,
-    private val methodUnderTest: UtMethod<*>,
+    /** methodUnderTest is internal to use in [org.utbot.framework.plugin.api.TestFlow.processGenerics]. **/
+    internal val methodUnderTest: UtMethod<*>,
     classpath: String,
     dependencyPaths: String,
     mockStrategy: MockStrategy = NO_MOCKS,

utbot-framework/src/main/kotlin/org/utbot/framework/codegen/Domain.kt

@@ -37,19 +37,26 @@ data class TestClassFile(val packageName: String, val imports: List<Import>, val
 
 sealed class Import(internal val order: Int) : Comparable<Import> {
     abstract val qualifiedName: String
+    abstract val classId: ClassId
 
     override fun compareTo(other: Import) = importComparator.compare(this, other)
 }
 
 private val importComparator = compareBy<Import> { it.order }.thenBy { it.qualifiedName }
 
-data class StaticImport(val qualifierClass: String, val memberName: String) : Import(1) {
-    override val qualifiedName: String = "$qualifierClass.$memberName"
+data class StaticImport(val methodId: MethodId) : Import(1) {
+    override val qualifiedName: String = "${methodId.classId.canonicalName}.${methodId.name}"
+    override val classId: ClassId
+        get() = methodId.classId
 }
 
-data class RegularImport(val packageName: String, val className: String) : Import(2) {
-    override val qualifiedName: String
-        get() = if (packageName.isNotEmpty()) "$packageName.$className" else className
+data class RegularImport(override val classId: ClassId) : Import(2) {
+    override val qualifiedName: String =
+        if (classId.packageName.isNotEmpty()) {
+            "${classId.packageName}.${classId.simpleNameWithEnclosings}"
+        } else {
+            classId.simpleNameWithEnclosings
+        }
 
     // TODO: check without equals() and hashCode()
     override fun equals(other: Any?): Boolean {
@@ -58,15 +65,14 @@ data class RegularImport(val packageName: String, val className: String) : Impor
 
         other as RegularImport
 
-        if (packageName != other.packageName) return false
-        if (className != other.className) return false
+        if (classId != other.classId) return false
 
         return true
     }
 
     override fun hashCode(): Int {
-        var result = packageName.hashCode()
-        result = 31 * result + className.hashCode()
+        var result = classId.hashCode()
+        result = 31 * result + qualifiedName.hashCode()
         return result
     }
 }

utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/name/CgNameGenerator.kt

@@ -148,8 +148,9 @@ internal class CgNameGeneratorImpl(private val context: CgContext)
     private fun createNameFromKeyword(baseName: String): String = when(codegenLanguage) {
         CodegenLanguage.JAVA -> nextIndexedVarName(baseName)
         CodegenLanguage.KOTLIN -> {
+            val backticksBaseName = "`$baseName`"
             // use backticks for first variable with keyword name and use indexed names for all next such variables
-            if (baseName !in existingVariableNames) "`$baseName`" else nextIndexedVarName(baseName)
+            if (backticksBaseName !in existingVariableNames) backticksBaseName else nextIndexedVarName(baseName)
         }
     }