Some fixes in Python fuzzer logic

utbot-python/samples/easy_samples/field.py

@@ -0,0 +1,10 @@
+class NoTestsProblem:
+    def __init__(self):
+        self.board = []
+
+    def set_position(self, row, col, symbol):
+        self.board[row][col] = symbol
+        return symbol
+
+    def start(self):
+        self.set_position(1, 2, "O")

utbot-python/samples/easy_samples/general.py

@@ -2,7 +2,7 @@
 import heapq
 import typing
 from socket import socket
-from typing import List, Dict, Set, Optional
+from typing import List, Dict, Set, Optional, AbstractSet
 from dataclasses import dataclass
 import logging
 import datetime
@@ -94,7 +94,7 @@ def empty():
     return 1
 
 
-def id_(x):
+def id_(x: AbstractSet[int]):
     return x
 
 

utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt

@@ -11,6 +11,7 @@ import org.utbot.fuzzing.utils.Trie
 import org.utbot.python.evaluation.*
 import org.utbot.python.evaluation.serialiation.MemoryDump
 import org.utbot.python.evaluation.serialiation.toPythonTree
+import org.utbot.python.framework.api.python.PythonTree
 import org.utbot.python.framework.api.python.PythonTreeModel
 import org.utbot.python.framework.api.python.PythonTreeWrapper
 import org.utbot.python.fuzzing.*
@@ -259,6 +260,12 @@ class PythonEngine(
                             return@PythonFuzzing PythonFeedback(control = Control.STOP)
                         }
 
+                        if (arguments.any { PythonTree.containsFakeNode(it.tree) }) {
+                            logger.debug("FakeNode in Python model")
+                            emit(InvalidExecution(UtError("Bad input object", Throwable())))
+                            return@PythonFuzzing PythonFeedback(control = Control.CONTINUE)
+                        }
+
                         val pair = Pair(description, arguments.map { PythonTreeWrapper(it.tree) })
                         val mem = cache.get(pair)
                         if (mem != null) {
@@ -278,7 +285,7 @@ class PythonEngine(
                         return@PythonFuzzing result.fuzzingPlatformFeedback
                     }.fuzz(pmd)
                 } catch (_: Exception) { // e.g. NoSeedValueException
-                    logger.info { "Cannot fuzz values for types: $parameters" }
+                    logger.info { "Cannot fuzz values for types: ${parameters.map { it.pythonTypeRepresentation() }}" }
                 }
             }
             manager.disconnect()

utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt

@@ -29,6 +29,7 @@ import org.utbot.python.utils.TimeoutMode
 import java.io.File
 
 private val logger = KotlinLogging.logger {}
+private const val RANDOM_TYPE_FREQUENCY = 6
 
 class PythonTestCaseGenerator(
     private val withMinimization: Boolean = true,
@@ -117,28 +118,28 @@ class PythonTestCaseGenerator(
         }.take(maxSubstitutions)
     }
 
-    fun generate(method: PythonMethod, until: Long): PythonTestSet {
-        storageForMypyMessages.clear()
+    private fun methodHandler(
+        method: PythonMethod,
+        typeStorage: PythonTypeStorage,
+        coveredLines: MutableSet<Int>,
+        errors: MutableList<UtError>,
+        executions: MutableList<UtExecution>,
+        initMissingLines: Set<Int>?,
+        until: Long,
+        additionalVars: String = ""
+    ): Set<Int>? {  // returns missing lines
         val limitManager = TestGenerationLimitManager(
             ExecutionWithTimeoutMode,
             until,
         )
-
-        val typeStorage = PythonTypeStorage.get(mypyStorage)
+        var missingLines = initMissingLines
 
         val (hintCollector, constantCollector) = constructCollectors(mypyStorage, typeStorage, method)
         val constants = constantCollector.result.map { (type, value) ->
             logger.debug("Collected constant: ${type.pythonTypeRepresentation()}: $value")
             PythonFuzzedConcreteValue(type, value)
         }
 
-        val executions = mutableListOf<UtExecution>()
-        val errors = mutableListOf<UtError>()
-        var missingLines: Set<Int>? = null
-        val coveredLines = mutableSetOf<Int>()
-        var generated = 0
-
-        logger.info("Start test generation for ${method.name}")
         substituteTypeParameters(method, typeStorage).forEach { newMethod ->
             inferAnnotations(
                 newMethod,
@@ -148,6 +149,7 @@ class PythonTestCaseGenerator(
                 mypyReportLine,
                 mypyConfigFile,
                 limitManager,
+                additionalVars
             ) { functionType ->
                 val args = (functionType as FunctionType).arguments
 
@@ -168,7 +170,6 @@ class PythonTestCaseGenerator(
                 val fuzzerCancellation = { isCancelled() || limitManager.isCancelled() }
 
                 engine.fuzzing(args, fuzzerCancellation, until).collect {
-                    generated += 1
                     when (it) {
                         is ValidExecution -> {
                             executions += it.utFuzzedExecution
@@ -196,6 +197,42 @@ class PythonTestCaseGenerator(
                 feedback
             }
         }
+        return missingLines
+    }
+
+    fun generate(method: PythonMethod, until: Long): PythonTestSet {
+        storageForMypyMessages.clear()
+
+        val typeStorage = PythonTypeStorage.get(mypyStorage)
+
+        val executions = mutableListOf<UtExecution>()
+        val errors = mutableListOf<UtError>()
+        val coveredLines = mutableSetOf<Int>()
+
+        logger.info("Start test generation for ${method.name}")
+        val meta = method.definition.type.pythonDescription() as PythonCallableTypeDescription
+        val argKinds = meta.argumentKinds
+        if (argKinds.any { it != PythonCallableTypeDescription.ArgKind.ARG_POS }) {
+            val now = System.currentTimeMillis()
+            val firstUntil = (until - now) / 2 + now
+            val originalDef = method.definition
+            val shortType = meta.removeNonPositionalArgs(originalDef.type)
+            val shortMeta = PythonFuncItemDescription(
+                originalDef.meta.name,
+                originalDef.meta.args.take(shortType.arguments.size)
+            )
+            val additionalVars = originalDef.meta.args
+                .drop(shortType.arguments.size)
+                .joinToString(separator="\n", prefix="\n") { arg ->
+                    "${arg.name}: ${pythonAnyType.pythonTypeRepresentation()}"  // TODO: better types
+                }
+            method.definition = PythonFunctionDefinition(shortMeta, shortType)
+            val missingLines = methodHandler(method, typeStorage, coveredLines, errors, executions, null, firstUntil, additionalVars)
+            method.definition = originalDef
+            methodHandler(method, typeStorage, coveredLines, errors, executions, missingLines, until)
+        } else {
+            methodHandler(method, typeStorage, coveredLines, errors, executions, null, until)
+        }
 
         logger.info("Collect all test executions for ${method.name}")
         val (successfulExecutions, failedExecutions) = executions.partition { it.result is UtExecutionSuccess }
@@ -234,6 +271,7 @@ class PythonTestCaseGenerator(
         report: List<MypyReportLine>,
         mypyConfigFile: File,
         limitManager: TestGenerationLimitManager,
+        additionalVars: String,
         annotationHandler: suspend (Type) -> InferredTypeFeedback,
     ) {
         val namesInModule = mypyStorage.names
@@ -257,7 +295,9 @@ class PythonTestCaseGenerator(
                 getOffsetLine(sourceFileContent, method.ast.beginOffset),
                 getOffsetLine(sourceFileContent, method.ast.endOffset)
             ),
-            mypyConfigFile
+            mypyConfigFile,
+            additionalVars,
+            randomTypeFrequency = RANDOM_TYPE_FREQUENCY
         )
 
         runBlocking breaking@{

utbot-python/src/main/kotlin/org/utbot/python/code/CodeGen.kt

@@ -45,7 +45,8 @@ object PythonCodeGenerator {
         methodAnnotations: Map<String, Type>,
         directoriesForSysPath: Set<String>,
         moduleToImport: String,
-        namesInModule: Collection<String>
+        namesInModule: Collection<String>,
+        additionalVars: String
     ): String {
         val context = UtContext(this::class.java.classLoader)
         withUtContext(context) {
@@ -60,7 +61,8 @@ object PythonCodeGenerator {
                 methodAnnotations,
                 directoriesForSysPath,
                 moduleToImport,
-                namesInModule
+                namesInModule,
+                additionalVars
             )
         }
     }

utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt

@@ -26,6 +26,19 @@ object PythonTree {
         return tree.children.any { isRecursiveObjectDFS(it, visited) }
     }
 
+    fun containsFakeNode(tree: PythonTreeNode): Boolean {
+        return containsFakeNodeDFS(tree, mutableSetOf())
+    }
+
+    private fun containsFakeNodeDFS(tree: PythonTreeNode, visited: MutableSet<PythonTreeNode>): Boolean {
+        if (visited.contains(tree))
+            return false
+        if (tree is FakeNode)
+            return true
+        visited.add(tree)
+        return tree.children.any { containsFakeNodeDFS(it, visited) }
+    }
+
     open class PythonTreeNode(
         val id: Long,
         val type: PythonClassId,
@@ -69,6 +82,8 @@ object PythonTree {
             1 + children.fold(0) { acc, child -> acc + child.diversity() }
     }
 
+    object FakeNode: PythonTreeNode(0L, PythonClassId(""))
+
     class PrimitiveNode(
         id: Long,
         type: PythonClassId,

utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/PythonCodeGenerator.kt

@@ -211,7 +211,8 @@ class PythonCodeGenerator(
         methodAnnotations: Map<String, Type>,
         directoriesForSysPath: Set<String>,
         moduleToImport: String,
-        namesInModule: Collection<String>
+        namesInModule: Collection<String>,
+        additionalVars: String
     ): String {
         val cgRendererContext = CgRendererContext.fromCgContext(context)
         val printer = CgPrinterImpl()
@@ -243,6 +244,8 @@ class PythonCodeGenerator(
         val mypyCheckCode = listOf(
             renderer.toString(),
             "",
+            additionalVars,
+            "",
             functionName,
         ) + method.codeAsString.split("\n").map { "    $it" }
         return mypyCheckCode.joinToString("\n")

utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt

@@ -16,11 +16,10 @@ import org.utbot.fuzzing.utils.Trie
 import org.utbot.python.framework.api.python.PythonTree
 import org.utbot.python.fuzzing.provider.*
 import org.utbot.python.fuzzing.provider.utils.isAny
-import org.utbot.python.newtyping.PythonProtocolDescription
-import org.utbot.python.newtyping.PythonSubtypeChecker
-import org.utbot.python.newtyping.PythonTypeStorage
+import org.utbot.python.fuzzing.provider.utils.isConcreteType
+import org.utbot.python.newtyping.*
+import org.utbot.python.newtyping.general.DefaultSubstitutionProvider
 import org.utbot.python.newtyping.general.Type
-import org.utbot.python.newtyping.pythonTypeRepresentation
 
 private val logger = KotlinLogging.logger {}
 
@@ -59,7 +58,7 @@ class PythonFuzzedValue(
     val summary: String? = null,
 )
 
-fun pythonDefaultValueProviders() = listOf(
+fun pythonDefaultValueProviders(typeStorage: PythonTypeStorage) = listOf(
     NoneValueProvider,
     BoolValueProvider,
     IntValueProvider,
@@ -76,36 +75,22 @@ fun pythonDefaultValueProviders() = listOf(
     BytearrayValueProvider,
     ReduceValueProvider,
     ConstantValueProvider,
-    TypeAliasValueProvider
+    TypeAliasValueProvider,
+    SubtypeValueProvider(typeStorage)
 )
 
 class PythonFuzzing(
     private val pythonTypeStorage: PythonTypeStorage,
     val execute: suspend (description: PythonMethodDescription, values: List<PythonFuzzedValue>) -> PythonFeedback,
 ) : Fuzzing<Type, PythonFuzzedValue, PythonMethodDescription, PythonFeedback> {
 
-    private fun generateDefault(description: PythonMethodDescription, type: Type): Sequence<Seed<Type, PythonFuzzedValue>> {
-        var providers = emptyList<Seed<Type, PythonFuzzedValue>>().asSequence()
-        pythonDefaultValueProviders().asSequence().forEach { provider ->
+    private fun generateDefault(description: PythonMethodDescription, type: Type)= sequence<Seed<Type, PythonFuzzedValue>> {
+        pythonDefaultValueProviders(pythonTypeStorage).asSequence().forEach { provider ->
             if (provider.accept(type)) {
                 logger.debug { "Provider ${provider.javaClass.simpleName} accepts type ${type.pythonTypeRepresentation()}" }
-                providers += provider.generate(description, type)
+                yieldAll(provider.generate(description, type))
             }
         }
-        return providers
-    }
-
-    private fun generateSubtype(description: PythonMethodDescription, type: Type): Sequence<Seed<Type, PythonFuzzedValue>> {
-        var providers = emptyList<Seed<Type, PythonFuzzedValue>>().asSequence()
-        if (type.meta is PythonProtocolDescription) {
-            val subtypes = pythonTypeStorage.allTypes.filter {
-                PythonSubtypeChecker.checkIfRightIsSubtypeOfLeft(type, it, pythonTypeStorage)
-            }
-            subtypes.forEach {
-                providers += generateDefault(description, it)
-            }
-        }
-        return providers
     }
 
     override fun generate(description: PythonMethodDescription, type: Type): Sequence<Seed<Type, PythonFuzzedValue>> {
@@ -115,7 +100,6 @@ class PythonFuzzing(
             logger.debug("Any does not have provider")
         } else {
             providers += generateDefault(description, type)
-            providers += generateSubtype(description, type)
         }
 
         return providers

utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ReduceValueProvider.kt

@@ -8,6 +8,7 @@ import org.utbot.python.framework.api.python.PythonTree
 import org.utbot.python.framework.api.python.util.toPythonRepr
 import org.utbot.python.fuzzing.PythonFuzzedValue
 import org.utbot.python.fuzzing.PythonMethodDescription
+import org.utbot.python.fuzzing.provider.utils.isConcreteType
 import org.utbot.python.newtyping.*
 import org.utbot.python.newtyping.general.FunctionType
 import org.utbot.python.newtyping.general.Type
@@ -30,7 +31,7 @@ object ReduceValueProvider : ValueProvider<Type, PythonFuzzedValue, PythonMethod
     override fun accept(type: Type): Boolean {
         val hasSupportedType =
             !unsupportedTypes.contains(type.pythonTypeName())
-        return hasSupportedType && type.meta is PythonConcreteCompositeTypeDescription // && (hasInit || hasNew)
+        return hasSupportedType && isConcreteType(type) // && (hasInit || hasNew)
     }
 
     override fun generate(description: PythonMethodDescription, type: Type) = sequence {
@@ -100,12 +101,7 @@ object ReduceValueProvider : ValueProvider<Type, PythonFuzzedValue, PythonMethod
                 )
             },
             modify = modifications,
-            empty = Routine.Empty {
-                PythonFuzzedValue(
-                    PythonTree.fromObject(),
-                    "%var% = ${type.pythonTypeRepresentation()}"
-                )
-            }
+            empty = Routine.Empty { PythonFuzzedValue(PythonTree.FakeNode) }
         )
     }
 

utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/SubtypeValueProvider.kt

@@ -0,0 +1,39 @@
+package org.utbot.python.fuzzing.provider
+
+import org.utbot.fuzzing.Routine
+import org.utbot.fuzzing.Seed
+import org.utbot.fuzzing.ValueProvider
+import org.utbot.python.framework.api.python.PythonTree
+import org.utbot.python.fuzzing.PythonFuzzedValue
+import org.utbot.python.fuzzing.PythonMethodDescription
+import org.utbot.python.fuzzing.provider.utils.isConcreteType
+import org.utbot.python.newtyping.*
+import org.utbot.python.newtyping.PythonSubtypeChecker.Companion.checkIfRightIsSubtypeOfLeft
+import org.utbot.python.newtyping.general.DefaultSubstitutionProvider
+import org.utbot.python.newtyping.general.Type
+
+class SubtypeValueProvider(
+    private val typeStorage: PythonTypeStorage
+) : ValueProvider<Type, PythonFuzzedValue, PythonMethodDescription> {
+    override fun accept(type: Type): Boolean {
+        return type.meta is PythonProtocolDescription ||
+                ((type.meta as? PythonConcreteCompositeTypeDescription)?.isAbstract == true)
+    }
+
+    private val concreteTypes = typeStorage.allTypes.filter {
+        isConcreteType(it) && it.pythonDescription().name.name.first() != '_'  // Don't substitute private classes
+    }.map {
+        DefaultSubstitutionProvider.substituteAll(it, it.parameters.map { pythonAnyType })
+    }
+
+    override fun generate(description: PythonMethodDescription, type: Type) = sequence {
+        val subtypes = concreteTypes.filter { checkIfRightIsSubtypeOfLeft(type, it, typeStorage) }
+        subtypes.forEach { subtype ->
+            yield(
+                Seed.Recursive(
+                construct = Routine.Create(listOf(subtype)) { v -> v.first() },
+                empty = Routine.Empty { PythonFuzzedValue(PythonTree.FakeNode) }
+            ))
+        }
+    }
+}