Skip to content

Commit 71cc758

Browse files
tochilinaktochilinak
committed
fixed bug in MypyAnnotations (it didn't work previously); added PythonTypeCollector; added annotation normalization; some other fixes
1 parent dabe4af commit 71cc758

File tree

18 files changed

+644
-411
lines changed

18 files changed

+644
-411
lines changed

‎utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -237,34 +237,34 @@ class PythonDefaultModel(val repr: String, val type: String): PythonModel(ClassI
237237
override fun toString() = repr
238238
}
239239

240-
val pythonAnyClassId = ClassId("Any")
240+
val pythonAnyClassId = ClassId("typing.Any")
241241

242242
class PythonIntModel(val value: BigInteger): PythonModel(classId) {
243243
override fun toString() = "$value"
244244
companion object {
245-
val classId = ClassId("int")
245+
val classId = ClassId("builtins.int")
246246
}
247247
}
248248

249249
class PythonFloatModel(val value: BigDecimal): PythonModel(classId) {
250250
override fun toString() = "$value"
251251
companion object {
252-
val classId = ClassId("float")
252+
val classId = ClassId("builtins.float")
253253
}
254254
}
255255

256256
class PythonStrModel(val value: String): PythonModel(classId) {
257257
override fun toString() = "\"\"\"" + value + "\"\"\""
258258
companion object {
259-
val classId = ClassId("str")
259+
val classId = ClassId("builtins.str")
260260
}
261261
}
262262

263263
class PythonBoolModel(val value: Boolean): PythonModel(classId) {
264264
override fun toString() =
265265
if (value) "True" else "False"
266266
companion object {
267-
val classId = ClassId("bool")
267+
val classId = ClassId("builtins.bool")
268268
}
269269
}
270270

‎utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/python/PythonDialogProcessor.kt

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,14 @@ import org.jetbrains.kotlin.idea.util.projectStructure.sdk
1616
import org.utbot.common.PathUtil.toPath
1717
import org.utbot.intellij.plugin.ui.utils.showErrorDialogLater
1818
import org.utbot.intellij.plugin.ui.utils.testModule
19-
import org.utbot.python.PythonCodeCollector
2019
import org.utbot.python.code.PythonCode
2120
import org.utbot.python.code.PythonCode.Companion.getFromString
2221
import org.utbot.python.code.PythonCodeGenerator.generateTestCode
2322
import org.utbot.python.code.PythonCodeGenerator.saveToFile
2423
import org.utbot.python.PythonMethod
2524
import org.utbot.python.PythonTestCaseGenerator
25+
import org.utbot.python.normalizeAnnotation
26+
import org.utbot.python.typing.PythonTypesStorage
2627

2728

2829
object PythonDialogProcessor {
@@ -89,18 +90,29 @@ object PythonDialogProcessor {
8990
ProgressManager.getInstance().run(object : Backgroundable(project, "Generate python tests") {
9091
override fun run(indicator: ProgressIndicator) {
9192

93+
val pythonPath = model.srcModule.sdk?.homePath ?: error("Couldn't find Python interpreter")
94+
val testSourceRoot = model.testSourceRoot!!.path
95+
val filePath = model.file.virtualFile.path
96+
9297
// PythonCodeCollector.refreshProjectClassesList(model.project.basePath!!)
93-
PythonCodeCollector.refreshProjectClassesList(model.file.virtualFile.path)
98+
PythonTypesStorage.refreshProjectClassesList(
99+
filePath,
100+
pythonPath,
101+
model.project.basePath!!,
102+
model.directoriesForSysPath,
103+
testSourceRoot
104+
)
94105

95106
val pythonMethods = findSelectedPythonMethods(model)
96-
val testSourceRoot = model.testSourceRoot!!.path
97107

98108
val testCaseGenerator = PythonTestCaseGenerator.apply {
99109
init(
100110
testSourceRoot,
101111
model.directoriesForSysPath,
102112
model.moduleToImport,
103-
model.srcModule.sdk?.homePath ?: error("Couldn't find Python interpreter")
113+
pythonPath,
114+
model.project.basePath!!,
115+
filePath
104116
)
105117
}
106118

‎utbot-python/src/main/kotlin/org/utbot/python/PythonCodeCollector.kt

Lines changed: 0 additions & 40 deletions
This file was deleted.

‎utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt

Lines changed: 32 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -6,34 +6,44 @@ import org.utbot.fuzzer.fuzz
66
import org.utbot.fuzzer.names.MethodBasedNameSuggester
77
import org.utbot.fuzzer.names.ModelBasedNameSuggester
88
import org.utbot.python.code.ArgInfoCollector
9-
import org.utbot.python.providers.PythonTypesStorage
109
import org.utbot.python.providers.concreteTypesModelProvider
1110
import org.utbot.python.providers.substituteTypesByIndex
1211
import org.utbot.python.typing.MypyAnnotations
12+
import org.utbot.python.typing.PythonTypesStorage
13+
import org.utbot.python.typing.ReturnRenderType
1314
import org.utbot.python.typing.StubFileFinder
14-
import org.utbot.python.typing.StubFileStructures
15+
import java.io.File
1516

1617
class PythonEngine(
1718
private val methodUnderTest: PythonMethod,
1819
private val testSourceRoot: String,
1920
private val directoriesForSysPath: List<String>,
2021
private val moduleToImport: String,
21-
private val pythonPath: String
22+
private val pythonPath: String,
23+
private val projectRoot: String,
24+
private val fileOfMethod: String
2225
) {
2326
fun fuzzing(): Sequence<PythonResult> = sequence {
24-
val returnType = methodUnderTest.returnType ?: ClassId("")
25-
val argumentTypes = methodUnderTest.arguments.map { it.type }
26-
27-
val existingAnnotations = methodUnderTest.arguments.filter {
28-
it.type.name != "Any"
29-
}.associate {
30-
it.name to it.type.name
27+
val argumentTypes = methodUnderTest.arguments.map {
28+
annotationToClassId(
29+
it.annotation,
30+
pythonPath,
31+
projectRoot,
32+
fileOfMethod,
33+
directoriesForSysPath,
34+
testSourceRoot
35+
)
36+
}
37+
val existingAnnotations = mutableMapOf<String, String>()
38+
argumentTypes.forEachIndexed { index, classId ->
39+
if (classId != pythonAnyClassId)
40+
existingAnnotations[methodUnderTest.arguments[index].name] = classId.name
3141
}
3242

33-
val argInfoCollector = ArgInfoCollector(methodUnderTest)
43+
val argInfoCollector = ArgInfoCollector(methodUnderTest, argumentTypes)
3444
val methodUnderTestDescription = FuzzedMethodDescription(
3545
methodUnderTest.name,
36-
returnType,
46+
pythonAnyClassId,
3747
argumentTypes,
3848
argInfoCollector.getConstants()
3949
).apply {
@@ -44,9 +54,7 @@ class PythonEngine(
4454
val annotations: Sequence<Map<String, ClassId>> =
4555
if (existingAnnotations.size == methodUnderTest.arguments.size)
4656
sequenceOf(
47-
methodUnderTest.arguments.associate {
48-
it.name to it.type
49-
}
57+
existingAnnotations.mapValues { entry -> ClassId(entry.value) }
5058
)
5159
else
5260
joinAnnotations(
@@ -56,7 +64,8 @@ class PythonEngine(
5664
testSourceRoot,
5765
moduleToImport,
5866
directoriesForSysPath,
59-
pythonPath
67+
pythonPath,
68+
fileOfMethod
6069
)
6170

6271
// model provider argwith fallback?
@@ -91,7 +100,9 @@ class PythonEngine(
91100
yield(PythonError(UtError(resultJSON.output, Throwable()), modelList))
92101
} else {
93102

94-
if (PythonTypesStorage.typeNameMap[resultJSON.type]?.useAsReturn != true)
103+
// some types cannot be used as return types in tests (like socket or memoryview)
104+
val outputType = ClassId(resultJSON.type)
105+
if (PythonTypesStorage.getTypeByName(outputType)?.returnRenderType == ReturnRenderType.NONE)
95106
return@sequence
96107

97108
val resultAsModel = PythonDefaultModel(resultJSON.output, "")
@@ -136,6 +147,7 @@ class PythonEngine(
136147
moduleToImport: String,
137148
directoriesForSysPath: List<String>,
138149
pythonPath: String,
150+
fileOfMethod: String
139151
): Sequence<Map<String, ClassId>> {
140152
val storageMap = argInfoCollector.getPriorityStorages()
141153
val userAnnotations = existingAnnotations.entries.associate {
@@ -145,8 +157,8 @@ class PythonEngine(
145157
name to storages.map { storage ->
146158
when (storage) {
147159
is ArgInfoCollector.TypeStorage -> setOf(storage.name)
148-
is ArgInfoCollector.MethodStorage -> StubFileFinder.findTypeWithMethod(storage.name)
149-
is ArgInfoCollector.FieldStorage -> StubFileFinder.findTypeWithField(storage.name)
160+
is ArgInfoCollector.MethodStorage -> PythonTypesStorage.findTypeWithMethod(storage.name)
161+
is ArgInfoCollector.FieldStorage -> PythonTypesStorage.findTypeWithField(storage.name)
150162
is ArgInfoCollector.FunctionArgStorage -> StubFileFinder.findTypeByFunctionWithArgumentPosition(
151163
storage.name,
152164
argumentPosition = storage.index
@@ -163,7 +175,7 @@ class PythonEngine(
163175
userAnnotations + annotationCombinations,
164176
testSourceRoot,
165177
moduleToImport,
166-
directoriesForSysPath,
178+
directoriesForSysPath + listOf(File(fileOfMethod).parentFile.path),
167179
pythonPath
168180
)
169181
}

‎utbot-python/src/main/kotlin/org/utbot/python/PythonScriptUtils.kt

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
package org.utbot.python
2+
3+
import org.utbot.framework.plugin.api.ClassId
4+
import org.utbot.framework.plugin.api.pythonAnyClassId
5+
import org.utbot.python.code.PythonCodeGenerator.saveToFile
6+
import org.utbot.python.typing.PythonTypesStorage
7+
import java.io.BufferedReader
8+
import java.io.InputStreamReader
9+
10+
11+
fun normalizeAnnotation(
12+
annotation: String,
13+
pythonPath: String,
14+
projectRoot: String,
15+
fileOfAnnotation: String,
16+
filesToAddToSysPath: List<String>,
17+
testSourcePath: String
18+
): String {
19+
20+
val codeFilename = "${testSourcePath}/__annotation_check.py"
21+
val scriptContent = PythonTypesStorage::class.java.getResource("/normalize_annotation.py")
22+
?.readText()
23+
?: error("Didn't find normalize_annotation.py")
24+
saveToFile(codeFilename, scriptContent)
25+
26+
val command = "$pythonPath $codeFilename $annotation $projectRoot $fileOfAnnotation " +
27+
filesToAddToSysPath.joinToString(separator = " ")
28+
val process = Runtime.getRuntime().exec(command)
29+
process.waitFor()
30+
return process.inputStream.readBytes().decodeToString().trimIndent()
31+
}
32+
33+
fun annotationToClassId(
34+
annotation: String?,
35+
pythonPath: String,
36+
projectRoot: String,
37+
fileOfAnnotation: String,
38+
filesToAddToSysPath: List<String>,
39+
testSourcePath: String
40+
): ClassId =
41+
if (annotation == null)
42+
pythonAnyClassId
43+
else
44+
ClassId(
45+
normalizeAnnotation(
46+
annotation,
47+
pythonPath,
48+
projectRoot,
49+
fileOfAnnotation,
50+
filesToAddToSysPath,
51+
testSourcePath
52+
)
53+
)

‎utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,17 +7,23 @@ object PythonTestCaseGenerator {
77
lateinit var directoriesForSysPath: List<String>
88
lateinit var moduleToImport: String
99
lateinit var pythonPath: String
10+
lateinit var projectRoot: String
11+
lateinit var fileOfMethod: String
1012

1113
fun init(
1214
testSourceRoot: String,
1315
directoriesForSysPath: List<String>,
1416
moduleToImport: String,
15-
pythonPath: String
17+
pythonPath: String,
18+
projectRoot: String,
19+
fileOfMethod: String
1620
) {
1721
this.testSourceRoot = testSourceRoot
1822
this.directoriesForSysPath = directoriesForSysPath
1923
this.moduleToImport = moduleToImport
2024
this.pythonPath = pythonPath
25+
this.projectRoot = projectRoot
26+
this.fileOfMethod = fileOfMethod
2127
}
2228

2329
fun generate(method: PythonMethod): PythonTestSet {
@@ -26,7 +32,9 @@ object PythonTestCaseGenerator {
2632
testSourceRoot,
2733
directoriesForSysPath,
2834
moduleToImport,
29-
pythonPath
35+
pythonPath,
36+
projectRoot,
37+
fileOfMethod
3038
)
3139
val executions = mutableListOf<PythonExecution>()
3240
val errors = mutableListOf<PythonError>()

‎utbot-python/src/main/kotlin/org/utbot/python/UTPythonAPI.kt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,11 @@ import org.utbot.framework.plugin.api.ClassId
55
import org.utbot.framework.plugin.api.UtExecution
66
import org.utbot.framework.plugin.api.*
77

8-
data class PythonArgument(val name: String, val type: ClassId)
8+
data class PythonArgument(val name: String, val annotation: String?)
99

1010
interface PythonMethod {
1111
val name: String
12-
val returnType: ClassId?
12+
val returnAnnotation: String?
1313
val arguments: List<PythonArgument>
1414
fun asString(): String
1515
fun ast(): FunctionDef

0 commit comments

Comments
 (0)