Skip to content

Commit 1a709a9

Browse files
tamarinvs19tamarinvs19
committed
Add fuzzing for generic types
1 parent a2c4a98 commit 1a709a9

File tree

7 files changed

+128
-87
lines changed

7 files changed

+128
-87
lines changed

‎utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -293,35 +293,35 @@ data class PythonListModel(
293293
}
294294

295295
companion object {
296-
val classId = ClassId("list")
296+
val classId = ClassId("builtins.list")
297297
}
298298
}
299299

300300
data class PythonDictModel(
301301
override val classId: ClassId,
302302
val length: Int = 0,
303-
val stores: Map<PythonModel, PythonModel>
303+
val stores: Map<UtModel, UtModel>
304304
) : PythonModel(classId) {
305305
override fun toString() = withToStringThreadLocalReentrancyGuard {
306306
stores.entries.joinToString(", ", "{", "}") { "${it.key}: ${it.value}" }
307307
}
308308

309309
companion object {
310-
val classId = ClassId("dict")
310+
val classId = ClassId("builtins.dict")
311311
}
312312
}
313313

314314
data class PythonSetModel(
315315
override val classId: ClassId,
316316
val length: Int = 0,
317-
val stores: Set<PythonModel>
317+
val stores: Set<UtModel>
318318
) : PythonModel(classId) {
319319
override fun toString() = withToStringThreadLocalReentrancyGuard {
320320
stores.joinToString(", ", "{", "}") { it.toString() }
321321
}
322322

323323
companion object {
324-
val classId = ClassId("set")
324+
val classId = ClassId("builtins.set")
325325
}
326326
}
327327

‎utbot-python/src/main/kotlin/org/utbot/python/PythonScriptUtils.kt

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,18 @@ fun normalizeAnnotation(
2323
?: error("Didn't find normalize_annotation.py")
2424
saveToFile(codeFilename, scriptContent)
2525

26-
val command = "$pythonPath $codeFilename $annotation $projectRoot $fileOfAnnotation " +
27-
filesToAddToSysPath.joinToString(separator = " ")
28-
val process = Runtime.getRuntime().exec(command)
26+
val process = ProcessBuilder(
27+
listOf(
28+
pythonPath,
29+
codeFilename,
30+
annotation,
31+
projectRoot,
32+
fileOfAnnotation,
33+
) + filesToAddToSysPath,
34+
).start()
35+
// val command = "$pythonPath $codeFilename '$annotation' $projectRoot $fileOfAnnotation " +
36+
// filesToAddToSysPath.joinToString(separator = " ")
37+
// val process = Runtime.getRuntime().exec(command)
2938
process.waitFor()
3039
return process.inputStream.readBytes().decodeToString().trimIndent()
3140
}

‎utbot-python/src/main/kotlin/org/utbot/python/code/CodeGen.kt

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import io.github.danielnaczo.python3parser.model.stmts.smallStmts.assignStmts.As
2727
import io.github.danielnaczo.python3parser.visitors.prettyprint.IndentationPrettyPrint
2828
import io.github.danielnaczo.python3parser.visitors.prettyprint.ModulePrettyPrintVisitor
2929
import org.utbot.framework.plugin.api.UtModel
30+
import org.utbot.framework.plugin.api.pythonAnyClassId
3031
import org.utbot.python.*
3132
import org.utbot.python.typing.StubFileStructures
3233
import java.io.File
@@ -185,7 +186,12 @@ object PythonCodeGenerator {
185186
directoriesForSysPath: List<String>,
186187
additionalModules: List<String> = emptyList(),
187188
): List<Statement> {
188-
val systemImport = Import(listOf(Alias("sys"), Alias("typing"), Alias("json")))
189+
val systemImport = Import(listOf(
190+
Alias("sys"),
191+
Alias("typing"),
192+
Alias("json"),
193+
Alias("builtins"),
194+
))
189195
val systemCalls = directoriesForSysPath.map { path ->
190196
Atom(
191197
Name("sys.path.append"),
@@ -196,14 +202,20 @@ object PythonCodeGenerator {
196202
)
197203
)
198204
}
199-
val additionalImport = additionalModules.mapNotNull {
200-
if (it.contains(".")) {
201-
val module = it.split(".").dropLast(1).joinToString(".")
202-
Import(listOf(Alias(module)))
203-
} else {
204-
null
205+
val additionalImport = additionalModules
206+
.asSequence()
207+
.map { it.split("[", "]", ",", "|") }
208+
.flatten()
209+
.map { it.replace("\\s".toRegex(), "") }
210+
.mapNotNull {
211+
if (it.contains(".")) {
212+
val module = it.split(".").dropLast(1).joinToString(".")
213+
Import(listOf(Alias(module)))
214+
} else {
215+
null
216+
}
205217
}
206-
}
218+
.toSet().toList()
207219

208220
val mathImport = ImportFrom("math", listOf(Alias("*")))
209221
val typingImport = ImportFrom("typing", listOf(Alias("*")))
@@ -309,7 +321,7 @@ object PythonCodeGenerator {
309321

310322
val parameters = Parameters(
311323
method.arguments.map { argument ->
312-
Parameter("${argument.name}: ${methodAnnotations[argument.name] ?: "Any"}")
324+
Parameter("${argument.name}: ${methodAnnotations[argument.name] ?: pythonAnyClassId.name }")
313325
},
314326
)
315327

‎utbot-python/src/main/kotlin/org/utbot/python/code/PythonCodeAPI.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ class PythonMethodBody(private val ast: FunctionDef): PythonMethod {
9494
fun typeAsStringToClassId(typeAsString: String): ClassId = ClassId(typeAsString)
9595

9696
fun annotationToString(annotation: Optional<Expression>): String? =
97-
if (annotation.isPresent) astToString(annotation.get()) else null
97+
if (annotation.isPresent) astToString(annotation.get()).trim() else null
9898
}
9999
}
100100

‎utbot-python/src/main/kotlin/org/utbot/python/providers/GeneralPythonModelProvider.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import org.utbot.fuzzer.FuzzedConcreteValue
55
import org.utbot.fuzzer.FuzzedMethodDescription
66
import org.utbot.fuzzer.ModelProvider
77

8-
val concreteTypesModelProvider = ModelProvider.of(ConstantModelProvider, DefaultValuesModelProvider, InitModelProvider)
8+
val concreteTypesModelProvider = ModelProvider.of(ConstantModelProvider, DefaultValuesModelProvider, InitModelProvider, GenericModelProvider)
99

1010
fun substituteType(description: FuzzedMethodDescription, typeMap: Map<ClassId, ClassId>): FuzzedMethodDescription {
1111
val newReturnType = typeMap[description.returnType] ?: description.returnType

‎utbot-python/src/main/kotlin/org/utbot/python/providers/GenericModelProvider.kt

Lines changed: 66 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -13,55 +13,90 @@ object GenericModelProvider: ModelProvider {
1313
)
1414

1515
override fun generate(description: FuzzedMethodDescription): Sequence<FuzzedParameter> = sequence {
16-
fun parseList(matchResult: MatchResult) = sequence {
17-
val genericType = matchResult.groupValues[0]
16+
fun <T: UtModel> fuzzGeneric(parameters: List<ClassId>, modelConstructor: (List<List<FuzzedValue>>) -> T) = sequence {
1817
val syntheticGenericType = FuzzedMethodDescription(
1918
"${description.name}<syntheticGenericList>",
2019
voidClassId,
21-
listOf(ClassId(genericType)),
20+
parameters,
2221
description.concreteValues
2322
)
24-
val models = fuzz(syntheticGenericType, concreteTypesModelProvider)
25-
.chunked(100) { list -> list.drop(Random.nextInt(list.size)) } // 100 is random max size
26-
.map { list ->
27-
PythonListModel(
28-
PythonListModel.classId,
23+
fuzz(syntheticGenericType, concreteTypesModelProvider)
24+
.randomChunked()
25+
.map(modelConstructor)
26+
.forEach {
27+
yield(FuzzedParameter(0, it.fuzzed()))
28+
}
29+
}
30+
31+
fun parseList(matchResult: MatchResult): Sequence<FuzzedParameter> {
32+
val genericType = if (matchResult.groupValues.size >= 2) ClassId(matchResult.groupValues[1]) else pythonAnyClassId
33+
return fuzzGeneric(listOf(genericType)) { list ->
34+
PythonListModel(
35+
PythonListModel.classId,
36+
list.size,
37+
list.flatten().map { it.model }
38+
)
39+
}
40+
}
41+
42+
fun parseDict(matchResult: MatchResult): Sequence<FuzzedParameter> {
43+
val genericKeyType = if (matchResult.groupValues.size >= 2) ClassId(matchResult.groupValues[1]) else pythonAnyClassId
44+
val genericValueType = if (matchResult.groupValues.size >= 3) ClassId(matchResult.groupValues[2]) else pythonAnyClassId
45+
return fuzzGeneric(listOf(genericKeyType, genericValueType)) { list ->
46+
PythonDictModel(
47+
PythonDictModel.classId,
2948
list.size,
30-
list.flatten().map { it.model },
49+
list.associate { pair ->
50+
pair[0].model to pair[1].model
51+
}
3152
)
3253
}
33-
models.forEach {
34-
yield(FuzzedParameter(0, it.fuzzed()))
35-
}
3654
}
3755

38-
// fun parseDict(matchResult: MatchResult) = sequence {
39-
// TODO("NotImplementedError")
40-
// }
41-
//
42-
// fun parseSet(matchResult: MatchResult) = sequence {
43-
// TODO("NotImplementedError")
44-
// }
56+
fun parseSet(matchResult: MatchResult): Sequence<FuzzedParameter> {
57+
val genericType = if (matchResult.groupValues.size >= 2) ClassId(matchResult.groupValues[1]) else pythonAnyClassId
58+
return fuzzGeneric(listOf(genericType)) { list ->
59+
PythonSetModel(
60+
PythonSetModel.classId,
61+
list.size,
62+
list.flatten().map { it.model }.toSet(),
63+
)
64+
}
65+
}
66+
67+
val modelRegexMap = mapOf<Regex, (MatchResult) -> Sequence<FuzzedParameter>>(
68+
Regex("builtins.list\\[(.*)]") to { matchResult -> parseList(matchResult) },
69+
Regex("[Ll]ist\\[(.*)]") to { matchResult -> parseList(matchResult) },
70+
Regex("typing.List\\[(.*)]") to { matchResult -> parseList(matchResult) },
4571

46-
val modelRegexMap = mapOf<Regex, (MatchResult) -> Unit>(
47-
Regex(".*[Ll]ist\\[(.*)]}") to { matchResult -> parseList(matchResult) },
48-
Regex("typing.List\\[(.*)]}") to { matchResult -> parseList(matchResult) },
49-
// Regex(".*[Dd]ict\\[(.*)]}") to { matchResult -> parseDict(matchResult) },
50-
// Regex("typing.Dict\\[(.*)]}") to { matchResult -> parseDict(matchResult) },
51-
// Regex(".*[Ss]et\\[(.*)]}") to { matchResult -> parseSet(matchResult) },
52-
// Regex("typing.Set\\[(.*)]}") to { matchResult -> parseSet(matchResult) },
72+
Regex("builtins.dict\\[(.*), *(.*)]") to { matchResult -> parseDict(matchResult) },
73+
Regex("[Dd]ict\\[(.*), *(.*)]") to { matchResult -> parseDict(matchResult) },
74+
Regex("typing.Dict\\[(.*)]") to { matchResult -> parseDict(matchResult) },
75+
76+
Regex("builtins.set\\[(.*)]") to { matchResult -> parseSet(matchResult) },
77+
Regex("[Ss]et\\[(.*)]") to { matchResult -> parseSet(matchResult) },
78+
Regex("typing.Set\\[(.*)]") to { matchResult -> parseSet(matchResult) },
5379
)
5480

55-
// val generated = Array(description.parameters.size) { 0 }
5681
description.parametersMap.forEach { (classId, parameterIndices) ->
5782
val annotation = classId.name
58-
modelRegexMap.entries.forEach { (regex, action) ->
59-
val result = regex.matchEntire(annotation)
60-
if (result != null) {
61-
action(result)
83+
parameterIndices.forEach { _ ->
84+
modelRegexMap.entries.forEach { (regex, action) ->
85+
val result = regex.matchEntire(annotation)
86+
if (result != null) {
87+
yieldAll(action(result).take(10))
88+
}
6289
}
6390
}
6491
}
6592
}
6693
}
6794

95+
fun Sequence<List<FuzzedValue>>.randomChunked(): Sequence<List<List<FuzzedValue>>> {
96+
val seq = this
97+
val maxSize = 15
98+
val listOfLists = (0 until maxSize).map { seq.take(10).toList() }
99+
return CartesianProduct(listOfLists, Random(0)).asSequence().map {
100+
it.take(Random.nextInt(maxSize))
101+
}
102+
}

‎utbot-python/types/typeshed_stub.py

Lines changed: 22 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -2,43 +2,16 @@
22
import importlib
33
import json
44
import tqdm
5+
import sys, os
56

7+
from contextlib import contextmanager
68
from collections import defaultdict
7-
from pprint import pprint
89
from typing import Any
10+
from typeshed_client import get_stub_names, get_search_context, OverloadedName, ImportedName
11+
12+
from ast_json_encoders import AstClassEncoder, AstFunctionDefEncoder
913

10-
import astor.code_gen
1114

12-
from typeshed_client import get_stub_names, get_search_context, OverloadedName, ImportedName
13-
from typeshed_client.resolver import Resolver
14-
15-
from ast_json_encoders import AstClassEncoder, AstFunctionDefEncoder, AstAnnAssignEncoder
16-
17-
18-
# MODULES = [
19-
# 'getpass', 'runpy', 'gettext', 'sched', 'glob', 'secrets', 'graphlib', 'select', 'grp', 'selectors', 'gzip',
20-
# 'setuptools', 'hashlib', 'shelve', 'heapq', 'shlex', 'hmac', 'shutil', 'html', 'signal', 'http', 'site', 'idlelib',
21-
# 'smtpd', 'abc', 'imaplib', 'smtplib', 'aifc', 'imghdr', 'sndhdr', 'antigravity', 'imp', 'socket', 'argparse',
22-
# 'importlib', 'socketserver', 'array', 'inspect', 'spwd', 'ast', 'io', 'sqlite3', 'asynchat', 'ipaddress',
23-
# 'sre_compile', 'asyncio', 'itertools', 'sre_constants', 'asyncore', 'json', 'sre_parse', 'atexit', 'keyword',
24-
# 'ssl', 'audioop', 'lib2to3', 'stat', 'base64', 'linecache', 'statistics', 'bdb', 'locale', 'string', 'binascii',
25-
# 'logging', 'stringprep', 'binhex', 'lzma', 'struct', 'bisect', 'mailbox', 'subprocess', 'builtins', 'mailcap',
26-
# 'sunau', 'bz2', 'marshal', 'symtable', 'cProfile', 'math', 'sys', 'calendar', 'mimetypes', 'sysconfig', 'cgi',
27-
# 'mmap', 'syslog', 'cgitb', 'modulefinder', 'tabnanny', 'chunk', 'multiprocessing', 'tarfile', 'cmath', 'netrc',
28-
# 'telnetlib', 'cmd', 'nis', 'tempfile', 'code', 'nntplib', 'termios', 'codecs', 'ntpath', 'test', 'codeop',
29-
# 'nturl2path', 'textwrap', 'collections', 'numbers', 'this', 'colorsys', 'opcode', 'threading', 'compileall',
30-
# 'operator', 'time', 'concurrent', 'optparse', 'timeit', 'configparser', 'os', 'tkinter', 'contextlib',
31-
# 'ossaudiodev', 'token', 'contextvars', 'pathlib', 'tokenize', 'copy', 'pdb', 'trace', 'copyreg', 'pickle',
32-
# 'traceback', 'crypt', 'pickletools', 'tracemalloc', 'csv', 'pip', 'tty', 'ctypes', 'pipes', 'turtle', 'curses',
33-
# 'pkg_resources', 'turtledemo', 'dataclasses', 'pkgutil', 'types', 'datetime', 'platform', 'typing', 'dbm',
34-
# 'plistlib', 'unicodedata', 'decimal', 'poplib', 'unittest', 'difflib', 'posix', 'urllib', 'dis', 'posixpath',
35-
# 'uu', 'distutils', 'pprint', 'uuid', 'doctest', 'profile', 'venv', 'email', 'pstats', 'warnings', 'encodings',
36-
# 'pty', 'wave', 'ensurepip', 'pwd', 'weakref', 'enum', 'py_compile', 'webbrowser', 'errno', 'pyclbr', 'wsgiref',
37-
# 'faulthandler', 'pydoc', 'xdrlib', 'fcntl', 'pydoc_data', 'xml', 'filecmp', 'pyexpat', 'xmlrpc', 'fileinput',
38-
# 'queue', 'xxlimited', 'fnmatch', 'quopri', 'xxlimited_35', 'fractions', 'random', 'xxsubtype', 'ftplib', 're',
39-
# 'zipapp', 'functools', 'readline', 'zipfile', 'gc', 'reprlib', 'zipimport', 'genericpath', 'resource', 'zlib',
40-
# 'getopt', 'rlcompleter', 'zoneinfo'
41-
# ]
4215
MODULES = [
4316
'__future__', '_testinternalcapi', 'getpass', 'runpy', '_abc', '_testmultiphase', 'gettext', 'sched',
4417
'_aix_support', '_thread', 'glob', 'secrets', '_ast', '_threading_local', 'graphlib', 'select', '_asyncio',
@@ -250,13 +223,25 @@ def parse_submodule(module_name: str, collector_: StubFileCollector):
250223
pass
251224

252225

226+
@contextmanager
227+
def suppress_stdout():
228+
with open(os.devnull, "w") as devnull:
229+
old_stdout = sys.stdout
230+
sys.stdout = devnull
231+
try:
232+
yield
233+
finally:
234+
sys.stdout = old_stdout
235+
236+
253237
if __name__ == '__main__':
254238
# create_method_table(BUILTIN_TYPES, 'builtins')
255239
# create_functions_table(BUILTIN_FUNCTIONS, 'builtins')
256240
# create_module_table('builtins')
257241

258-
collector = StubFileCollector('stub_datasets')
259-
for module in tqdm.tqdm(MODULES):
260-
if not module.startswith('_'):
261-
parse_submodule(module, collector)
262-
collector.save_method_annotations()
242+
with suppress_stdout():
243+
collector = StubFileCollector('stub_datasets')
244+
for module in tqdm.tqdm(MODULES):
245+
if not module.startswith('_'):
246+
parse_submodule(module, collector)
247+
collector.save_method_annotations()

0 commit comments

Comments
 (0)