Skip to content

Commit 408f7fb

Browse files
tamarinvs19tamarinvs19
committed
New fuzzing and type inference connection
1 parent 66ca2ca commit 408f7fb

File tree

8 files changed

+366
-212
lines changed

8 files changed

+366
-212
lines changed

‎utbot-python/src/main/kotlin/org/utbot/python/FuzzingManager.kt

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
package org.utbot.python
2+
3+
import org.utbot.python.fuzzing.PythonFuzzing
4+
import org.utbot.python.fuzzing.PythonMethodDescription
5+
import org.utbot.python.newtyping.general.FunctionType
6+
import org.utbot.python.newtyping.inference.InferredTypeFeedback
7+
import java.util.Random
8+
9+
class FuzzingManager {
10+
private val random: Random = Random(0)
11+
private val annotationsStatistic: MutableMap<FunctionType, AnnotationStatistic> = emptyMap<FunctionType, AnnotationStatistic>().toMutableMap()
12+
var currentFuzzer: PythonFuzzing? = null
13+
val fuzzers: MutableMap<FunctionType, PythonFuzzing> = emptyMap<FunctionType, PythonFuzzing>().toMutableMap()
14+
val descriptions: MutableMap<FunctionType, PythonMethodDescription> = emptyMap<FunctionType, PythonMethodDescription>().toMutableMap()
15+
val fuzzerFeedback: MutableMap<FunctionType, MutableList<FuzzingExecutionFeedback>> = emptyMap<FunctionType, MutableList<FuzzingExecutionFeedback>>().toMutableMap()
16+
val inferFeedback: MutableMap<FunctionType, MutableList<InferredTypeFeedback>> = emptyMap<FunctionType, MutableList<InferredTypeFeedback>>().toMutableMap()
17+
18+
fun addFeedback(type: FunctionType, feedback: FuzzingExecutionFeedback) {
19+
fuzzerFeedback.getOrPut(type) {
20+
emptyList<FuzzingExecutionFeedback>().toMutableList()
21+
}.add(feedback)
22+
}
23+
24+
fun addInferFeedback(type: FunctionType, feedback: InferredTypeFeedback) {
25+
inferFeedback.getOrPut(type) {
26+
emptyList<InferredTypeFeedback>().toMutableList()
27+
}.add(feedback)
28+
}
29+
30+
fun findTheBestAnnotation(): FunctionType? {
31+
if (annotationsStatistic.isEmpty()) return null
32+
val allAnnotations = annotationsStatistic.keys.toList()
33+
return allAnnotations[random.nextInt(allAnnotations.size)]
34+
}
35+
36+
fun update(type: FunctionType, statistic: AnnotationStatistic) {
37+
val newStatistic = annotationsStatistic[type] ?: AnnotationStatistic(emptySet(), 0, 0, 0, 0)
38+
newStatistic.coveredLines += statistic.coveredLines
39+
newStatistic.executionsCount += statistic.executionsCount
40+
newStatistic.successExecutionsCount += statistic.successExecutionsCount
41+
newStatistic.invalidInitializationCount += statistic.invalidInitializationCount
42+
newStatistic.increaseCoverageCount += newStatistic.increaseCoverageCount
43+
annotationsStatistic[type] = newStatistic
44+
}
45+
}
46+
47+
data class AnnotationStatistic(
48+
var coveredLines: Set<Int>,
49+
var executionsCount: Int,
50+
var successExecutionsCount: Int,
51+
var invalidInitializationCount: Int,
52+
var increaseCoverageCount: Int
53+
)

‎utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt

Lines changed: 200 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
package org.utbot.python
22

3-
import kotlinx.coroutines.flow.Flow
4-
import kotlinx.coroutines.flow.flow
3+
import kotlinx.coroutines.delay
4+
import kotlinx.coroutines.launch
55
import mu.KotlinLogging
6+
import org.utbot.common.runBlockingWithCancellationPredicate
67
import org.utbot.framework.plugin.api.DocRegularStmt
78
import org.utbot.framework.plugin.api.EnvironmentModels
89
import org.utbot.framework.plugin.api.UtError
10+
import org.utbot.framework.plugin.api.UtExecution
911
import org.utbot.framework.plugin.api.UtExecutionResult
1012
import org.utbot.framework.plugin.api.UtExecutionSuccess
1113
import org.utbot.framework.plugin.api.UtExplicitlyThrownException
@@ -28,11 +30,19 @@ import org.utbot.python.fuzzing.PythonFuzzedValue
2830
import org.utbot.python.fuzzing.PythonFuzzing
2931
import org.utbot.python.fuzzing.PythonMethodDescription
3032
import org.utbot.python.newtyping.PythonTypeStorage
33+
import org.utbot.python.newtyping.general.FunctionType
3134
import org.utbot.python.newtyping.general.Type
35+
import org.utbot.python.newtyping.inference.InferredTypeFeedback
36+
import org.utbot.python.newtyping.inference.InvalidTypeFeedback
37+
import org.utbot.python.newtyping.inference.SuccessFeedback
38+
import org.utbot.python.newtyping.inference.TypeInferenceAlgorithm
3239
import org.utbot.python.newtyping.pythonModules
3340
import org.utbot.python.newtyping.pythonTypeRepresentation
3441
import org.utbot.python.utils.camelToSnakeCase
42+
import org.utbot.python.utils.updateCoverage
3543
import org.utbot.summary.fuzzer.names.TestSuggestedInfo
44+
import java.lang.Exception
45+
import java.util.Random
3646

3747
private val logger = KotlinLogging.logger {}
3848

@@ -42,16 +52,144 @@ class InvalidExecution(val utError: UtError): FuzzingExecutionFeedback
4252
class TypeErrorFeedback(val message: String) : FuzzingExecutionFeedback
4353
class ArgumentsTypeErrorFeedback(val message: String) : FuzzingExecutionFeedback
4454

55+
private const val COVERAGE_LIMIT = 20
56+
4557
class PythonEngine(
4658
private val methodUnderTest: PythonMethod,
4759
private val directoriesForSysPath: Set<String>,
4860
private val moduleToImport: String,
4961
private val pythonPath: String,
5062
private val fuzzedConcreteValues: List<PythonFuzzedConcreteValue>,
5163
private val timeoutForRun: Long,
52-
private val initialCoveredLines: Set<Int>,
5364
private val pythonTypeStorage: PythonTypeStorage,
65+
private val algorithm: TypeInferenceAlgorithm,
66+
private val isCancelled: () -> Boolean,
67+
private val until: Long,
5468
) {
69+
val executions: MutableList<UtExecution> = emptyList<UtExecution>().toMutableList()
70+
val errors: MutableList<UtError> = emptyList<UtError>().toMutableList()
71+
var missedLines: Set<Int>? = null
72+
73+
private val coveredLines: MutableSet<Int> = emptyList<Int>().toMutableSet()
74+
private val fuzzingManager = FuzzingManager()
75+
private var engineIsCancelled = { isCancelled() || System.currentTimeMillis() >= until || missedLines?.size == 0 }
76+
77+
suspend fun run(functionType: FunctionType) {
78+
if (!fuzzingManager.fuzzers.contains(functionType)) {
79+
buildFuzzer(functionType)
80+
}
81+
fuzzingManager.currentFuzzer = fuzzingManager.fuzzers[functionType]
82+
runBlockingWithCancellationPredicate(engineIsCancelled) {
83+
launch {
84+
try {
85+
fuzzingManager.currentFuzzer?.fuzz(fuzzingManager.descriptions[functionType]!!)
86+
} catch (_: Exception) {
87+
fuzzingManager.currentFuzzer = null
88+
}
89+
}
90+
launch {
91+
while (fuzzingManager.currentFuzzer != null) {
92+
delay(100)
93+
}
94+
val feedback = fuzzingManager.inferFeedback.getOrDefault(functionType, emptyList<InferredTypeFeedback>().toMutableList()).let {
95+
if (it.contains(SuccessFeedback)) SuccessFeedback
96+
else InvalidTypeFeedback
97+
}
98+
algorithm.pushFeedback(feedback)
99+
100+
val nextAlgorithmType = algorithm.getType(isCancelled)
101+
val nextHistoryType = fuzzingManager.findTheBestAnnotation()
102+
val nextType = nextAlgorithmType?.let {
103+
if (Random().nextDouble() > 0.3) it else null
104+
} ?: nextHistoryType
105+
106+
if (nextType is FunctionType) {
107+
run(nextType)
108+
} else {
109+
engineIsCancelled = { true }
110+
}
111+
}
112+
}
113+
}
114+
115+
private fun buildFuzzer(functionType: FunctionType) {
116+
val arguments = functionType.arguments
117+
logger.info { "Prepare to fuzzing ${arguments.joinToString { it.pythonTypeRepresentation() }}" }
118+
119+
val pmd = PythonMethodDescription(
120+
methodUnderTest.name,
121+
arguments,
122+
fuzzedConcreteValues,
123+
pythonTypeStorage,
124+
)
125+
126+
var coverageLimit = COVERAGE_LIMIT
127+
var coveredBefore = coveredLines.size
128+
129+
var badExecutionCounter = 0
130+
var errorsCounter = 0
131+
var executionCounter = 0
132+
var coverageCounter = 0
133+
134+
val fuzzingCancellation = { isCancelled() || System.currentTimeMillis() > until || coverageLimit <= 0 || fuzzingManager.currentFuzzer == null }
135+
val fuzzer = getFuzzer(functionType, isCancelled, until, {fuzzingManager.currentFuzzer}) {
136+
if (fuzzingCancellation()) {
137+
fuzzingManager.update(
138+
functionType,
139+
AnnotationStatistic(
140+
coveredLines.toSet(),
141+
executionCounter + badExecutionCounter + errorsCounter,
142+
executionCounter,
143+
badExecutionCounter,
144+
coverageCounter
145+
)
146+
)
147+
fuzzingManager.currentFuzzer = null
148+
coverageLimit = COVERAGE_LIMIT
149+
coveredBefore = coveredLines.size
150+
151+
badExecutionCounter = 0
152+
errorsCounter = 0
153+
executionCounter = 0
154+
coverageCounter = 0
155+
} else {
156+
val feedback = when (it) {
157+
is ValidExecution -> {
158+
executions += it.utFuzzedExecution
159+
missedLines = updateCoverage(it.utFuzzedExecution, coveredLines, missedLines)
160+
executionCounter++
161+
SuccessFeedback
162+
}
163+
is InvalidExecution -> {
164+
errors += it.utError
165+
errorsCounter++
166+
SuccessFeedback
167+
}
168+
is ArgumentsTypeErrorFeedback, is TypeErrorFeedback -> {
169+
badExecutionCounter++
170+
InvalidTypeFeedback
171+
}
172+
}
173+
fuzzingManager.addInferFeedback(functionType, feedback)
174+
175+
val coveredAfter = coveredLines.size
176+
if (coveredAfter == coveredBefore) {
177+
coverageLimit -= 1
178+
}
179+
if (coveredAfter > coveredBefore) {
180+
coverageCounter++
181+
}
182+
coveredBefore = coveredAfter
183+
}
184+
}
185+
fuzzingManager.fuzzers[functionType] = fuzzer
186+
fuzzingManager.descriptions[functionType] = pmd
187+
fuzzingManager.currentFuzzer = fuzzer
188+
189+
// fuzzer.fuzz(pmd)
190+
191+
logger.info { "Fuzzer created" }
192+
}
55193

56194
private fun suggestExecutionName(
57195
description: PythonMethodDescription,
@@ -170,84 +308,80 @@ class PythonEngine(
170308
)
171309
}
172310

173-
fun fuzzing(parameters: List<Type>, isCancelled: () -> Boolean, until: Long): Flow<FuzzingExecutionFeedback> = flow {
311+
private fun getFuzzer(
312+
functionType: FunctionType,
313+
isCancelled: () -> Boolean,
314+
until: Long,
315+
activeFuzzing: suspend () -> PythonFuzzing?,
316+
handleFeedback: suspend (FuzzingExecutionFeedback) -> Unit,
317+
): PythonFuzzing {
318+
val parameters = functionType.arguments
174319
val additionalModules = parameters.flatMap { it.pythonModules() }
175320

176-
val pmd = PythonMethodDescription(
177-
methodUnderTest.name,
178-
parameters,
179-
fuzzedConcreteValues,
180-
pythonTypeStorage,
181-
)
182-
183-
val coveredLines = initialCoveredLines.toMutableSet()
184-
var sourceLinesCount = Long.MAX_VALUE
321+
val coveredLines = coveredLines.toMutableSet()
322+
var sourceLinesCount: Long? = null
185323

186-
PythonFuzzing(pmd.pythonTypeStorage) { description, arguments ->
187-
if (isCancelled()) {
188-
logger.info { "Fuzzing process was interrupted" }
189-
return@PythonFuzzing PythonFeedback(control = Control.STOP)
190-
}
191-
if (System.currentTimeMillis() >= until) {
192-
logger.info { "Fuzzing process was interrupted by timeout" }
193-
return@PythonFuzzing PythonFeedback(control = Control.STOP)
194-
}
195-
196-
val codeExecutor = constructEvaluationInput(arguments, additionalModules)
197-
// val jobResult = evaluationInput.evaluate()
198-
199-
when (val evaluationResult = codeExecutor.run()) {
200-
is PythonEvaluationError -> {
201-
val utError = UtError(
202-
"Error evaluation: ${evaluationResult.status}, ${evaluationResult.message}",
203-
Throwable(evaluationResult.stackTrace.joinToString("\n"))
204-
)
205-
logger.debug(evaluationResult.stackTrace.joinToString("\n"))
206-
emit(InvalidExecution(utError))
207-
return@PythonFuzzing PythonFeedback(control = Control.PASS)
324+
return fuzzingManager.fuzzers.getOrPut(functionType) {
325+
PythonFuzzing(pythonTypeStorage, activeFuzzing) { description, arguments ->
326+
logger.info {arguments.first().summary}
327+
if (isCancelled()) {
328+
logger.info { "Fuzzing process was interrupted" }
329+
return@PythonFuzzing PythonFeedback(control = Control.STOP)
208330
}
209-
210-
is PythonEvaluationTimeout -> {
211-
val utError = UtError(evaluationResult.message, Throwable())
212-
emit(InvalidExecution(utError))
213-
return@PythonFuzzing PythonFeedback(control = Control.PASS)
331+
if (System.currentTimeMillis() >= until) {
332+
logger.info { "Fuzzing process was interrupted by timeout" }
333+
return@PythonFuzzing PythonFeedback(control = Control.STOP)
214334
}
215335

216-
is PythonEvaluationSuccess -> {
217-
evaluationResult.coverage.coveredInstructions.forEach { coveredLines.add(it.lineNumber) }
218-
val instructionsCount = evaluationResult.coverage.instructionsCount
219-
if (instructionsCount != null) {
220-
sourceLinesCount = instructionsCount
336+
val codeExecutor = constructEvaluationInput(arguments, additionalModules)
337+
when (val evaluationResult = codeExecutor.run()) {
338+
is PythonEvaluationError -> {
339+
val utError = UtError(
340+
"Error evaluation: ${evaluationResult.status}, ${evaluationResult.message}",
341+
Throwable(evaluationResult.stackTrace.joinToString("\n"))
342+
)
343+
logger.debug(evaluationResult.stackTrace.joinToString("\n"))
344+
fuzzingManager.addFeedback(functionType, InvalidExecution(utError))
345+
handleFeedback(InvalidExecution(utError))
346+
return@PythonFuzzing PythonFeedback(control = Control.PASS)
221347
}
222348

223-
val summary = arguments
224-
.zip(methodUnderTest.arguments)
225-
.mapNotNull { it.first.summary?.replace("%var%", it.second.name) }
226-
227-
val hasThisObject = codeExecutor.methodArguments.thisObject != null
228-
val result = handleSuccessResult(parameters, evaluationResult, description, hasThisObject, summary)
229-
emit(result)
230-
231-
if (coveredLines.size.toLong() == sourceLinesCount) {
232-
return@PythonFuzzing PythonFeedback(control = Control.STOP)
349+
is PythonEvaluationTimeout -> {
350+
val utError = UtError(evaluationResult.message, Throwable())
351+
handleFeedback(InvalidExecution(utError))
352+
return@PythonFuzzing PythonFeedback(control = Control.PASS)
233353
}
234354

235-
when (result) {
236-
is ValidExecution -> {
237-
return@PythonFuzzing PythonFeedback(control = Control.CONTINUE)
238-
}
239-
is ArgumentsTypeErrorFeedback -> {
240-
return@PythonFuzzing PythonFeedback(control = Control.PASS)
355+
is PythonEvaluationSuccess -> {
356+
evaluationResult.coverage.coveredInstructions.forEach { coveredLines.add(it.lineNumber) }
357+
val instructionsCount = evaluationResult.coverage.instructionsCount
358+
if (instructionsCount != null) {
359+
sourceLinesCount = instructionsCount
241360
}
242-
is TypeErrorFeedback -> {
243-
return@PythonFuzzing PythonFeedback(control = Control.PASS)
361+
362+
val summary = arguments
363+
.zip(methodUnderTest.arguments)
364+
.mapNotNull { it.first.summary?.replace("%var%", it.second.name) }
365+
366+
val hasThisObject = codeExecutor.methodArguments.thisObject != null
367+
val result = handleSuccessResult(parameters, evaluationResult, description, hasThisObject, summary)
368+
handleFeedback(result)
369+
370+
if (coveredLines.size.toLong() == sourceLinesCount) {
371+
return@PythonFuzzing PythonFeedback(control = Control.STOP)
244372
}
245-
is InvalidExecution -> {
246-
return@PythonFuzzing PythonFeedback(control = Control.CONTINUE)
373+
374+
when (result) {
375+
is ValidExecution, is InvalidExecution -> {
376+
return@PythonFuzzing PythonFeedback(control = Control.CONTINUE)
377+
}
378+
is ArgumentsTypeErrorFeedback, is TypeErrorFeedback -> {
379+
return@PythonFuzzing PythonFeedback(control = Control.PASS)
380+
}
247381
}
248382
}
249383
}
250384
}
251-
}.fuzz(pmd)
385+
}
252386
}
253387
}

0 commit comments

Comments
 (0)