Skip to content

Commit 529ab98

Browse files
tamarinvs19tamarinvs19
authored
Fix python fuzzing strategy (#1914)
1 parent cf9c3f3 commit 529ab98

File tree

4 files changed

+77
-128
lines changed

4 files changed

+77
-128
lines changed

‎utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt

Lines changed: 62 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ import org.utbot.python.newtyping.PythonTypeStorage
1818
import org.utbot.python.newtyping.general.Type
1919
import org.utbot.python.newtyping.pythonModules
2020
import org.utbot.python.newtyping.pythonTypeRepresentation
21-
import org.utbot.python.utils.TestGenerationLimitManager
2221
import org.utbot.python.utils.camelToSnakeCase
2322
import org.utbot.summary.fuzzer.names.TestSuggestedInfo
2423
import java.net.ServerSocket
@@ -143,30 +142,30 @@ class PythonEngine(
143142
)
144143
}
145144

146-
fun fuzzing(parameters: List<Type>, isCancelled: () -> Boolean, limitManager: TestGenerationLimitManager): Flow<FuzzingExecutionFeedback> = flow {
145+
fun fuzzing(parameters: List<Type>, isCancelled: () -> Boolean, until: Long): Flow<FuzzingExecutionFeedback> = flow {
147146
val additionalModules = parameters.flatMap { it.pythonModules() }
148147

149148
ServerSocket(0).use { serverSocket ->
150-
logger.debug { "Server port: ${serverSocket.localPort}" }
149+
logger.info { "Server port: ${serverSocket.localPort}" }
151150
val manager = try {
152151
PythonWorkerManager(
153152
serverSocket,
154153
pythonPath,
155-
limitManager.until,
154+
until,
156155
{ constructEvaluationInput(it) },
157156
timeoutForRun.toInt()
158157
)
159158
} catch (_: TimeoutException) {
160-
logger.info { "Cannot connect to python executor" }
161159
return@flow
162160
}
163161
logger.info { "Executor manager was created successfully" }
164162

165-
fun runWithFuzzedValues(
166-
arguments: List<PythonFuzzedValue>,
167-
): PythonEvaluationResult? {
163+
fun fuzzingResultHandler(
164+
description: PythonMethodDescription,
165+
arguments: List<PythonFuzzedValue>
166+
): PythonExecutionResult? {
168167
val argumentValues = arguments.map { PythonTreeModel(it.tree, it.tree.type) }
169-
logger.debug(argumentValues.map { it.tree }.toString())
168+
logger.debug(argumentValues.map { it.tree } .toString())
170169
val argumentModules = argumentValues
171170
.flatMap { it.allContainingClassIds }
172171
.map { it.moduleName }
@@ -184,83 +183,52 @@ class PythonEngine(
184183
modelList,
185184
methodUnderTest.argumentsNames
186185
)
187-
return try {
188-
manager.run(functionArguments, localAdditionalModules)
189-
} catch (_: TimeoutException) {
190-
logger.info { "Fuzzing process was interrupted by timeout" }
191-
null
192-
}
193-
}
194-
195-
fun handleExecutionResult(
196-
result: PythonEvaluationResult,
197-
arguments: List<PythonFuzzedValue>,
198-
description: PythonMethodDescription,
199-
): Pair<PythonExecutionResult, Boolean> {
200-
val executionFeedback: FuzzingExecutionFeedback
201-
val fuzzingFeedback: PythonFeedback
202-
203-
when(result) {
204-
is PythonEvaluationError -> {
205-
val utError = UtError(
206-
"Error evaluation: ${result.status}, ${result.message}",
207-
Throwable(result.stackTrace.joinToString("\n"))
208-
)
209-
logger.debug(result.stackTrace.joinToString("\n"))
210-
211-
limitManager.addSuccessExecution()
212-
executionFeedback = InvalidExecution(utError)
213-
fuzzingFeedback = PythonFeedback(control = Control.PASS)
214-
return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), true)
215-
}
216-
217-
is PythonEvaluationTimeout -> {
218-
val utError = UtError(result.message, Throwable())
219-
limitManager.addInvalidExecution()
220-
executionFeedback = InvalidExecution(utError)
221-
fuzzingFeedback = PythonFeedback(control = Control.PASS)
222-
return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false)
223-
}
186+
try {
187+
return when (val evaluationResult = manager.run(functionArguments, localAdditionalModules)) {
188+
is PythonEvaluationError -> {
189+
val utError = UtError(
190+
"Error evaluation: ${evaluationResult.status}, ${evaluationResult.message}",
191+
Throwable(evaluationResult.stackTrace.joinToString("\n"))
192+
)
193+
logger.debug(evaluationResult.stackTrace.joinToString("\n"))
194+
PythonExecutionResult(InvalidExecution(utError), PythonFeedback(control = Control.PASS))
195+
}
224196

225-
is PythonEvaluationSuccess -> {
226-
val coveredInstructions = result.coverage.coveredInstructions
227-
executionFeedback = handleSuccessResult(
228-
arguments,
229-
parameters,
230-
result,
231-
description,
232-
)
197+
is PythonEvaluationTimeout -> {
198+
val utError = UtError(evaluationResult.message, Throwable())
199+
PythonExecutionResult(InvalidExecution(utError), PythonFeedback(control = Control.PASS))
200+
}
233201

234-
val trieNode: Trie.Node<Instruction> = description.tracer.add(coveredInstructions)
235-
when (executionFeedback) {
236-
is ValidExecution -> {
237-
limitManager.addSuccessExecution()
238-
if (trieNode.count > 1) {
239-
fuzzingFeedback = PythonFeedback(control = Control.CONTINUE, result = trieNode)
240-
return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false)
202+
is PythonEvaluationSuccess -> {
203+
val coveredInstructions = evaluationResult.coverage.coveredInstructions
204+
205+
when (val result = handleSuccessResult(
206+
arguments,
207+
parameters,
208+
evaluationResult,
209+
description,
210+
)) {
211+
is ValidExecution -> {
212+
val trieNode: Trie.Node<Instruction> = description.tracer.add(coveredInstructions)
213+
PythonExecutionResult(
214+
result,
215+
PythonFeedback(control = Control.CONTINUE, result = trieNode)
216+
)
241217
}
242-
}
243-
244-
is ArgumentsTypeErrorFeedback -> {
245-
fuzzingFeedback = PythonFeedback(control = Control.PASS)
246-
return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false)
247-
}
248218

249-
is TypeErrorFeedback -> {
250-
limitManager.addInvalidExecution()
251-
fuzzingFeedback = PythonFeedback(control = Control.PASS)
252-
return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false)
253-
}
219+
is ArgumentsTypeErrorFeedback, is TypeErrorFeedback -> {
220+
PythonExecutionResult(result, PythonFeedback(control = Control.PASS))
221+
}
254222

255-
is InvalidExecution -> {
256-
limitManager.addInvalidExecution()
257-
fuzzingFeedback = PythonFeedback(control = Control.CONTINUE)
258-
return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false)
223+
is InvalidExecution -> {
224+
PythonExecutionResult(result, PythonFeedback(control = Control.CONTINUE))
225+
}
259226
}
260227
}
261-
fuzzingFeedback = PythonFeedback(control = Control.CONTINUE, result = trieNode)
262-
return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), true)
263228
}
229+
} catch (_: TimeoutException) {
230+
logger.info { "Fuzzing process was interrupted by timeout" }
231+
return null
264232
}
265233
}
266234

@@ -273,14 +241,10 @@ class PythonEngine(
273241
)
274242

275243
if (parameters.isEmpty()) {
276-
val result = runWithFuzzedValues(emptyList())
244+
val result = fuzzingResultHandler(pmd, emptyList())
277245
result?.let {
278-
val (executionResult, needToEmit) = handleExecutionResult(result, emptyList(), pmd)
279-
if (needToEmit) {
280-
emit(executionResult.fuzzingExecutionFeedback)
281-
}
246+
emit(it.fuzzingExecutionFeedback)
282247
}
283-
manager.disconnect()
284248
} else {
285249
try {
286250
PythonFuzzing(pmd.pythonTypeStorage) { description, arguments ->
@@ -289,32 +253,35 @@ class PythonEngine(
289253
manager.disconnect()
290254
return@PythonFuzzing PythonFeedback(control = Control.STOP)
291255
}
256+
if (System.currentTimeMillis() >= until) {
257+
logger.info { "Fuzzing process was interrupted by timeout" }
258+
manager.disconnect()
259+
return@PythonFuzzing PythonFeedback(control = Control.STOP)
260+
}
292261

293262
val pair = Pair(description, arguments.map { PythonTreeWrapper(it.tree) })
294263
val mem = cache.get(pair)
295264
if (mem != null) {
296265
logger.debug("Repeat in fuzzing")
266+
emit(mem.fuzzingExecutionFeedback)
297267
return@PythonFuzzing mem.fuzzingPlatformFeedback
298268
}
299-
300-
val result = runWithFuzzedValues(arguments)
269+
val result = fuzzingResultHandler(description, arguments)
301270
if (result == null) { // timeout
271+
logger.info { "Fuzzing process was interrupted by timeout" }
302272
manager.disconnect()
303273
return@PythonFuzzing PythonFeedback(control = Control.STOP)
304274
}
305275

306-
val (executionResult, needToEmit) = handleExecutionResult(result, arguments, description)
307-
cache.add(pair, executionResult)
308-
if (needToEmit) {
309-
emit(executionResult.fuzzingExecutionFeedback)
310-
}
311-
return@PythonFuzzing executionResult.fuzzingPlatformFeedback
276+
cache.add(pair, result)
277+
emit(result.fuzzingExecutionFeedback)
278+
return@PythonFuzzing result.fuzzingPlatformFeedback
312279
}.fuzz(pmd)
313-
} catch (ex: Exception) { // NoSeedValueException
280+
} catch (_: Exception) { // e.g. NoSeedValueException
314281
logger.info { "Cannot fuzz values for types: $parameters" }
315282
}
316-
manager.disconnect()
317283
}
284+
manager.disconnect()
318285
}
319286
}
320-
}
287+
}

‎utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,23 +167,27 @@ class PythonTestCaseGenerator(
167167

168168
val fuzzerCancellation = { isCancelled() || limitManager.isCancelled() }
169169

170-
engine.fuzzing(args, fuzzerCancellation, limitManager).collect {
170+
engine.fuzzing(args, fuzzerCancellation, until).collect {
171171
generated += 1
172172
when (it) {
173173
is ValidExecution -> {
174174
executions += it.utFuzzedExecution
175175
missingLines = updateCoverage(it.utFuzzedExecution, coveredLines, missingLines)
176176
feedback = SuccessFeedback
177+
limitManager.addSuccessExecution()
177178
}
178179
is InvalidExecution -> {
179180
errors += it.utError
180181
feedback = SuccessFeedback
182+
limitManager.addSuccessExecution()
181183
}
182184
is ArgumentsTypeErrorFeedback -> {
183185
feedback = InvalidTypeFeedback
186+
limitManager.addInvalidExecution()
184187
}
185188
is TypeErrorFeedback -> {
186189
feedback = InvalidTypeFeedback
190+
limitManager.addInvalidExecution()
187191
}
188192
}
189193
limitManager.missedLines = missingLines?.size

‎utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt

Lines changed: 9 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,6 @@ import mu.KotlinLogging
44
import org.utbot.framework.plugin.api.Instruction
55
import org.utbot.framework.plugin.api.UtError
66
import org.utbot.fuzzer.FuzzedContext
7-
import org.utbot.fuzzer.IdGenerator
8-
import org.utbot.fuzzer.IdentityPreservingIdGenerator
97
import org.utbot.fuzzer.UtFuzzedExecution
108
import org.utbot.fuzzing.Configuration
119
import org.utbot.fuzzing.Control
@@ -23,8 +21,6 @@ import org.utbot.python.newtyping.PythonSubtypeChecker
2321
import org.utbot.python.newtyping.PythonTypeStorage
2422
import org.utbot.python.newtyping.general.Type
2523
import org.utbot.python.newtyping.pythonTypeRepresentation
26-
import java.util.*
27-
import java.util.concurrent.atomic.AtomicLong
2824

2925
private val logger = KotlinLogging.logger {}
3026

@@ -63,7 +59,7 @@ class PythonFuzzedValue(
6359
val summary: String? = null,
6460
)
6561

66-
fun pythonDefaultValueProviders(idGenerator: IdGenerator<Long>) = listOf(
62+
fun pythonDefaultValueProviders() = listOf(
6763
NoneValueProvider,
6864
BoolValueProvider,
6965
IntValueProvider,
@@ -88,39 +84,38 @@ class PythonFuzzing(
8884
val execute: suspend (description: PythonMethodDescription, values: List<PythonFuzzedValue>) -> PythonFeedback,
8985
) : Fuzzing<Type, PythonFuzzedValue, PythonMethodDescription, PythonFeedback> {
9086

91-
private fun generateDefault(description: PythonMethodDescription, type: Type, idGenerator: IdGenerator<Long>): Sequence<Seed<Type, PythonFuzzedValue>> {
87+
private fun generateDefault(description: PythonMethodDescription, type: Type): Sequence<Seed<Type, PythonFuzzedValue>> {
9288
var providers = emptyList<Seed<Type, PythonFuzzedValue>>().asSequence()
93-
pythonDefaultValueProviders(idGenerator).asSequence().forEach { provider ->
89+
pythonDefaultValueProviders().asSequence().forEach { provider ->
9490
if (provider.accept(type)) {
95-
logger.info { "Provider ${provider.javaClass.simpleName} accepts type ${type.pythonTypeRepresentation()}" }
91+
logger.debug { "Provider ${provider.javaClass.simpleName} accepts type ${type.pythonTypeRepresentation()}" }
9692
providers += provider.generate(description, type)
9793
}
9894
}
9995
return providers
10096
}
10197

102-
private fun generateSubtype(description: PythonMethodDescription, type: Type, idGenerator: IdGenerator<Long>): Sequence<Seed<Type, PythonFuzzedValue>> {
98+
private fun generateSubtype(description: PythonMethodDescription, type: Type): Sequence<Seed<Type, PythonFuzzedValue>> {
10399
var providers = emptyList<Seed<Type, PythonFuzzedValue>>().asSequence()
104100
if (type.meta is PythonProtocolDescription) {
105101
val subtypes = pythonTypeStorage.allTypes.filter {
106102
PythonSubtypeChecker.checkIfRightIsSubtypeOfLeft(type, it, pythonTypeStorage)
107103
}
108104
subtypes.forEach {
109-
providers += generateDefault(description, it, idGenerator)
105+
providers += generateDefault(description, it)
110106
}
111107
}
112108
return providers
113109
}
114110

115111
override fun generate(description: PythonMethodDescription, type: Type): Sequence<Seed<Type, PythonFuzzedValue>> {
116-
val idGenerator = PythonIdGenerator()
117112
var providers = emptyList<Seed<Type, PythonFuzzedValue>>().asSequence()
118113

119114
if (type.isAny()) {
120-
logger.info("Any does not have provider")
115+
logger.debug("Any does not have provider")
121116
} else {
122-
providers += generateDefault(description, type, idGenerator)
123-
providers += generateSubtype(description, type, idGenerator)
117+
providers += generateDefault(description, type)
118+
providers += generateSubtype(description, type)
124119
}
125120

126121
return providers
@@ -138,20 +133,3 @@ class PythonFuzzing(
138133
super.update(description, statistic, configuration)
139134
}
140135
}
141-
142-
class PythonIdGenerator(lowerBound: Long = DEFAULT_LOWER_BOUND) : IdentityPreservingIdGenerator<Long> {
143-
private val lastId: AtomicLong = AtomicLong(lowerBound)
144-
private val cache: IdentityHashMap<Any?, Long> = IdentityHashMap()
145-
146-
override fun getOrCreateIdForValue(value: Any): Long {
147-
return cache.getOrPut(value) { createId() }
148-
}
149-
150-
override fun createId(): Long {
151-
return lastId.incrementAndGet()
152-
}
153-
154-
companion object {
155-
const val DEFAULT_LOWER_BOUND: Long = 1500_000_000
156-
}
157-
}

‎utbot-python/src/main/kotlin/org/utbot/python/utils/TestGenerationLimitManager.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ object TimeoutMode : LimitManagerMode {
5757
object ExecutionMode : LimitManagerMode {
5858
override fun isCancelled(manager: TestGenerationLimitManager): Boolean {
5959
if (manager.invalidExecutions <= 0 || manager.executions <= 0) {
60-
return min(manager.invalidExecutions, 0) + min(manager.executions, 0) <= manager.additionalExecutions
60+
return min(manager.invalidExecutions, 0) + min(manager.executions, 0) + manager.additionalExecutions <= 0
6161
}
6262
return false
6363
}

0 commit comments

Comments
 (0)