Skip to content

Commit a3278ef

Browse files
tamarinvs19tamarinvs19
committed
Add cached executions filtration (#1932)
1 parent b5b94a3 commit a3278ef

File tree

4 files changed

+66
-47
lines changed

4 files changed

+66
-47
lines changed

‎utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -216,14 +216,12 @@ class PythonEngine(
216216
PythonFeedback(control = Control.CONTINUE, result = trieNode)
217217
)
218218
}
219-
220-
is ArgumentsTypeErrorFeedback, is TypeErrorFeedback -> {
221-
PythonExecutionResult(result, PythonFeedback(control = Control.PASS))
222-
}
223-
224219
is InvalidExecution -> {
225220
PythonExecutionResult(result, PythonFeedback(control = Control.CONTINUE))
226221
}
222+
else -> {
223+
PythonExecutionResult(result, PythonFeedback(control = Control.PASS))
224+
}
227225
}
228226
}
229227
}
@@ -262,20 +260,19 @@ class PythonEngine(
262260

263261
if (arguments.any { PythonTree.containsFakeNode(it.tree) }) {
264262
logger.debug("FakeNode in Python model")
265-
emit(InvalidExecution(UtError("Bad input object", Throwable())))
263+
emit(FakeNodeFeedback)
266264
return@PythonFuzzing PythonFeedback(control = Control.CONTINUE)
267265
}
268266

269267
val pair = Pair(description, arguments.map { PythonTreeWrapper(it.tree) })
270268
val mem = cache.get(pair)
271269
if (mem != null) {
272270
logger.debug("Repeat in fuzzing")
273-
emit(mem.fuzzingExecutionFeedback)
271+
emit(CachedExecutionFeedback(mem.fuzzingExecutionFeedback))
274272
return@PythonFuzzing mem.fuzzingPlatformFeedback
275273
}
276274
val result = fuzzingResultHandler(description, arguments)
277275
if (result == null) { // timeout
278-
logger.info { "Fuzzing process was interrupted by timeout" }
279276
manager.disconnect()
280277
return@PythonFuzzing PythonFeedback(control = Control.STOP)
281278
}

‎utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt

Lines changed: 51 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@ import java.io.File
3030

3131
private val logger = KotlinLogging.logger {}
3232
private const val RANDOM_TYPE_FREQUENCY = 6
33-
private const val MAX_EXECUTIONS = 50000
3433

3534
class PythonTestCaseGenerator(
3635
private val withMinimization: Boolean = true,
@@ -168,9 +167,7 @@ class PythonTestCaseGenerator(
168167

169168
var feedback: InferredTypeFeedback = SuccessFeedback
170169

171-
val fuzzerCancellation = {
172-
isCancelled() || limitManager.isCancelled() || (errors.size + executions.size) >= MAX_EXECUTIONS
173-
}
170+
val fuzzerCancellation = { isCancelled() || limitManager.isCancelled() }
174171

175172
engine.fuzzing(args, fuzzerCancellation, until).collect {
176173
when (it) {
@@ -182,8 +179,8 @@ class PythonTestCaseGenerator(
182179
}
183180
is InvalidExecution -> {
184181
errors += it.utError
185-
feedback = SuccessFeedback
186-
limitManager.addSuccessExecution()
182+
feedback = InvalidTypeFeedback
183+
limitManager.addInvalidExecution()
187184
}
188185
is ArgumentsTypeErrorFeedback -> {
189186
feedback = InvalidTypeFeedback
@@ -193,6 +190,19 @@ class PythonTestCaseGenerator(
193190
feedback = InvalidTypeFeedback
194191
limitManager.addInvalidExecution()
195192
}
193+
is CachedExecutionFeedback -> {
194+
when (it.cachedFeedback) {
195+
is ValidExecution -> {
196+
limitManager.addSuccessExecution()
197+
}
198+
else -> {
199+
limitManager.addInvalidExecution()
200+
}
201+
}
202+
}
203+
is FakeNodeFeedback -> {
204+
limitManager.addFakeNodeExecutions()
205+
}
196206
}
197207
limitManager.missedLines = missingLines?.size
198208
}
@@ -213,28 +223,41 @@ class PythonTestCaseGenerator(
213223
val coveredLines = mutableSetOf<Int>()
214224

215225
logger.info("Start test generation for ${method.name}")
216-
val meta = method.definition.type.pythonDescription() as PythonCallableTypeDescription
217-
val argKinds = meta.argumentKinds
218-
if (argKinds.any { it != PythonCallableTypeDescription.ArgKind.ARG_POS }) {
219-
val now = System.currentTimeMillis()
220-
val firstUntil = (until - now) / 2 + now
221-
val originalDef = method.definition
222-
val shortType = meta.removeNonPositionalArgs(originalDef.type)
223-
val shortMeta = PythonFuncItemDescription(
224-
originalDef.meta.name,
225-
originalDef.meta.args.take(shortType.arguments.size)
226-
)
227-
val additionalVars = originalDef.meta.args
228-
.drop(shortType.arguments.size)
229-
.joinToString(separator="\n", prefix="\n") { arg ->
230-
"${arg.name}: ${pythonAnyType.pythonTypeRepresentation()}" // TODO: better types
231-
}
232-
method.definition = PythonFunctionDefinition(shortMeta, shortType)
233-
val missingLines = methodHandler(method, typeStorage, coveredLines, errors, executions, null, firstUntil, additionalVars)
234-
method.definition = originalDef
235-
methodHandler(method, typeStorage, coveredLines, errors, executions, missingLines, until)
236-
} else {
237-
methodHandler(method, typeStorage, coveredLines, errors, executions, null, until)
226+
try {
227+
val meta = method.definition.type.pythonDescription() as PythonCallableTypeDescription
228+
val argKinds = meta.argumentKinds
229+
if (argKinds.any { it != PythonCallableTypeDescription.ArgKind.ARG_POS }) {
230+
val now = System.currentTimeMillis()
231+
val firstUntil = (until - now) / 2 + now
232+
val originalDef = method.definition
233+
val shortType = meta.removeNonPositionalArgs(originalDef.type)
234+
val shortMeta = PythonFuncItemDescription(
235+
originalDef.meta.name,
236+
originalDef.meta.args.take(shortType.arguments.size)
237+
)
238+
val additionalVars = originalDef.meta.args
239+
.drop(shortType.arguments.size)
240+
.joinToString(separator = "\n", prefix = "\n") { arg ->
241+
"${arg.name}: ${pythonAnyType.pythonTypeRepresentation()}" // TODO: better types
242+
}
243+
method.definition = PythonFunctionDefinition(shortMeta, shortType)
244+
val missingLines = methodHandler(
245+
method,
246+
typeStorage,
247+
coveredLines,
248+
errors,
249+
executions,
250+
null,
251+
firstUntil,
252+
additionalVars
253+
)
254+
method.definition = originalDef
255+
methodHandler(method, typeStorage, coveredLines, errors, executions, missingLines, until)
256+
} else {
257+
methodHandler(method, typeStorage, coveredLines, errors, executions, null, until)
258+
}
259+
} catch (_: OutOfMemoryError) {
260+
logger.info { "Out of memory error. Stop test generation process" }
238261
}
239262

240263
logger.info("Collect all test executions for ${method.name}")

‎utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,7 @@ import org.utbot.fuzzing.utils.Trie
1616
import org.utbot.python.framework.api.python.PythonTree
1717
import org.utbot.python.fuzzing.provider.*
1818
import org.utbot.python.fuzzing.provider.utils.isAny
19-
import org.utbot.python.fuzzing.provider.utils.isConcreteType
2019
import org.utbot.python.newtyping.*
21-
import org.utbot.python.newtyping.general.DefaultSubstitutionProvider
2220
import org.utbot.python.newtyping.general.Type
2321

2422
private val logger = KotlinLogging.logger {}
@@ -42,6 +40,8 @@ class ValidExecution(val utFuzzedExecution: UtFuzzedExecution): FuzzingExecution
4240
class InvalidExecution(val utError: UtError): FuzzingExecutionFeedback
4341
class TypeErrorFeedback(val message: String) : FuzzingExecutionFeedback
4442
class ArgumentsTypeErrorFeedback(val message: String) : FuzzingExecutionFeedback
43+
class CachedExecutionFeedback(val cachedFeedback: FuzzingExecutionFeedback) : FuzzingExecutionFeedback
44+
object FakeNodeFeedback : FuzzingExecutionFeedback
4545

4646
data class PythonExecutionResult(
4747
val fuzzingExecutionFeedback: FuzzingExecutionFeedback,

‎utbot-python/src/main/kotlin/org/utbot/python/utils/TestGenerationLimitManager.kt

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
package org.utbot.python.utils
22

3-
import kotlin.math.min
4-
53
class TestGenerationLimitManager(
64
// global settings
75
var mode: LimitManagerMode,
@@ -10,18 +8,18 @@ class TestGenerationLimitManager(
108
// local settings: one type inference iteration
119
var executions: Int = 150,
1210
var invalidExecutions: Int = 10,
13-
var additionalExecutions: Int = 5,
11+
var fakeNodeExecutions: Int = 20,
1412
var missedLines: Int? = null,
1513
) {
1614
private val initExecution = executions
1715
private val initInvalidExecutions = invalidExecutions
18-
private val initAdditionalExecutions = additionalExecutions
16+
private val initFakeNodeExecutions = fakeNodeExecutions
1917
private val initMissedLines = missedLines
2018

2119
fun restart() {
2220
executions = initExecution
2321
invalidExecutions = initInvalidExecutions
24-
additionalExecutions = initAdditionalExecutions
22+
fakeNodeExecutions = initFakeNodeExecutions
2523
missedLines = initMissedLines
2624
}
2725

@@ -33,6 +31,10 @@ class TestGenerationLimitManager(
3331
invalidExecutions -= 1
3432
}
3533

34+
fun addFakeNodeExecutions() {
35+
fakeNodeExecutions -= 1
36+
}
37+
3638
fun isCancelled(): Boolean {
3739
return mode.isCancelled(this)
3840
}
@@ -56,10 +58,7 @@ object TimeoutMode : LimitManagerMode {
5658

5759
object ExecutionMode : LimitManagerMode {
5860
override fun isCancelled(manager: TestGenerationLimitManager): Boolean {
59-
if (manager.invalidExecutions <= 0 || manager.executions <= 0) {
60-
return min(manager.invalidExecutions, 0) + min(manager.executions, 0) + manager.additionalExecutions <= 0
61-
}
62-
return false
61+
return manager.invalidExecutions <= 0 || manager.executions <= 0 || manager.fakeNodeExecutions <= 0
6362
}
6463
}
6564

0 commit comments

Comments
 (0)