Skip to content

Add cached executions filtration #1932

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Mar 13, 2023
Merged

Add cached executions filtration #1932

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e37913e
Add filter of cached executions
tamarinvs19 Mar 13, 2023
b4ba484
Remove addition execution limit
tamarinvs19 Mar 13, 2023
2d19392
Add try-catch for OutOfMemoryLimit
tamarinvs19 Mar 13, 2023
209139a
Add FakeNodeFeedback
tamarinvs19 Mar 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 5 additions & 8 deletions utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,14 +216,12 @@ class PythonEngine(
PythonFeedback(control = Control.CONTINUE, result = trieNode)
)
}

is ArgumentsTypeErrorFeedback, is TypeErrorFeedback -> {
PythonExecutionResult(result, PythonFeedback(control = Control.PASS))
}

is InvalidExecution -> {
PythonExecutionResult(result, PythonFeedback(control = Control.CONTINUE))
}
else -> {
PythonExecutionResult(result, PythonFeedback(control = Control.PASS))
}
}
}
}
Expand Down Expand Up @@ -262,20 +260,19 @@ class PythonEngine(

if (arguments.any { PythonTree.containsFakeNode(it.tree) }) {
logger.debug("FakeNode in Python model")
emit(InvalidExecution(UtError("Bad input object", Throwable())))
emit(FakeNodeFeedback)
return@PythonFuzzing PythonFeedback(control = Control.CONTINUE)
}

val pair = Pair(description, arguments.map { PythonTreeWrapper(it.tree) })
val mem = cache.get(pair)
if (mem != null) {
logger.debug("Repeat in fuzzing")
emit(mem.fuzzingExecutionFeedback)
emit(CachedExecutionFeedback(mem.fuzzingExecutionFeedback))
return@PythonFuzzing mem.fuzzingPlatformFeedback
}
val result = fuzzingResultHandler(description, arguments)
if (result == null) { // timeout
logger.info { "Fuzzing process was interrupted by timeout" }
manager.disconnect()
return@PythonFuzzing PythonFeedback(control = Control.STOP)
}
Expand Down
79 changes: 51 additions & 28 deletions utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ import java.io.File

private val logger = KotlinLogging.logger {}
private const val RANDOM_TYPE_FREQUENCY = 6
private const val MAX_EXECUTIONS = 50000

class PythonTestCaseGenerator(
private val withMinimization: Boolean = true,
Expand Down Expand Up @@ -168,9 +167,7 @@ class PythonTestCaseGenerator(

var feedback: InferredTypeFeedback = SuccessFeedback

val fuzzerCancellation = {
isCancelled() || limitManager.isCancelled() || (errors.size + executions.size) >= MAX_EXECUTIONS
}
val fuzzerCancellation = { isCancelled() || limitManager.isCancelled() }

engine.fuzzing(args, fuzzerCancellation, until).collect {
when (it) {
Expand All @@ -182,8 +179,8 @@ class PythonTestCaseGenerator(
}
is InvalidExecution -> {
errors += it.utError
feedback = SuccessFeedback
limitManager.addSuccessExecution()
feedback = InvalidTypeFeedback
limitManager.addInvalidExecution()
}
is ArgumentsTypeErrorFeedback -> {
feedback = InvalidTypeFeedback
Expand All @@ -193,6 +190,19 @@ class PythonTestCaseGenerator(
feedback = InvalidTypeFeedback
limitManager.addInvalidExecution()
}
is CachedExecutionFeedback -> {
when (it.cachedFeedback) {
is ValidExecution -> {
limitManager.addSuccessExecution()
}
else -> {
limitManager.addInvalidExecution()
}
}
}
is FakeNodeFeedback -> {
limitManager.addFakeNodeExecutions()
}
}
limitManager.missedLines = missingLines?.size
}
Expand All @@ -213,28 +223,41 @@ class PythonTestCaseGenerator(
val coveredLines = mutableSetOf<Int>()

logger.info("Start test generation for ${method.name}")
val meta = method.definition.type.pythonDescription() as PythonCallableTypeDescription
val argKinds = meta.argumentKinds
if (argKinds.any { it != PythonCallableTypeDescription.ArgKind.ARG_POS }) {
val now = System.currentTimeMillis()
val firstUntil = (until - now) / 2 + now
val originalDef = method.definition
val shortType = meta.removeNonPositionalArgs(originalDef.type)
val shortMeta = PythonFuncItemDescription(
originalDef.meta.name,
originalDef.meta.args.take(shortType.arguments.size)
)
val additionalVars = originalDef.meta.args
.drop(shortType.arguments.size)
.joinToString(separator="\n", prefix="\n") { arg ->
"${arg.name}: ${pythonAnyType.pythonTypeRepresentation()}" // TODO: better types
}
method.definition = PythonFunctionDefinition(shortMeta, shortType)
val missingLines = methodHandler(method, typeStorage, coveredLines, errors, executions, null, firstUntil, additionalVars)
method.definition = originalDef
methodHandler(method, typeStorage, coveredLines, errors, executions, missingLines, until)
} else {
methodHandler(method, typeStorage, coveredLines, errors, executions, null, until)
try {
val meta = method.definition.type.pythonDescription() as PythonCallableTypeDescription
val argKinds = meta.argumentKinds
if (argKinds.any { it != PythonCallableTypeDescription.ArgKind.ARG_POS }) {
val now = System.currentTimeMillis()
val firstUntil = (until - now) / 2 + now
val originalDef = method.definition
val shortType = meta.removeNonPositionalArgs(originalDef.type)
val shortMeta = PythonFuncItemDescription(
originalDef.meta.name,
originalDef.meta.args.take(shortType.arguments.size)
)
val additionalVars = originalDef.meta.args
.drop(shortType.arguments.size)
.joinToString(separator = "\n", prefix = "\n") { arg ->
"${arg.name}: ${pythonAnyType.pythonTypeRepresentation()}" // TODO: better types
}
method.definition = PythonFunctionDefinition(shortMeta, shortType)
val missingLines = methodHandler(
method,
typeStorage,
coveredLines,
errors,
executions,
null,
firstUntil,
additionalVars
)
method.definition = originalDef
methodHandler(method, typeStorage, coveredLines, errors, executions, missingLines, until)
} else {
methodHandler(method, typeStorage, coveredLines, errors, executions, null, until)
}
} catch (_: OutOfMemoryError) {
logger.info { "Out of memory error. Stop test generation process" }
}

logger.info("Collect all test executions for ${method.name}")
Expand Down
4 changes: 2 additions & 2 deletions utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,7 @@ import org.utbot.fuzzing.utils.Trie
import org.utbot.python.framework.api.python.PythonTree
import org.utbot.python.fuzzing.provider.*
import org.utbot.python.fuzzing.provider.utils.isAny
import org.utbot.python.fuzzing.provider.utils.isConcreteType
import org.utbot.python.newtyping.*
import org.utbot.python.newtyping.general.DefaultSubstitutionProvider
import org.utbot.python.newtyping.general.Type

private val logger = KotlinLogging.logger {}
Expand All @@ -42,6 +40,8 @@ class ValidExecution(val utFuzzedExecution: UtFuzzedExecution): FuzzingExecution
class InvalidExecution(val utError: UtError): FuzzingExecutionFeedback
class TypeErrorFeedback(val message: String) : FuzzingExecutionFeedback
class ArgumentsTypeErrorFeedback(val message: String) : FuzzingExecutionFeedback
class CachedExecutionFeedback(val cachedFeedback: FuzzingExecutionFeedback) : FuzzingExecutionFeedback
object FakeNodeFeedback : FuzzingExecutionFeedback

data class PythonExecutionResult(
val fuzzingExecutionFeedback: FuzzingExecutionFeedback,
Expand Down
17 changes: 8 additions & 9 deletions utbot-python/src/main/kotlin/org/utbot/python/utils/TestGenerationLimitManager.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
package org.utbot.python.utils

import kotlin.math.min

class TestGenerationLimitManager(
// global settings
var mode: LimitManagerMode,
Expand All @@ -10,18 +8,18 @@ class TestGenerationLimitManager(
// local settings: one type inference iteration
var executions: Int = 150,
var invalidExecutions: Int = 10,
var additionalExecutions: Int = 5,
var fakeNodeExecutions: Int = 20,
var missedLines: Int? = null,
) {
private val initExecution = executions
private val initInvalidExecutions = invalidExecutions
private val initAdditionalExecutions = additionalExecutions
private val initFakeNodeExecutions = fakeNodeExecutions
private val initMissedLines = missedLines

fun restart() {
executions = initExecution
invalidExecutions = initInvalidExecutions
additionalExecutions = initAdditionalExecutions
fakeNodeExecutions = initFakeNodeExecutions
missedLines = initMissedLines
}

Expand All @@ -33,6 +31,10 @@ class TestGenerationLimitManager(
invalidExecutions -= 1
}

fun addFakeNodeExecutions() {
fakeNodeExecutions -= 1
}

fun isCancelled(): Boolean {
return mode.isCancelled(this)
}
Expand All @@ -56,10 +58,7 @@ object TimeoutMode : LimitManagerMode {

object ExecutionMode : LimitManagerMode {
override fun isCancelled(manager: TestGenerationLimitManager): Boolean {
if (manager.invalidExecutions <= 0 || manager.executions <= 0) {
return min(manager.invalidExecutions, 0) + min(manager.executions, 0) + manager.additionalExecutions <= 0
}
return false
return manager.invalidExecutions <= 0 || manager.executions <= 0 || manager.fakeNodeExecutions <= 0
}
}

Expand Down