Skip to content

Fix python fuzzing strategy #1914

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Mar 9, 2023
Merged

Fix python fuzzing strategy #1914

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
1669370
Revert fuzzing minimization
tamarinvs19 Mar 9, 2023
743061b
Fix limit manager bug
tamarinvs19 Mar 9, 2023
1b7affe
Fix logging in python fuzzing and remove unused code
tamarinvs19 Mar 9, 2023
6a09016
Merge branch 'main' into tamarinvs19/python-fix-timeout-management
tamarinvs19 Mar 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
157 changes: 62 additions & 95 deletions utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ import org.utbot.python.newtyping.PythonTypeStorage
import org.utbot.python.newtyping.general.Type
import org.utbot.python.newtyping.pythonModules
import org.utbot.python.newtyping.pythonTypeRepresentation
import org.utbot.python.utils.TestGenerationLimitManager
import org.utbot.python.utils.camelToSnakeCase
import org.utbot.summary.fuzzer.names.TestSuggestedInfo
import java.net.ServerSocket
Expand Down Expand Up @@ -143,30 +142,30 @@ class PythonEngine(
)
}

fun fuzzing(parameters: List<Type>, isCancelled: () -> Boolean, limitManager: TestGenerationLimitManager): Flow<FuzzingExecutionFeedback> = flow {
fun fuzzing(parameters: List<Type>, isCancelled: () -> Boolean, until: Long): Flow<FuzzingExecutionFeedback> = flow {
val additionalModules = parameters.flatMap { it.pythonModules() }

ServerSocket(0).use { serverSocket ->
logger.debug { "Server port: ${serverSocket.localPort}" }
logger.info { "Server port: ${serverSocket.localPort}" }
val manager = try {
PythonWorkerManager(
serverSocket,
pythonPath,
limitManager.until,
until,
{ constructEvaluationInput(it) },
timeoutForRun.toInt()
)
} catch (_: TimeoutException) {
logger.info { "Cannot connect to python executor" }
return@flow
}
logger.info { "Executor manager was created successfully" }

fun runWithFuzzedValues(
arguments: List<PythonFuzzedValue>,
): PythonEvaluationResult? {
fun fuzzingResultHandler(
description: PythonMethodDescription,
arguments: List<PythonFuzzedValue>
): PythonExecutionResult? {
val argumentValues = arguments.map { PythonTreeModel(it.tree, it.tree.type) }
logger.debug(argumentValues.map { it.tree }.toString())
logger.debug(argumentValues.map { it.tree } .toString())
val argumentModules = argumentValues
.flatMap { it.allContainingClassIds }
.map { it.moduleName }
Expand All @@ -184,83 +183,52 @@ class PythonEngine(
modelList,
methodUnderTest.argumentsNames
)
return try {
manager.run(functionArguments, localAdditionalModules)
} catch (_: TimeoutException) {
logger.info { "Fuzzing process was interrupted by timeout" }
null
}
}

fun handleExecutionResult(
result: PythonEvaluationResult,
arguments: List<PythonFuzzedValue>,
description: PythonMethodDescription,
): Pair<PythonExecutionResult, Boolean> {
val executionFeedback: FuzzingExecutionFeedback
val fuzzingFeedback: PythonFeedback

when(result) {
is PythonEvaluationError -> {
val utError = UtError(
"Error evaluation: ${result.status}, ${result.message}",
Throwable(result.stackTrace.joinToString("\n"))
)
logger.debug(result.stackTrace.joinToString("\n"))

limitManager.addSuccessExecution()
executionFeedback = InvalidExecution(utError)
fuzzingFeedback = PythonFeedback(control = Control.PASS)
return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), true)
}

is PythonEvaluationTimeout -> {
val utError = UtError(result.message, Throwable())
limitManager.addInvalidExecution()
executionFeedback = InvalidExecution(utError)
fuzzingFeedback = PythonFeedback(control = Control.PASS)
return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false)
}
try {
return when (val evaluationResult = manager.run(functionArguments, localAdditionalModules)) {
is PythonEvaluationError -> {
val utError = UtError(
"Error evaluation: ${evaluationResult.status}, ${evaluationResult.message}",
Throwable(evaluationResult.stackTrace.joinToString("\n"))
)
logger.debug(evaluationResult.stackTrace.joinToString("\n"))
PythonExecutionResult(InvalidExecution(utError), PythonFeedback(control = Control.PASS))
}

is PythonEvaluationSuccess -> {
val coveredInstructions = result.coverage.coveredInstructions
executionFeedback = handleSuccessResult(
arguments,
parameters,
result,
description,
)
is PythonEvaluationTimeout -> {
val utError = UtError(evaluationResult.message, Throwable())
PythonExecutionResult(InvalidExecution(utError), PythonFeedback(control = Control.PASS))
}

val trieNode: Trie.Node<Instruction> = description.tracer.add(coveredInstructions)
when (executionFeedback) {
is ValidExecution -> {
limitManager.addSuccessExecution()
if (trieNode.count > 1) {
fuzzingFeedback = PythonFeedback(control = Control.CONTINUE, result = trieNode)
return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false)
is PythonEvaluationSuccess -> {
val coveredInstructions = evaluationResult.coverage.coveredInstructions

when (val result = handleSuccessResult(
arguments,
parameters,
evaluationResult,
description,
)) {
is ValidExecution -> {
val trieNode: Trie.Node<Instruction> = description.tracer.add(coveredInstructions)
PythonExecutionResult(
result,
PythonFeedback(control = Control.CONTINUE, result = trieNode)
)
}
}

is ArgumentsTypeErrorFeedback -> {
fuzzingFeedback = PythonFeedback(control = Control.PASS)
return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false)
}

is TypeErrorFeedback -> {
limitManager.addInvalidExecution()
fuzzingFeedback = PythonFeedback(control = Control.PASS)
return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false)
}
is ArgumentsTypeErrorFeedback, is TypeErrorFeedback -> {
PythonExecutionResult(result, PythonFeedback(control = Control.PASS))
}

is InvalidExecution -> {
limitManager.addInvalidExecution()
fuzzingFeedback = PythonFeedback(control = Control.CONTINUE)
return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), false)
is InvalidExecution -> {
PythonExecutionResult(result, PythonFeedback(control = Control.CONTINUE))
}
}
}
fuzzingFeedback = PythonFeedback(control = Control.CONTINUE, result = trieNode)
return Pair(PythonExecutionResult(executionFeedback, fuzzingFeedback), true)
}
} catch (_: TimeoutException) {
logger.info { "Fuzzing process was interrupted by timeout" }
return null
}
}

Expand All @@ -273,14 +241,10 @@ class PythonEngine(
)

if (parameters.isEmpty()) {
val result = runWithFuzzedValues(emptyList())
val result = fuzzingResultHandler(pmd, emptyList())
result?.let {
val (executionResult, needToEmit) = handleExecutionResult(result, emptyList(), pmd)
if (needToEmit) {
emit(executionResult.fuzzingExecutionFeedback)
}
emit(it.fuzzingExecutionFeedback)
}
manager.disconnect()
} else {
try {
PythonFuzzing(pmd.pythonTypeStorage) { description, arguments ->
Expand All @@ -289,32 +253,35 @@ class PythonEngine(
manager.disconnect()
return@PythonFuzzing PythonFeedback(control = Control.STOP)
}
if (System.currentTimeMillis() >= until) {
logger.info { "Fuzzing process was interrupted by timeout" }
manager.disconnect()
return@PythonFuzzing PythonFeedback(control = Control.STOP)
}

val pair = Pair(description, arguments.map { PythonTreeWrapper(it.tree) })
val mem = cache.get(pair)
if (mem != null) {
logger.debug("Repeat in fuzzing")
emit(mem.fuzzingExecutionFeedback)
return@PythonFuzzing mem.fuzzingPlatformFeedback
}

val result = runWithFuzzedValues(arguments)
val result = fuzzingResultHandler(description, arguments)
if (result == null) { // timeout
logger.info { "Fuzzing process was interrupted by timeout" }
manager.disconnect()
return@PythonFuzzing PythonFeedback(control = Control.STOP)
}

val (executionResult, needToEmit) = handleExecutionResult(result, arguments, description)
cache.add(pair, executionResult)
if (needToEmit) {
emit(executionResult.fuzzingExecutionFeedback)
}
return@PythonFuzzing executionResult.fuzzingPlatformFeedback
cache.add(pair, result)
emit(result.fuzzingExecutionFeedback)
return@PythonFuzzing result.fuzzingPlatformFeedback
}.fuzz(pmd)
} catch (ex: Exception) { // NoSeedValueException
} catch (_: Exception) { // e.g. NoSeedValueException
logger.info { "Cannot fuzz values for types: $parameters" }
}
manager.disconnect()
}
manager.disconnect()
}
}
}
}
6 changes: 5 additions & 1 deletion utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,23 +167,27 @@ class PythonTestCaseGenerator(

val fuzzerCancellation = { isCancelled() || limitManager.isCancelled() }

engine.fuzzing(args, fuzzerCancellation, limitManager).collect {
engine.fuzzing(args, fuzzerCancellation, until).collect {
generated += 1
when (it) {
is ValidExecution -> {
executions += it.utFuzzedExecution
missingLines = updateCoverage(it.utFuzzedExecution, coveredLines, missingLines)
feedback = SuccessFeedback
limitManager.addSuccessExecution()
}
is InvalidExecution -> {
errors += it.utError
feedback = SuccessFeedback
limitManager.addSuccessExecution()
}
is ArgumentsTypeErrorFeedback -> {
feedback = InvalidTypeFeedback
limitManager.addInvalidExecution()
}
is TypeErrorFeedback -> {
feedback = InvalidTypeFeedback
limitManager.addInvalidExecution()
}
}
limitManager.missedLines = missingLines?.size
Expand Down
40 changes: 9 additions & 31 deletions utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@ import mu.KotlinLogging
import org.utbot.framework.plugin.api.Instruction
import org.utbot.framework.plugin.api.UtError
import org.utbot.fuzzer.FuzzedContext
import org.utbot.fuzzer.IdGenerator
import org.utbot.fuzzer.IdentityPreservingIdGenerator
import org.utbot.fuzzer.UtFuzzedExecution
import org.utbot.fuzzing.Configuration
import org.utbot.fuzzing.Control
Expand All @@ -23,8 +21,6 @@ import org.utbot.python.newtyping.PythonSubtypeChecker
import org.utbot.python.newtyping.PythonTypeStorage
import org.utbot.python.newtyping.general.Type
import org.utbot.python.newtyping.pythonTypeRepresentation
import java.util.*
import java.util.concurrent.atomic.AtomicLong

private val logger = KotlinLogging.logger {}

Expand Down Expand Up @@ -63,7 +59,7 @@ class PythonFuzzedValue(
val summary: String? = null,
)

fun pythonDefaultValueProviders(idGenerator: IdGenerator<Long>) = listOf(
fun pythonDefaultValueProviders() = listOf(
NoneValueProvider,
BoolValueProvider,
IntValueProvider,
Expand All @@ -88,39 +84,38 @@ class PythonFuzzing(
val execute: suspend (description: PythonMethodDescription, values: List<PythonFuzzedValue>) -> PythonFeedback,
) : Fuzzing<Type, PythonFuzzedValue, PythonMethodDescription, PythonFeedback> {

private fun generateDefault(description: PythonMethodDescription, type: Type, idGenerator: IdGenerator<Long>): Sequence<Seed<Type, PythonFuzzedValue>> {
private fun generateDefault(description: PythonMethodDescription, type: Type): Sequence<Seed<Type, PythonFuzzedValue>> {
var providers = emptyList<Seed<Type, PythonFuzzedValue>>().asSequence()
pythonDefaultValueProviders(idGenerator).asSequence().forEach { provider ->
pythonDefaultValueProviders().asSequence().forEach { provider ->
if (provider.accept(type)) {
logger.info { "Provider ${provider.javaClass.simpleName} accepts type ${type.pythonTypeRepresentation()}" }
logger.debug { "Provider ${provider.javaClass.simpleName} accepts type ${type.pythonTypeRepresentation()}" }
providers += provider.generate(description, type)
}
}
return providers
}

private fun generateSubtype(description: PythonMethodDescription, type: Type, idGenerator: IdGenerator<Long>): Sequence<Seed<Type, PythonFuzzedValue>> {
private fun generateSubtype(description: PythonMethodDescription, type: Type): Sequence<Seed<Type, PythonFuzzedValue>> {
var providers = emptyList<Seed<Type, PythonFuzzedValue>>().asSequence()
if (type.meta is PythonProtocolDescription) {
val subtypes = pythonTypeStorage.allTypes.filter {
PythonSubtypeChecker.checkIfRightIsSubtypeOfLeft(type, it, pythonTypeStorage)
}
subtypes.forEach {
providers += generateDefault(description, it, idGenerator)
providers += generateDefault(description, it)
}
}
return providers
}

override fun generate(description: PythonMethodDescription, type: Type): Sequence<Seed<Type, PythonFuzzedValue>> {
val idGenerator = PythonIdGenerator()
var providers = emptyList<Seed<Type, PythonFuzzedValue>>().asSequence()

if (type.isAny()) {
logger.info("Any does not have provider")
logger.debug("Any does not have provider")
} else {
providers += generateDefault(description, type, idGenerator)
providers += generateSubtype(description, type, idGenerator)
providers += generateDefault(description, type)
providers += generateSubtype(description, type)
}

return providers
Expand All @@ -138,20 +133,3 @@ class PythonFuzzing(
super.update(description, statistic, configuration)
}
}

class PythonIdGenerator(lowerBound: Long = DEFAULT_LOWER_BOUND) : IdentityPreservingIdGenerator<Long> {
private val lastId: AtomicLong = AtomicLong(lowerBound)
private val cache: IdentityHashMap<Any?, Long> = IdentityHashMap()

override fun getOrCreateIdForValue(value: Any): Long {
return cache.getOrPut(value) { createId() }
}

override fun createId(): Long {
return lastId.incrementAndGet()
}

companion object {
const val DEFAULT_LOWER_BOUND: Long = 1500_000_000
}
}
2 changes: 1 addition & 1 deletion utbot-python/src/main/kotlin/org/utbot/python/utils/TestGenerationLimitManager.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ object TimeoutMode : LimitManagerMode {
object ExecutionMode : LimitManagerMode {
override fun isCancelled(manager: TestGenerationLimitManager): Boolean {
if (manager.invalidExecutions <= 0 || manager.executions <= 0) {
return min(manager.invalidExecutions, 0) + min(manager.executions, 0) <= manager.additionalExecutions
return min(manager.invalidExecutions, 0) + min(manager.executions, 0) + manager.additionalExecutions <= 0
}
return false
}
Expand Down