Skip to content

Add fuzzer executions as standard tests in parameterized tests #1161

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 21, 2022
Merged

Add fuzzer executions as standard tests in parameterized tests #1161

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 49 additions & 32 deletions ...ramework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/CgMethodTestSet.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,49 @@ data class CgMethodTestSet private constructor(
executions = from.executions
}

fun prepareTestSetsForParameterizedTestGeneration(): List<CgMethodTestSet> {
val testSetList = mutableListOf<CgMethodTestSet>()

// Mocks are not supported in parametrized tests, so we exclude them
val testSetWithoutMocking = this.excludeExecutionsWithMocking()
for (splitByExecutionTestSet in testSetWithoutMocking.splitExecutionsByResult()) {
for (splitByChangedStaticsTestSet in splitByExecutionTestSet.splitExecutionsByChangedStatics()) {
testSetList += splitByChangedStaticsTestSet
}
}

return testSetList
}

/**
* Finds a [ClassId] of all result models in executions.
*
* Tries to find a unique result type in testSets or
* gets executable return type.
*/
fun resultType(): ClassId {
return when (executableId.returnType) {
voidClassId -> executableId.returnType
else -> {
val successfulExecutions = executions.filter { it.result is UtExecutionSuccess }
if (successfulExecutions.isNotEmpty()) {
successfulExecutions
.map { (it.result as UtExecutionSuccess).model.classId }
.distinct()
.singleOrNull()
?: executableId.returnType
} else {
executableId.returnType
}
}
}
}

/**
* Splits [CgMethodTestSet] into separate test sets having
* unique result model [ClassId] in each subset.
*/
fun splitExecutionsByResult() : List<CgMethodTestSet> {
private fun splitExecutionsByResult() : List<CgMethodTestSet> {
val successfulExecutions = executions.filter { it.result is UtExecutionSuccess }
val failureExecutions = executions.filter { it.result is UtExecutionFailure }

Expand All @@ -60,47 +98,26 @@ data class CgMethodTestSet private constructor(
*
* A separate test set is created for each combination of modified statics.
*/
fun splitExecutionsByChangedStatics(): List<CgMethodTestSet> {
private fun splitExecutionsByChangedStatics(): List<CgMethodTestSet> {
val executionsByStaticsUsage: Map<Set<FieldId>, List<UtExecution>> =
executions.groupBy { it.stateBefore.statics.keys }

return executionsByStaticsUsage.map { (_, executions) -> substituteExecutions(executions) }
}

/*
* Excludes executions with mocking from [CgMethodTestSet].
* */
fun excludeExecutionsWithMocking(): CgMethodTestSet {
val fuzzedExecutions = executions.filterIsInstance<UtFuzzedExecution>()
/**
* Excludes [UtFuzzedExecution] and [UtSymbolicExecution] with mocking from [CgMethodTestSet].
*
* It is used in parameterized test generation.
* We exclude them because we cannot track force mocking occurrences in fuzzing process
* and cannot deal with mocking in parameterized mode properly.
*/
private fun excludeExecutionsWithMocking(): CgMethodTestSet {
val symbolicExecutionsWithoutMocking = executions
.filterIsInstance<UtSymbolicExecution>()
.filter { !it.containsMocking }

return substituteExecutions(symbolicExecutionsWithoutMocking + fuzzedExecutions)
}

/**
* Finds a [ClassId] of all result models in executions.
*
* Tries to find a unique result type in testSets or
* gets executable return type.
*/
fun resultType(): ClassId {
return when (executableId.returnType) {
voidClassId -> executableId.returnType
else -> {
val successfulExecutions = executions.filter { it.result is UtExecutionSuccess }
if (successfulExecutions.isNotEmpty()) {
successfulExecutions
.map { (it.result as UtExecutionSuccess).model.classId }
.distinct()
.singleOrNull()
?: executableId.returnType
} else {
executableId.returnType
}
}
}
return substituteExecutions(symbolicExecutionsWithoutMocking)
}

private fun substituteExecutions(newExecutions: List<UtExecution>): CgMethodTestSet =
Expand Down
101 changes: 53 additions & 48 deletions .../main/kotlin/org/utbot/framework/codegen/model/constructor/tree/CgTestClassConstructor.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import org.utbot.framework.codegen.model.constructor.util.CgStatementConstructor
import org.utbot.framework.codegen.model.tree.CgAuxiliaryClass
import org.utbot.framework.codegen.model.tree.CgExecutableUnderTestCluster
import org.utbot.framework.codegen.model.tree.CgMethod
import org.utbot.framework.codegen.model.tree.CgParameterDeclaration
import org.utbot.framework.codegen.model.tree.CgRegion
import org.utbot.framework.codegen.model.tree.CgSimpleRegion
import org.utbot.framework.codegen.model.tree.CgStaticsRegion
Expand All @@ -37,12 +36,12 @@ import org.utbot.framework.codegen.model.tree.buildTestClassBody
import org.utbot.framework.codegen.model.tree.buildTestClassFile
import org.utbot.framework.codegen.model.visitor.importUtilMethodDependencies
import org.utbot.framework.plugin.api.ClassId
import org.utbot.framework.plugin.api.ExecutableId
import org.utbot.framework.plugin.api.MethodId
import org.utbot.framework.plugin.api.UtExecutionSuccess
import org.utbot.framework.plugin.api.UtMethodTestSet
import org.utbot.framework.plugin.api.util.description
import org.utbot.framework.plugin.api.util.humanReadableName
import org.utbot.fuzzer.UtFuzzedExecution

internal class CgTestClassConstructor(val context: CgContext) :
CgContextOwner by context,
Expand Down Expand Up @@ -140,45 +139,18 @@ internal class CgTestClassConstructor(val context: CgContext) :
.filter { it.result is UtExecutionSuccess }
.map { (it.result as UtExecutionSuccess).model }

val (methodUnderTest, _, _, clustersInfo) = testSet
val regions = mutableListOf<CgRegion<CgMethod>>()
val requiredFields = mutableListOf<CgParameterDeclaration>()

when (context.parametrizedTestSource) {
ParametrizedTestSource.DO_NOT_PARAMETRIZE -> {
for ((clusterSummary, executionIndices) in clustersInfo) {
val currentTestCaseTestMethods = mutableListOf<CgTestMethod>()
emptyLineIfNeeded()
for (i in executionIndices) {
runCatching {
currentTestCaseTestMethods += methodConstructor.createTestMethod(methodUnderTest, testSet.executions[i])
}.onFailure { e -> processFailure(testSet, e) }
}
val clusterHeader = clusterSummary?.header
val clusterContent = clusterSummary?.content
?.split('\n')
?.let { CgTripleSlashMultilineComment(it) }
regions += CgTestMethodCluster(clusterHeader, clusterContent, currentTestCaseTestMethods)

testsGenerationReport.addTestsByType(testSet, currentTestCaseTestMethods)
}
}
ParametrizedTestSource.PARAMETRIZE -> {
// Mocks are not supported in parametrized tests, we should exclude them
val testSetWithoutMocking = testSet.excludeExecutionsWithMocking()

for (splitByExecutionTestSet in testSetWithoutMocking.splitExecutionsByResult()) {
for (splitByChangedStaticsTestSet in splitByExecutionTestSet.splitExecutionsByChangedStatics()) {
createParametrizedTestAndDataProvider(
splitByChangedStaticsTestSet,
requiredFields,
regions,
methodUnderTest
)
}
}
runCatching {
when (context.parametrizedTestSource) {
ParametrizedTestSource.DO_NOT_PARAMETRIZE -> createTest(testSet, regions)
ParametrizedTestSource.PARAMETRIZE ->
createParametrizedTestAndDataProvider(
testSet,
regions
)
}
}
}.onFailure { e -> processFailure(testSet, e) }

val errors = testSet.allErrors
if (errors.isNotEmpty()) {
Expand All @@ -195,29 +167,62 @@ internal class CgTestClassConstructor(val context: CgContext) :
.merge(failure.description, 1, Int::plus)
}

private fun createTest(
testSet: CgMethodTestSet,
regions: MutableList<CgRegion<CgMethod>>
) {
val (methodUnderTest, _, _, clustersInfo) = testSet

for ((clusterSummary, executionIndices) in clustersInfo) {
val currentTestCaseTestMethods = mutableListOf<CgTestMethod>()
emptyLineIfNeeded()
for (i in executionIndices) {
currentTestCaseTestMethods += methodConstructor.createTestMethod(methodUnderTest, testSet.executions[i])
}
val clusterHeader = clusterSummary?.header
val clusterContent = clusterSummary?.content
?.split('\n')
?.let { CgTripleSlashMultilineComment(it) }
regions += CgTestMethodCluster(clusterHeader, clusterContent, currentTestCaseTestMethods)

testsGenerationReport.addTestsByType(testSet, currentTestCaseTestMethods)
}
}

private fun createParametrizedTestAndDataProvider(
testSet: CgMethodTestSet,
requiredFields: MutableList<CgParameterDeclaration>,
regions: MutableList<CgRegion<CgMethod>>,
methodUnderTest: ExecutableId,
regions: MutableList<CgRegion<CgMethod>>
) {
runCatching {
val dataProviderMethodName = nameGenerator.dataProviderMethodNameFor(testSet.executableId)
val (methodUnderTest, _, _, _) = testSet

val parameterizedTestMethod =
methodConstructor.createParameterizedTestMethod(testSet, dataProviderMethodName)
for (preparedTestSet in testSet.prepareTestSetsForParameterizedTestGeneration()) {
val dataProviderMethodName = nameGenerator.dataProviderMethodNameFor(preparedTestSet.executableId)

requiredFields += parameterizedTestMethod.requiredFields
val parameterizedTestMethod =
methodConstructor.createParameterizedTestMethod(preparedTestSet, dataProviderMethodName)

testFrameworkManager.addDataProvider(
methodConstructor.createParameterizedTestDataProvider(testSet, dataProviderMethodName)
methodConstructor.createParameterizedTestDataProvider(preparedTestSet, dataProviderMethodName)
)

regions += CgSimpleRegion(
"Parameterized test for method ${methodUnderTest.humanReadableName}",
listOf(parameterizedTestMethod),
)
}.onFailure { error -> processFailure(testSet, error) }
}

// We cannot track mocking in fuzzed executions,
// so we generate standard tests for each [UtFuzzedExecution].
// [https://github.com/UnitTestBot/UTBotJava/issues/1137]
val testCaseTestMethods = mutableListOf<CgTestMethod>()
for (execution in testSet.executions.filterIsInstance<UtFuzzedExecution>()) {
testCaseTestMethods += methodConstructor.createTestMethod(methodUnderTest, execution)
}

regions += CgSimpleRegion(
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Something strange, changed the logic of clustering in the apropriate place

"FUZZER: EXECUTIONS for method ${methodUnderTest.humanReadableName}",
testCaseTestMethods,
)
}

/**
Expand Down