Skip to content

Commit 21b13af

Browse files
sofurihafesofurihafe
committed
Add fuzzer executions as standard tests in parameterized tests
1 parent 7ca87d0 commit 21b13af

File tree

2 files changed

+102
-80
lines changed

2 files changed

+102
-80
lines changed

‎utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/CgMethodTestSet.kt

Lines changed: 49 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -32,11 +32,49 @@ data class CgMethodTestSet private constructor(
3232
executions = from.executions
3333
}
3434

35+
fun prepareTestSetsForParameterizedTestGeneration(): List<CgMethodTestSet> {
36+
val testSetList = mutableListOf<CgMethodTestSet>()
37+
38+
// Mocks are not supported in parametrized tests, so we exclude them
39+
val testSetWithoutMocking = this.excludeExecutionsWithMocking()
40+
for (splitByExecutionTestSet in testSetWithoutMocking.splitExecutionsByResult()) {
41+
for (splitByChangedStaticsTestSet in splitByExecutionTestSet.splitExecutionsByChangedStatics()) {
42+
testSetList += splitByChangedStaticsTestSet
43+
}
44+
}
45+
46+
return testSetList
47+
}
48+
49+
/**
50+
* Finds a [ClassId] of all result models in executions.
51+
*
52+
* Tries to find a unique result type in testSets or
53+
* gets executable return type.
54+
*/
55+
fun resultType(): ClassId {
56+
return when (executableId.returnType) {
57+
voidClassId -> executableId.returnType
58+
else -> {
59+
val successfulExecutions = executions.filter { it.result is UtExecutionSuccess }
60+
if (successfulExecutions.isNotEmpty()) {
61+
successfulExecutions
62+
.map { (it.result as UtExecutionSuccess).model.classId }
63+
.distinct()
64+
.singleOrNull()
65+
?: executableId.returnType
66+
} else {
67+
executableId.returnType
68+
}
69+
}
70+
}
71+
}
72+
3573
/**
3674
* Splits [CgMethodTestSet] into separate test sets having
3775
* unique result model [ClassId] in each subset.
3876
*/
39-
fun splitExecutionsByResult() : List<CgMethodTestSet> {
77+
private fun splitExecutionsByResult() : List<CgMethodTestSet> {
4078
val successfulExecutions = executions.filter { it.result is UtExecutionSuccess }
4179
val failureExecutions = executions.filter { it.result is UtExecutionFailure }
4280

@@ -60,47 +98,26 @@ data class CgMethodTestSet private constructor(
6098
*
6199
* A separate test set is created for each combination of modified statics.
62100
*/
63-
fun splitExecutionsByChangedStatics(): List<CgMethodTestSet> {
101+
private fun splitExecutionsByChangedStatics(): List<CgMethodTestSet> {
64102
val executionsByStaticsUsage: Map<Set<FieldId>, List<UtExecution>> =
65103
executions.groupBy { it.stateBefore.statics.keys }
66104

67105
return executionsByStaticsUsage.map { (_, executions) -> substituteExecutions(executions) }
68106
}
69107

70-
/*
71-
* Excludes executions with mocking from [CgMethodTestSet].
72-
* */
73-
fun excludeExecutionsWithMocking(): CgMethodTestSet {
74-
val fuzzedExecutions = executions.filterIsInstance<UtFuzzedExecution>()
108+
/**
109+
* Excludes [UtFuzzedExecution] and [UtSymbolicExecution] with mocking from [CgMethodTestSet].
110+
*
111+
* It is used in parameterized test generation.
112+
* We exclude them because we cannot track force mocking occurrences in fuzzing process
113+
* and cannot deal with mocking in parameterized mode properly.
114+
*/
115+
private fun excludeExecutionsWithMocking(): CgMethodTestSet {
75116
val symbolicExecutionsWithoutMocking = executions
76117
.filterIsInstance<UtSymbolicExecution>()
77118
.filter { !it.containsMocking }
78119

79-
return substituteExecutions(symbolicExecutionsWithoutMocking + fuzzedExecutions)
80-
}
81-
82-
/**
83-
* Finds a [ClassId] of all result models in executions.
84-
*
85-
* Tries to find a unique result type in testSets or
86-
* gets executable return type.
87-
*/
88-
fun resultType(): ClassId {
89-
return when (executableId.returnType) {
90-
voidClassId -> executableId.returnType
91-
else -> {
92-
val successfulExecutions = executions.filter { it.result is UtExecutionSuccess }
93-
if (successfulExecutions.isNotEmpty()) {
94-
successfulExecutions
95-
.map { (it.result as UtExecutionSuccess).model.classId }
96-
.distinct()
97-
.singleOrNull()
98-
?: executableId.returnType
99-
} else {
100-
executableId.returnType
101-
}
102-
}
103-
}
120+
return substituteExecutions(symbolicExecutionsWithoutMocking)
104121
}
105122

106123
private fun substituteExecutions(newExecutions: List<UtExecution>): CgMethodTestSet =

‎utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/CgTestClassConstructor.kt

Lines changed: 53 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ import org.utbot.framework.codegen.model.constructor.util.CgStatementConstructor
2121
import org.utbot.framework.codegen.model.tree.CgAuxiliaryClass
2222
import org.utbot.framework.codegen.model.tree.CgExecutableUnderTestCluster
2323
import org.utbot.framework.codegen.model.tree.CgMethod
24-
import org.utbot.framework.codegen.model.tree.CgParameterDeclaration
2524
import org.utbot.framework.codegen.model.tree.CgRegion
2625
import org.utbot.framework.codegen.model.tree.CgSimpleRegion
2726
import org.utbot.framework.codegen.model.tree.CgStaticsRegion
@@ -37,12 +36,12 @@ import org.utbot.framework.codegen.model.tree.buildTestClassBody
3736
import org.utbot.framework.codegen.model.tree.buildTestClassFile
3837
import org.utbot.framework.codegen.model.visitor.importUtilMethodDependencies
3938
import org.utbot.framework.plugin.api.ClassId
40-
import org.utbot.framework.plugin.api.ExecutableId
4139
import org.utbot.framework.plugin.api.MethodId
4240
import org.utbot.framework.plugin.api.UtExecutionSuccess
4341
import org.utbot.framework.plugin.api.UtMethodTestSet
4442
import org.utbot.framework.plugin.api.util.description
4543
import org.utbot.framework.plugin.api.util.humanReadableName
44+
import org.utbot.fuzzer.UtFuzzedExecution
4645

4746
internal class CgTestClassConstructor(val context: CgContext) :
4847
CgContextOwner by context,
@@ -140,45 +139,18 @@ internal class CgTestClassConstructor(val context: CgContext) :
140139
.filter { it.result is UtExecutionSuccess }
141140
.map { (it.result as UtExecutionSuccess).model }
142141

143-
val (methodUnderTest, _, _, clustersInfo) = testSet
144142
val regions = mutableListOf<CgRegion<CgMethod>>()
145-
val requiredFields = mutableListOf<CgParameterDeclaration>()
146-
147-
when (context.parametrizedTestSource) {
148-
ParametrizedTestSource.DO_NOT_PARAMETRIZE -> {
149-
for ((clusterSummary, executionIndices) in clustersInfo) {
150-
val currentTestCaseTestMethods = mutableListOf<CgTestMethod>()
151-
emptyLineIfNeeded()
152-
for (i in executionIndices) {
153-
runCatching {
154-
currentTestCaseTestMethods += methodConstructor.createTestMethod(methodUnderTest, testSet.executions[i])
155-
}.onFailure { e -> processFailure(testSet, e) }
156-
}
157-
val clusterHeader = clusterSummary?.header
158-
val clusterContent = clusterSummary?.content
159-
?.split('\n')
160-
?.let { CgTripleSlashMultilineComment(it) }
161-
regions += CgTestMethodCluster(clusterHeader, clusterContent, currentTestCaseTestMethods)
162143

163-
testsGenerationReport.addTestsByType(testSet, currentTestCaseTestMethods)
164-
}
165-
}
166-
ParametrizedTestSource.PARAMETRIZE -> {
167-
// Mocks are not supported in parametrized tests, we should exclude them
168-
val testSetWithoutMocking = testSet.excludeExecutionsWithMocking()
169-
170-
for (splitByExecutionTestSet in testSetWithoutMocking.splitExecutionsByResult()) {
171-
for (splitByChangedStaticsTestSet in splitByExecutionTestSet.splitExecutionsByChangedStatics()) {
172-
createParametrizedTestAndDataProvider(
173-
splitByChangedStaticsTestSet,
174-
requiredFields,
175-
regions,
176-
methodUnderTest
177-
)
178-
}
179-
}
144+
runCatching {
145+
when (context.parametrizedTestSource) {
146+
ParametrizedTestSource.DO_NOT_PARAMETRIZE -> createTest(testSet, regions)
147+
ParametrizedTestSource.PARAMETRIZE ->
148+
createParametrizedTestAndDataProvider(
149+
testSet,
150+
regions
151+
)
180152
}
181-
}
153+
}.onFailure { e -> processFailure(testSet, e) }
182154

183155
val errors = testSet.allErrors
184156
if (errors.isNotEmpty()) {
@@ -195,29 +167,62 @@ internal class CgTestClassConstructor(val context: CgContext) :
195167
.merge(failure.description, 1, Int::plus)
196168
}
197169

170+
private fun createTest(
171+
testSet: CgMethodTestSet,
172+
regions: MutableList<CgRegion<CgMethod>>
173+
) {
174+
val (methodUnderTest, _, _, clustersInfo) = testSet
175+
176+
for ((clusterSummary, executionIndices) in clustersInfo) {
177+
val currentTestCaseTestMethods = mutableListOf<CgTestMethod>()
178+
emptyLineIfNeeded()
179+
for (i in executionIndices) {
180+
currentTestCaseTestMethods += methodConstructor.createTestMethod(methodUnderTest, testSet.executions[i])
181+
}
182+
val clusterHeader = clusterSummary?.header
183+
val clusterContent = clusterSummary?.content
184+
?.split('\n')
185+
?.let { CgTripleSlashMultilineComment(it) }
186+
regions += CgTestMethodCluster(clusterHeader, clusterContent, currentTestCaseTestMethods)
187+
188+
testsGenerationReport.addTestsByType(testSet, currentTestCaseTestMethods)
189+
}
190+
}
191+
198192
private fun createParametrizedTestAndDataProvider(
199193
testSet: CgMethodTestSet,
200-
requiredFields: MutableList<CgParameterDeclaration>,
201-
regions: MutableList<CgRegion<CgMethod>>,
202-
methodUnderTest: ExecutableId,
194+
regions: MutableList<CgRegion<CgMethod>>
203195
) {
204-
runCatching {
205-
val dataProviderMethodName = nameGenerator.dataProviderMethodNameFor(testSet.executableId)
196+
val (methodUnderTest, _, _, _) = testSet
206197

207-
val parameterizedTestMethod =
208-
methodConstructor.createParameterizedTestMethod(testSet, dataProviderMethodName)
198+
for (preparedTestSet in testSet.prepareTestSetsForParameterizedTestGeneration()) {
199+
val dataProviderMethodName = nameGenerator.dataProviderMethodNameFor(preparedTestSet.executableId)
209200

210-
requiredFields += parameterizedTestMethod.requiredFields
201+
val parameterizedTestMethod =
202+
methodConstructor.createParameterizedTestMethod(preparedTestSet, dataProviderMethodName)
211203

212204
testFrameworkManager.addDataProvider(
213-
methodConstructor.createParameterizedTestDataProvider(testSet, dataProviderMethodName)
205+
methodConstructor.createParameterizedTestDataProvider(preparedTestSet, dataProviderMethodName)
214206
)
215207

216208
regions += CgSimpleRegion(
217209
"Parameterized test for method ${methodUnderTest.humanReadableName}",
218210
listOf(parameterizedTestMethod),
219211
)
220-
}.onFailure { error -> processFailure(testSet, error) }
212+
}
213+
214+
// We cannot track mocking in fuzzed executions,
215+
// so we generate standard tests for each [UtFuzzedExecution].
216+
// [https://github.com/UnitTestBot/UTBotJava/issues/1137]
217+
val testCaseTestMethods = mutableListOf<CgTestMethod>()
218+
for (execution in testSet.executions.filterIsInstance<UtFuzzedExecution>()) {
219+
testCaseTestMethods += methodConstructor.createTestMethod(methodUnderTest, execution)
220+
}
221+
222+
regions += CgSimpleRegion(
223+
"FUZZER: EXECUTIONS for method ${methodUnderTest.humanReadableName}",
224+
testCaseTestMethods,
225+
)
221226
}
222227

223228
/**

0 commit comments

Comments
 (0)