Skip to content

Commit 8af19ca

Browse files
sofurihafesofurihafe
committed
Add fuzzer executions as standard tests in parameterized tests
1 parent 3cd8e7b commit 8af19ca

File tree

2 files changed

+112
-80
lines changed

2 files changed

+112
-80
lines changed

‎utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/CgMethodTestSet.kt

Lines changed: 61 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -32,11 +32,61 @@ data class CgMethodTestSet private constructor(
3232
executions = from.executions
3333
}
3434

35+
fun prepareTestSetsForParameterizedTestGeneration(): List<CgMethodTestSet> {
36+
val testSetList = mutableListOf<CgMethodTestSet>()
37+
38+
// Mocks are not supported in parametrized tests, so we exclude them
39+
val testSetWithoutMocking = this.excludeExecutionsWithMocking()
40+
for (splitByExecutionTestSet in testSetWithoutMocking.splitExecutionsByResult()) {
41+
for (splitByChangedStaticsTestSet in splitByExecutionTestSet.splitExecutionsByChangedStatics()) {
42+
testSetList += splitByChangedStaticsTestSet
43+
}
44+
}
45+
46+
return testSetList
47+
}
48+
49+
/**
50+
* Extract [UtFuzzedExecution] from [CgMethodTestSet].
51+
*
52+
* It is used in parameterized test generation to save them after
53+
* their excluding in order to process them later.
54+
*/
55+
fun prepareFuzzedExecutions(): CgMethodTestSet {
56+
val fuzzedExecutions = executions.filterIsInstance<UtFuzzedExecution>()
57+
58+
return substituteExecutions(fuzzedExecutions)
59+
}
60+
61+
/**
62+
* Finds a [ClassId] of all result models in executions.
63+
*
64+
* Tries to find a unique result type in testSets or
65+
* gets executable return type.
66+
*/
67+
fun resultType(): ClassId {
68+
return when (executableId.returnType) {
69+
voidClassId -> executableId.returnType
70+
else -> {
71+
val successfulExecutions = executions.filter { it.result is UtExecutionSuccess }
72+
if (successfulExecutions.isNotEmpty()) {
73+
successfulExecutions
74+
.map { (it.result as UtExecutionSuccess).model.classId }
75+
.distinct()
76+
.singleOrNull()
77+
?: executableId.returnType
78+
} else {
79+
executableId.returnType
80+
}
81+
}
82+
}
83+
}
84+
3585
/**
3686
* Splits [CgMethodTestSet] into separate test sets having
3787
* unique result model [ClassId] in each subset.
3888
*/
39-
fun splitExecutionsByResult() : List<CgMethodTestSet> {
89+
private fun splitExecutionsByResult() : List<CgMethodTestSet> {
4090
val successfulExecutions = executions.filter { it.result is UtExecutionSuccess }
4191
val failureExecutions = executions.filter { it.result is UtExecutionFailure }
4292

@@ -60,47 +110,26 @@ data class CgMethodTestSet private constructor(
60110
*
61111
* A separate test set is created for each combination of modified statics.
62112
*/
63-
fun splitExecutionsByChangedStatics(): List<CgMethodTestSet> {
113+
private fun splitExecutionsByChangedStatics(): List<CgMethodTestSet> {
64114
val executionsByStaticsUsage: Map<Set<FieldId>, List<UtExecution>> =
65115
executions.groupBy { it.stateBefore.statics.keys }
66116

67117
return executionsByStaticsUsage.map { (_, executions) -> substituteExecutions(executions) }
68118
}
69119

70-
/*
71-
* Excludes executions with mocking from [CgMethodTestSet].
72-
* */
73-
fun excludeExecutionsWithMocking(): CgMethodTestSet {
74-
val fuzzedExecutions = executions.filterIsInstance<UtFuzzedExecution>()
120+
/**
121+
* Excludes [UtFuzzedExecution] and [UtSymbolicExecution] with mocking from [CgMethodTestSet].
122+
*
123+
* It is used in parameterized test generation.
124+
* We exclude them because we cannot track force mocking occurrences in fuzzing process
125+
* and cannot deal with mocking in parameterized mode properly.
126+
*/
127+
private fun excludeExecutionsWithMocking(): CgMethodTestSet {
75128
val symbolicExecutionsWithoutMocking = executions
76129
.filterIsInstance<UtSymbolicExecution>()
77130
.filter { !it.containsMocking }
78131

79-
return substituteExecutions(symbolicExecutionsWithoutMocking + fuzzedExecutions)
80-
}
81-
82-
/**
83-
* Finds a [ClassId] of all result models in executions.
84-
*
85-
* Tries to find a unique result type in testSets or
86-
* gets executable return type.
87-
*/
88-
fun resultType(): ClassId {
89-
return when (executableId.returnType) {
90-
voidClassId -> executableId.returnType
91-
else -> {
92-
val successfulExecutions = executions.filter { it.result is UtExecutionSuccess }
93-
if (successfulExecutions.isNotEmpty()) {
94-
successfulExecutions
95-
.map { (it.result as UtExecutionSuccess).model.classId }
96-
.distinct()
97-
.singleOrNull()
98-
?: executableId.returnType
99-
} else {
100-
executableId.returnType
101-
}
102-
}
103-
}
132+
return substituteExecutions(symbolicExecutionsWithoutMocking)
104133
}
105134

106135
private fun substituteExecutions(newExecutions: List<UtExecution>): CgMethodTestSet =

‎utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/CgTestClassConstructor.kt

Lines changed: 51 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ import org.utbot.framework.codegen.model.constructor.util.CgStatementConstructor
2121
import org.utbot.framework.codegen.model.tree.CgAuxiliaryClass
2222
import org.utbot.framework.codegen.model.tree.CgExecutableUnderTestCluster
2323
import org.utbot.framework.codegen.model.tree.CgMethod
24-
import org.utbot.framework.codegen.model.tree.CgParameterDeclaration
2524
import org.utbot.framework.codegen.model.tree.CgRegion
2625
import org.utbot.framework.codegen.model.tree.CgSimpleRegion
2726
import org.utbot.framework.codegen.model.tree.CgStaticsRegion
@@ -37,7 +36,6 @@ import org.utbot.framework.codegen.model.tree.buildTestClassBody
3736
import org.utbot.framework.codegen.model.tree.buildTestClassFile
3837
import org.utbot.framework.codegen.model.visitor.importUtilMethodDependencies
3938
import org.utbot.framework.plugin.api.ClassId
40-
import org.utbot.framework.plugin.api.ExecutableId
4139
import org.utbot.framework.plugin.api.MethodId
4240
import org.utbot.framework.plugin.api.UtExecutionSuccess
4341
import org.utbot.framework.plugin.api.UtMethodTestSet
@@ -140,45 +138,18 @@ internal class CgTestClassConstructor(val context: CgContext) :
140138
.filter { it.result is UtExecutionSuccess }
141139
.map { (it.result as UtExecutionSuccess).model }
142140

143-
val (methodUnderTest, _, _, clustersInfo) = testSet
144141
val regions = mutableListOf<CgRegion<CgMethod>>()
145-
val requiredFields = mutableListOf<CgParameterDeclaration>()
146-
147-
when (context.parametrizedTestSource) {
148-
ParametrizedTestSource.DO_NOT_PARAMETRIZE -> {
149-
for ((clusterSummary, executionIndices) in clustersInfo) {
150-
val currentTestCaseTestMethods = mutableListOf<CgTestMethod>()
151-
emptyLineIfNeeded()
152-
for (i in executionIndices) {
153-
runCatching {
154-
currentTestCaseTestMethods += methodConstructor.createTestMethod(methodUnderTest, testSet.executions[i])
155-
}.onFailure { e -> processFailure(testSet, e) }
156-
}
157-
val clusterHeader = clusterSummary?.header
158-
val clusterContent = clusterSummary?.content
159-
?.split('\n')
160-
?.let { CgTripleSlashMultilineComment(it) }
161-
regions += CgTestMethodCluster(clusterHeader, clusterContent, currentTestCaseTestMethods)
162142

163-
testsGenerationReport.addTestsByType(testSet, currentTestCaseTestMethods)
164-
}
165-
}
166-
ParametrizedTestSource.PARAMETRIZE -> {
167-
// Mocks are not supported in parametrized tests, we should exclude them
168-
val testSetWithoutMocking = testSet.excludeExecutionsWithMocking()
169-
170-
for (splitByExecutionTestSet in testSetWithoutMocking.splitExecutionsByResult()) {
171-
for (splitByChangedStaticsTestSet in splitByExecutionTestSet.splitExecutionsByChangedStatics()) {
172-
createParametrizedTestAndDataProvider(
173-
splitByChangedStaticsTestSet,
174-
requiredFields,
175-
regions,
176-
methodUnderTest
177-
)
178-
}
179-
}
143+
runCatching {
144+
when (context.parametrizedTestSource) {
145+
ParametrizedTestSource.DO_NOT_PARAMETRIZE -> createTest(testSet, regions)
146+
ParametrizedTestSource.PARAMETRIZE ->
147+
createParametrizedTestAndDataProvider(
148+
testSet,
149+
regions
150+
)
180151
}
181-
}
152+
}.onFailure { e -> processFailure(testSet, e) }
182153

183154
val errors = testSet.allErrors
184155
if (errors.isNotEmpty()) {
@@ -195,29 +166,61 @@ internal class CgTestClassConstructor(val context: CgContext) :
195166
.merge(failure.description, 1, Int::plus)
196167
}
197168

169+
private fun createTest(
170+
testSet: CgMethodTestSet,
171+
regions: MutableList<CgRegion<CgMethod>>
172+
) {
173+
val (methodUnderTest, _, _, clustersInfo) = testSet
174+
175+
for ((clusterSummary, executionIndices) in clustersInfo) {
176+
val currentTestCaseTestMethods = mutableListOf<CgTestMethod>()
177+
emptyLineIfNeeded()
178+
for (i in executionIndices) {
179+
currentTestCaseTestMethods += methodConstructor.createTestMethod(methodUnderTest, testSet.executions[i])
180+
}
181+
val clusterHeader = clusterSummary?.header
182+
val clusterContent = clusterSummary?.content
183+
?.split('\n')
184+
?.let { CgTripleSlashMultilineComment(it) }
185+
regions += CgTestMethodCluster(clusterHeader, clusterContent, currentTestCaseTestMethods)
186+
187+
testsGenerationReport.addTestsByType(testSet, currentTestCaseTestMethods)
188+
}
189+
}
190+
198191
private fun createParametrizedTestAndDataProvider(
199192
testSet: CgMethodTestSet,
200-
requiredFields: MutableList<CgParameterDeclaration>,
201-
regions: MutableList<CgRegion<CgMethod>>,
202-
methodUnderTest: ExecutableId,
193+
regions: MutableList<CgRegion<CgMethod>>
203194
) {
204-
runCatching {
205-
val dataProviderMethodName = nameGenerator.dataProviderMethodNameFor(testSet.executableId)
195+
val (methodUnderTest, _, _, _) = testSet
206196

207-
val parameterizedTestMethod =
208-
methodConstructor.createParameterizedTestMethod(testSet, dataProviderMethodName)
197+
for (preparedTestSet in testSet.prepareTestSetsForParameterizedTestGeneration()) {
198+
val dataProviderMethodName = nameGenerator.dataProviderMethodNameFor(preparedTestSet.executableId)
209199

210-
requiredFields += parameterizedTestMethod.requiredFields
200+
val parameterizedTestMethod =
201+
methodConstructor.createParameterizedTestMethod(preparedTestSet, dataProviderMethodName)
211202

212203
testFrameworkManager.addDataProvider(
213-
methodConstructor.createParameterizedTestDataProvider(testSet, dataProviderMethodName)
204+
methodConstructor.createParameterizedTestDataProvider(preparedTestSet, dataProviderMethodName)
214205
)
215206

216207
regions += CgSimpleRegion(
217208
"Parameterized test for method ${methodUnderTest.humanReadableName}",
218209
listOf(parameterizedTestMethod),
219210
)
220-
}.onFailure { error -> processFailure(testSet, error) }
211+
}
212+
213+
// We cannot track mocking in fuzzed executions, so we generate standard tests for them
214+
// [https://github.com/UnitTestBot/UTBotJava/issues/1137]
215+
val testCaseTestMethods = mutableListOf<CgTestMethod>()
216+
for (execution in testSet.prepareFuzzedExecutions().executions) {
217+
testCaseTestMethods += methodConstructor.createTestMethod(methodUnderTest, execution)
218+
}
219+
220+
regions += CgSimpleRegion(
221+
"FUZZER: EXECUTIONS for method ${methodUnderTest.humanReadableName}",
222+
testCaseTestMethods,
223+
)
221224
}
222225

223226
/**

0 commit comments

Comments
 (0)