Skip to content

Commit 1cc84fd

Browse files
authored
Automatically remove uncompilable test methods (#2699)
1 parent dd5176f commit 1cc84fd

File tree

3 files changed

+71
-5
lines changed

3 files changed

+71
-5
lines changed

utbot-framework/src/main/kotlin/org/utbot/framework/codegen/renderer/CgAbstractRenderer.kt

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -251,12 +251,14 @@ abstract class CgAbstractRenderer(
251251
}
252252

253253
override fun visit(element: CgTestMethod) {
254+
visit(CgSingleLineComment(TEST_METHOD_START_MARKER))
254255
renderMethodDocumentation(element)
255256
for (annotation in element.annotations) {
256257
annotation.accept(this)
257258
}
258259
renderMethodSignature(element)
259260
visit(element as CgMethod)
261+
visit(CgSingleLineComment(TEST_METHOD_END_MARKER))
260262
}
261263

262264
override fun visit(element: CgErrorTestMethod) {
@@ -953,6 +955,9 @@ abstract class CgAbstractRenderer(
953955
private fun ClassId.isAccessibleBySimpleName(): Boolean = isAccessibleBySimpleNameImpl(this)
954956

955957
companion object {
958+
const val TEST_METHOD_START_MARKER = "test method start marker"
959+
const val TEST_METHOD_END_MARKER = "test method end marker"
960+
956961
fun makeRenderer(
957962
context: CgContext,
958963
printer: CgPrinter = CgPrinterImpl()

utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,16 @@ fun main(args: Array<String>) {
148148
expectedExceptions = ExpectedExceptionsForClass(),
149149
methodNameFilter = null
150150
)
151+
152+
val compiledClassFileDir = File(outputDir.absolutePath, "compiledClassFiles")
153+
compiledClassFileDir.mkdirs()
154+
compileClassAndRemoveUncompilableTests(
155+
testDir = compiledClassFileDir.absolutePath,
156+
classPath = classpathString,
157+
testClass = cut.generatedTestFile.absolutePath
158+
)
159+
compiledClassFileDir.deleteRecursively()
160+
151161
println("${ContestMessage.READY}")
152162
}
153163
}

utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt

Lines changed: 56 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import org.utbot.features.FeatureExtractorFactoryImpl
3232
import org.utbot.features.FeatureProcessorWithStatesRepetitionFactory
3333
import org.utbot.framework.PathSelectorType
3434
import org.utbot.framework.UtSettings
35+
import org.utbot.framework.codegen.renderer.CgAbstractRenderer
3536
import org.utbot.framework.plugin.api.util.id
3637
import org.utbot.framework.plugin.api.util.withUtContext
3738
import org.utbot.framework.plugin.services.JdkInfoService
@@ -49,7 +50,10 @@ private val javaHome = System.getenv("JAVA_HOME")
4950
private val javacCmd = "$javaHome/bin/javac"
5051
private val javaCmd = "$javaHome/bin/java"
5152

52-
private const val compileAttempts = 2
53+
// first attempt is for --add-opens
54+
// second attempt is for removing test methods that still don't compile
55+
// last attempt is for checking if final result compiles
56+
private const val compileAttempts = 3
5357

5458
private data class UnnamedPackageInfo(val pack: String, val module: String)
5559

@@ -62,7 +66,16 @@ private fun findAllNotExportedPackages(report: String): List<UnnamedPackageInfo>
6266
}.toList().distinct()
6367
}
6468

65-
private fun compileClass(testDir: String, classPath: String, testClass: String): Int {
69+
private fun findErrorLines(report: String): List<Int> {
70+
// (\d+) is line number
71+
// (:(\d+)) is optional column number
72+
val regex = """\.java:(\d+)(:(\d+))?: error""".toRegex()
73+
return regex.findAll(report).map {
74+
it.groupValues[1].toInt() - 1
75+
}.toList().distinct()
76+
}
77+
78+
fun compileClassAndRemoveUncompilableTests(testDir: String, classPath: String, testClass: String): Int = try {
6679
val exports = mutableSetOf<UnnamedPackageInfo>()
6780
var exitCode = 0
6881

@@ -93,10 +106,48 @@ private fun compileClass(testDir: String, classPath: String, testClass: String):
93106
if (errors.isNotEmpty())
94107
logger.error { "Compilation errors: $errors" }
95108
exports += findAllNotExportedPackages(errors)
109+
if (attemptNumber >= 1) {
110+
val testFile = File(testClass)
111+
val testClassLines = testFile.readLines()
112+
val testStarts = testClassLines.withIndex()
113+
.filter { it.value.contains(CgAbstractRenderer.TEST_METHOD_START_MARKER) }
114+
.map { it.index }
115+
val testEnds = testClassLines.withIndex()
116+
.filter { it.value.contains(CgAbstractRenderer.TEST_METHOD_END_MARKER) }
117+
.map { it.index }
118+
val errorLines = findErrorLines(errors)
119+
val errorRanges = errorLines.map { errorLine ->
120+
// if error is outside test method, we can't fix that by removing test methods
121+
val testStart = testStarts.filter { it <= errorLine }.maxOrNull() ?: return exitCode
122+
val testEnd = testEnds.filter { it >= errorLine }.minOrNull() ?: return exitCode
123+
testStart..testEnd
124+
}.distinct()
125+
if (errorRanges.size > testStarts.size / 5.0) {
126+
logger.error { "Over 20% of test are uncompilable" }
127+
logger.error { "Speculating that something is wrong with compilation settings, keeping all tests" }
128+
return exitCode
129+
}
130+
val linesToRemove = mutableSetOf<Int>()
131+
errorRanges.forEach { linesToRemove.addAll(it) }
132+
val removedText = testClassLines.withIndex()
133+
.filter { it.index in linesToRemove }
134+
.joinToString("\n") { "${it.index}: ${it.value}" }
135+
logger.info { "Removed uncompilable tests:\n$removedText" }
136+
testFile.writeText(testClassLines.filterIndexed { i, _ -> i !in linesToRemove }.joinToString("\n"))
137+
}
96138
}
97139
}
98140

99-
return exitCode
141+
exitCode
142+
} catch (e: Throwable) {
143+
logger.error(e) { "compileClass failed" }
144+
1
145+
} finally {
146+
val testFile = File(testClass)
147+
testFile.writeText(testFile.readLines().filter {
148+
!it.contains(CgAbstractRenderer.TEST_METHOD_START_MARKER)
149+
&& !it.contains(CgAbstractRenderer.TEST_METHOD_END_MARKER)
150+
}.joinToString("\n"))
100151
}
101152

102153
fun Array<String>.toText() = joinToString(separator = ",")
@@ -181,7 +232,7 @@ interface Tool {
181232
classStats.testClassFile = testClass
182233

183234
logger.info().measureTime({ "Compiling class ${testClass.absolutePath}" }) {
184-
val exitCode = compileClass(
235+
val exitCode = compileClassAndRemoveUncompilableTests(
185236
compiledTestDir.absolutePath,
186237
project.compileClasspathString,
187238
testClass.absolutePath
@@ -456,7 +507,7 @@ fun runEstimator(
456507
if (UtSettings.pathSelectorType == PathSelectorType.ML_SELECTOR || UtSettings.pathSelectorType == PathSelectorType.TORCH_SELECTOR) {
457508
Predictors.stateRewardPredictor = EngineAnalyticsContext.mlPredictorFactory()
458509
}
459-
510+
460511
logger.info { "PathSelectorType: ${UtSettings.pathSelectorType}" }
461512
if (UtSettings.pathSelectorType == PathSelectorType.ML_SELECTOR || UtSettings.pathSelectorType == PathSelectorType.TORCH_SELECTOR) {
462513
logger.info { "RewardModelPath: ${UtSettings.modelPath}" }

0 commit comments

Comments
 (0)