diff --git a/utbot-junit-contest/README.md b/utbot-junit-contest/README.md index 6dbdf99013..63717d7d24 100644 --- a/utbot-junit-contest/README.md +++ b/utbot-junit-contest/README.md @@ -28,6 +28,9 @@ The projects are provided to Contest estimator in advance. All available projects are placed in the [resources][resources] folder, which contains: - [projects][projects] consisting of the folders with the project JAR files in them. - [classes][classes] consisting of the folders — each named after the project and containing the `list` file with the fully qualified class names. +It also may contain an `exceptions` file with the description of the expected exceptions, that utbot should find. +Description is presented in the format: `.: ...`. +For example, see this [file](src/main/resources/classes/codeforces/exceptions). ### How to add a new project You should add both the JAR files to the `projects` folder and the file with a list of classes to the `classes` folder. diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt index c41a617309..cba7acaef4 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt @@ -14,14 +14,6 @@ import org.utbot.framework.codegen.domain.ForceStaticMocking import org.utbot.framework.codegen.domain.StaticsMocking import org.utbot.framework.codegen.domain.junitByVersion import org.utbot.framework.codegen.CodeGenerator -import org.utbot.framework.plugin.api.CodegenLanguage -import org.utbot.framework.plugin.api.Coverage -import org.utbot.framework.plugin.api.ExecutableId -import org.utbot.framework.plugin.api.MockStrategyApi -import org.utbot.framework.plugin.api.TestCaseGenerator -import org.utbot.framework.plugin.api.UtError -import org.utbot.framework.plugin.api.UtExecution -import org.utbot.framework.plugin.api.UtMethodTestSet import org.utbot.framework.plugin.api.util.UtContext import org.utbot.framework.plugin.api.util.executableId import org.utbot.framework.plugin.api.util.id @@ -59,10 +51,10 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.newSingleThreadContext import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withTimeoutOrNull -import kotlinx.coroutines.yield import org.utbot.framework.SummariesGenerationType import org.utbot.framework.codegen.services.language.CgLanguageAssistant import org.utbot.framework.minimization.minimizeExecutions +import org.utbot.framework.plugin.api.* import org.utbot.framework.plugin.api.util.isSynthetic import org.utbot.framework.util.jimpleBody import org.utbot.summary.summarize @@ -154,6 +146,7 @@ fun main(args: Array) { fuzzingRatio = 0.1, classpathString, runFromEstimator = false, + expectedExceptions = ExpectedExceptionsForClass(), methodNameFilter = null ) println("${ContestMessage.READY}") @@ -187,6 +180,7 @@ fun runGeneration( fuzzingRatio: Double, classpathString: String, runFromEstimator: Boolean, + expectedExceptions: ExpectedExceptionsForClass, methodNameFilter: String? = null // For debug purposes you can specify method name ): StatsForClass = runBlocking { val testsByMethod: MutableMap> = mutableMapOf() @@ -266,7 +260,10 @@ fun runGeneration( val methodJob = currentCoroutineContext().job logger.debug { " ... " } - val statsForMethod = StatsForMethod("${method.classId.simpleName}#${method.name}") + val statsForMethod = StatsForMethod( + "${method.classId.simpleName}#${method.name}", + expectedExceptions.getForMethod(method.name).exceptionNames + ) statsForClass.statsForMethods.add(statsForMethod) @@ -337,6 +334,9 @@ fun runGeneration( val className = Type.getInternalName(method.classId.jClass) logger.debug { "--new testCase collected, to generate: $testMethodName" } statsForMethod.testsGeneratedCount++ + result.result.exceptionOrNull()?.let { exception -> + statsForMethod.detectedExceptionFqns += exception::class.java.name + } result.coverage?.let { statsForClass.updateCoverage( newCoverage = it, diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt index a9fe1a25d7..199bae753b 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt @@ -137,7 +137,8 @@ enum class Tool { methodNameFilter: String?, statsForProject: StatsForProject, compiledTestDir: File, - classFqn: String + classFqn: String, + expectedExceptions: ExpectedExceptionsForClass ) = withUtContext(ContextManager.createNewContext(project.classloader)) { val classStats: StatsForClass = try { runGeneration( @@ -147,6 +148,7 @@ enum class Tool { fuzzingRatio, project.sootClasspathString, runFromEstimator = true, + expectedExceptions, methodNameFilter ) } catch (e: CancellationException) { @@ -206,7 +208,8 @@ enum class Tool { methodNameFilter: String?, statsForProject: StatsForProject, compiledTestDir: File, - classFqn: String + classFqn: String, + expectedExceptions: ExpectedExceptionsForClass ) { // EvoSuite has several phases, the variable below is responsible for assert generation // timeout. We want to give 10s for a big time budgets and timeLimit / 5 for small budgets. @@ -277,7 +280,8 @@ enum class Tool { methodNameFilter: String?, statsForProject: StatsForProject, compiledTestDir: File, - classFqn: String + classFqn: String, + expectedExceptions: ExpectedExceptionsForClass ) abstract fun moveProducedFilesIfNeeded() @@ -391,7 +395,13 @@ fun runEstimator( if (updatedMethodFilter != null) logger.info { "Filtering: class='$classFqnFilter', method ='$methodNameFilter'" } - val projectToClassFQNs = classesLists.listFiles()!!.associate { it.name to File(it, "list").readLines() } + val projectDirs = classesLists.listFiles()!! + val projectToClassFQNs = projectDirs.associate { + it.name to File(it, "list").readLines() + } + val projectToExpectedExceptions = projectDirs.associate { + it.name to parseExceptionsFile(File(it, "exceptions")) + } val projects = mutableListOf() @@ -400,6 +410,7 @@ fun runEstimator( val project = ProjectToEstimate( name, classesFQN, + projectToExpectedExceptions.getValue(name), File(classpathDir, name).listFiles()!!.filter { it.toString().endsWith("jar") }, testCandidatesDir, unzippedJars @@ -463,7 +474,8 @@ fun runEstimator( methodNameFilter, statsForProject, compiledTestDir, - classFqn + classFqn, + project.expectedExceptions.getForClass(classFqn) ) } catch (e: Throwable) { @@ -526,6 +538,7 @@ private fun classNamesByJar(jar: File): List { class ProjectToEstimate( val name: String, val classFQNs: List, + val expectedExceptions: ExpectedExceptionsForProject, private val jars: List, testCandidatesDir: File, unzippedJars: File diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ExpectedExceptions.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ExpectedExceptions.kt new file mode 100644 index 0000000000..ebf41a05a5 --- /dev/null +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ExpectedExceptions.kt @@ -0,0 +1,54 @@ +package org.utbot.contest + +import java.io.File + +class ExpectedExceptionsForMethod( + val exceptionNames: List +) + +class ExpectedExceptionsForClass { + + private val byMethodName: MutableMap = + mutableMapOf() + + fun getForMethod(methodName: String): ExpectedExceptionsForMethod = + byMethodName[methodName] ?: ExpectedExceptionsForMethod(listOf()) + + fun setForMethod(methodName: String, exceptionNames: List) { + byMethodName[methodName] = ExpectedExceptionsForMethod(exceptionNames) + } +} + +class ExpectedExceptionsForProject { + + private val byClassName: MutableMap = + mutableMapOf() + + fun getForClass(className: String): ExpectedExceptionsForClass = + byClassName[className] ?: ExpectedExceptionsForClass() + + fun setForClass(className: String, methodName: String, exceptionNames: List) { + val forClass = byClassName.getOrPut(className) { ExpectedExceptionsForClass() } + forClass.setForMethod(methodName, exceptionNames) + } +} + +fun parseExceptionsFile(exceptionsDescriptionFile: File): ExpectedExceptionsForProject { + if (!exceptionsDescriptionFile.exists()) { + return ExpectedExceptionsForProject() + } + + val forProject = ExpectedExceptionsForProject() + for (methodDescription in exceptionsDescriptionFile.readLines()) { + val methodFqn = methodDescription.substringBefore(':').trim() + val classFqn = methodFqn.substringBeforeLast('.') + val methodName = methodFqn.substringAfterLast('.') + val exceptionFqns = methodDescription.substringAfter(':') + .split(' ') + .map { it.trim() } + .filter { it.isNotBlank() } + forProject.setForClass(classFqn, methodName, exceptionFqns) + } + + return forProject +} diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Statistics.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Statistics.kt index 16918a8e4c..049947a6db 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Statistics.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Statistics.kt @@ -75,9 +75,15 @@ class StatsForProject(val project: String) { else this } + val detectedExceptionsCount: Int + get() = statsForClasses.sumOf { it.detectedExceptionsCount } + val expectedExceptionsCount: Int + get() = statsForClasses.sumOf { it.expectedExceptionsCount } + override fun toString(): String = "\n :" + "\n\t#classes for generation = $classesForGeneration" + "\n\t#tc generated = $testCasesGenerated" + + (if (expectedExceptionsCount > 0) "\n\t#detected exceptions = $detectedExceptionsCount/$expectedExceptionsCount" else "") + "\n\t#classes without problems = $classesWithoutProblems" + "\n\t#classes canceled by timeout = $classesCanceledByTimeout" + "\n----------------------------------------" + @@ -128,6 +134,11 @@ class StatsForClass(val project: String, val className: String) { val methodsWithAtLeastOneException: Int get() = statsForMethods.count { it.failReasons.isNotEmpty() } val testcasesGenerated: Int get() = statsForMethods.sumOf { it.testsGeneratedCount } + val detectedExceptionsCount: Int + get() = statsForMethods.sumOf { it.detectedExceptionsCount } + val expectedExceptionsCount: Int + get() = statsForMethods.sumOf { it.expectedExceptionsCount } + var coverage = CoverageInstructionsSet() var fuzzedCoverage = CoverageInstructionsSet() var concolicCoverage = CoverageInstructionsSet() @@ -149,23 +160,37 @@ class StatsForClass(val project: String, val className: String) { "\n\t#methods with at least one TC = ${statsForMethods.count { it.testsGeneratedCount > 0 }}" + "\n\t#methods with exceptions = $methodsWithAtLeastOneException" + "\n\t#generated TC = $testcasesGenerated" + + (if (expectedExceptionsCount > 0) "\n\t#detected exceptions = $detectedExceptionsCount/$expectedExceptionsCount" else "") + "\n\t#total coverage = ${coverage.prettyInfo()}" + "\n\t#fuzzed coverage = ${fuzzedCoverage.prettyInfo()}" + "\n\t#concolic coverage = ${concolicCoverage.prettyInfo()}" } -class StatsForMethod(val methodName: String) { +class StatsForMethod( + val methodName: String, + val expectedExceptionFqns: List +) { var testsGeneratedCount = 0 val failReasons: MutableMultiset = mutableMultisetOf() + val detectedExceptionFqns: MutableSet = mutableSetOf() + //generated no TC, nor exception val isSuspicious: Boolean get() = failReasons.isEmpty() && testsGeneratedCount == 0 + val detectedExceptionsCount: Int + get() = expectedExceptionFqns.toSet().intersect(detectedExceptionFqns).size + + val expectedExceptionsCount: Int = + expectedExceptionFqns.size + override fun toString(): String = "\n :" + (if (isSuspicious) " SUSPICIOUS" else "") + - "\n\t#generatedTC=$testsGeneratedCount\n\t" + + "\n\t#generated TC = $testsGeneratedCount" + + (if (expectedExceptionsCount > 0) "\n\t#detected exceptions = $detectedExceptionsCount/$expectedExceptionsCount" else "") + + "\n\t" + (if (failReasons.isEmpty()) "WITH NO EXCEPTIONS" else "FAILED ${failReasons.sumOfMultiplicities} time(s) with ${failReasons.size} different exception(s)\"") diff --git a/utbot-junit-contest/src/main/resources/classes/codeforces/exceptions b/utbot-junit-contest/src/main/resources/classes/codeforces/exceptions new file mode 100644 index 0000000000..9a5c69a095 --- /dev/null +++ b/utbot-junit-contest/src/main/resources/classes/codeforces/exceptions @@ -0,0 +1,11 @@ +org.utbot.examples.codeforces.error.stackof.Task_70A.solve: java.lang.StackOverflowError +org.utbot.examples.codeforces.exception.aiobe.Task_70A.solve: java.lang.ArrayIndexOutOfBoundsException +org.utbot.examples.codeforces.exception.aiobe.Task_71B.solve: java.lang.ArrayIndexOutOfBoundsException +org.utbot.examples.codeforces.exception.aiobe.Task_1712C.solve: java.lang.ArrayIndexOutOfBoundsException +org.utbot.examples.codeforces.exception.aiobe.Task_1749C.solve: java.lang.ArrayIndexOutOfBoundsException +org.utbot.examples.codeforces.exception.arithmetic.Task_1715B.solve: java.lang.ArithmeticException +org.utbot.examples.codeforces.exception.error.Task_1718A2.solve: java.lang.AssertionError +org.utbot.examples.codeforces.exception.iobe.Task_1740C.solve: java.lang.IndexOutOfBoundsException +org.utbot.examples.codeforces.exception.npe.Task_1703D.solve: java.lang.NullPointerException +org.utbot.examples.codeforces.exception.siobe.Task_75A.solve: java.lang.StringIndexOutOfBoundsException +org.utbot.examples.codeforces.exception.siobe.Task_1702B.solve: java.lang.StringIndexOutOfBoundsException \ No newline at end of file diff --git a/utbot-junit-contest/src/main/resources/classes/codeforces/list b/utbot-junit-contest/src/main/resources/classes/codeforces/list new file mode 100644 index 0000000000..2fa0b1a6a1 --- /dev/null +++ b/utbot-junit-contest/src/main/resources/classes/codeforces/list @@ -0,0 +1,11 @@ +org.utbot.examples.codeforces.error.stackof.Task_70A +org.utbot.examples.codeforces.exception.aiobe.Task_70A +org.utbot.examples.codeforces.exception.aiobe.Task_71B +org.utbot.examples.codeforces.exception.aiobe.Task_1712C +org.utbot.examples.codeforces.exception.aiobe.Task_1749C +org.utbot.examples.codeforces.exception.arithmetic.Task_1715B +org.utbot.examples.codeforces.exception.error.Task_1718A2 +org.utbot.examples.codeforces.exception.iobe.Task_1740C +org.utbot.examples.codeforces.exception.npe.Task_1703D +org.utbot.examples.codeforces.exception.siobe.Task_75A +org.utbot.examples.codeforces.exception.siobe.Task_1702B \ No newline at end of file diff --git a/utbot-junit-contest/src/main/resources/projects/codeforces/Codeforces-1.0-SNAPSHOT.jar b/utbot-junit-contest/src/main/resources/projects/codeforces/Codeforces-1.0-SNAPSHOT.jar new file mode 100644 index 0000000000..8e57691354 Binary files /dev/null and b/utbot-junit-contest/src/main/resources/projects/codeforces/Codeforces-1.0-SNAPSHOT.jar differ