Add codeforces examples to contest estimator

utbot-junit-contest/README.md

@@ -28,6 +28,9 @@ The projects are provided to Contest estimator in advance.
 All available projects are placed in the [resources][resources] folder, which contains:
 - [projects][projects] consisting of the folders with the project JAR files in them.
 - [classes][classes] consisting of the folders — each named after the project and containing the `list` file with the fully qualified class names.
+It also may contain an `exceptions` file with the description of the expected exceptions, that utbot should find.  
+Description is presented in the format: `<class fully qualified name>.<method name>: <expected exception fqn> <another fqn> ...`.
+For example, see this [file](src/main/resources/classes/codeforces/exceptions).
 
 ### How to add a new project
 You should add both the JAR files to the `projects` folder and the file with a list of classes to the `classes` folder.

utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt

@@ -14,14 +14,6 @@ import org.utbot.framework.codegen.domain.ForceStaticMocking
 import org.utbot.framework.codegen.domain.StaticsMocking
 import org.utbot.framework.codegen.domain.junitByVersion
 import org.utbot.framework.codegen.CodeGenerator
-import org.utbot.framework.plugin.api.CodegenLanguage
-import org.utbot.framework.plugin.api.Coverage
-import org.utbot.framework.plugin.api.ExecutableId
-import org.utbot.framework.plugin.api.MockStrategyApi
-import org.utbot.framework.plugin.api.TestCaseGenerator
-import org.utbot.framework.plugin.api.UtError
-import org.utbot.framework.plugin.api.UtExecution
-import org.utbot.framework.plugin.api.UtMethodTestSet
 import org.utbot.framework.plugin.api.util.UtContext
 import org.utbot.framework.plugin.api.util.executableId
 import org.utbot.framework.plugin.api.util.id
@@ -59,10 +51,10 @@ import kotlinx.coroutines.launch
 import kotlinx.coroutines.newSingleThreadContext
 import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.withTimeoutOrNull
-import kotlinx.coroutines.yield
 import org.utbot.framework.SummariesGenerationType
 import org.utbot.framework.codegen.services.language.CgLanguageAssistant
 import org.utbot.framework.minimization.minimizeExecutions
+import org.utbot.framework.plugin.api.*
 import org.utbot.framework.plugin.api.util.isSynthetic
 import org.utbot.framework.util.jimpleBody
 import org.utbot.summary.summarize
@@ -154,6 +146,7 @@ fun main(args: Array<String>) {
                 fuzzingRatio = 0.1,
                 classpathString,
                 runFromEstimator = false,
+                expectedExceptions = ExpectedExceptionsForClass(),
                 methodNameFilter = null
             )
             println("${ContestMessage.READY}")
@@ -187,6 +180,7 @@ fun runGeneration(
     fuzzingRatio: Double,
     classpathString: String,
     runFromEstimator: Boolean,
+    expectedExceptions: ExpectedExceptionsForClass,
     methodNameFilter: String? = null // For debug purposes you can specify method name
 ): StatsForClass = runBlocking {
     val testsByMethod: MutableMap<ExecutableId, MutableList<UtExecution>> = mutableMapOf()
@@ -266,7 +260,10 @@ fun runGeneration(
                     val methodJob = currentCoroutineContext().job
 
                     logger.debug { " ... " }
-                    val statsForMethod = StatsForMethod("${method.classId.simpleName}#${method.name}")
+                    val statsForMethod = StatsForMethod(
+                        "${method.classId.simpleName}#${method.name}",
+                        expectedExceptions.getForMethod(method.name).exceptionNames
+                    )
                     statsForClass.statsForMethods.add(statsForMethod)
 
 
@@ -337,6 +334,9 @@ fun runGeneration(
                                             val className = Type.getInternalName(method.classId.jClass)
                                             logger.debug { "--new testCase collected, to generate: $testMethodName" }
                                             statsForMethod.testsGeneratedCount++
+                                            result.result.exceptionOrNull()?.let { exception ->
+                                                statsForMethod.detectedExceptionFqns += exception::class.java.name
+                                            }
                                             result.coverage?.let {
                                                 statsForClass.updateCoverage(
                                                     newCoverage = it,

utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt

@@ -137,7 +137,8 @@ enum class Tool {
             methodNameFilter: String?,
             statsForProject: StatsForProject,
             compiledTestDir: File,
-            classFqn: String
+            classFqn: String,
+            expectedExceptions: ExpectedExceptionsForClass
         ) = withUtContext(ContextManager.createNewContext(project.classloader)) {
             val classStats: StatsForClass = try {
                 runGeneration(
@@ -147,6 +148,7 @@ enum class Tool {
                     fuzzingRatio,
                     project.sootClasspathString,
                     runFromEstimator = true,
+                    expectedExceptions,
                     methodNameFilter
                 )
             } catch (e: CancellationException) {
@@ -206,7 +208,8 @@ enum class Tool {
             methodNameFilter: String?,
             statsForProject: StatsForProject,
             compiledTestDir: File,
-            classFqn: String
+            classFqn: String,
+            expectedExceptions: ExpectedExceptionsForClass
         ) {
             // EvoSuite has several phases, the variable below is responsible for assert generation
             // timeout. We want to give 10s for a big time budgets and timeLimit / 5 for small budgets.
@@ -277,7 +280,8 @@ enum class Tool {
         methodNameFilter: String?,
         statsForProject: StatsForProject,
         compiledTestDir: File,
-        classFqn: String
+        classFqn: String,
+        expectedExceptions: ExpectedExceptionsForClass
     )
 
     abstract fun moveProducedFilesIfNeeded()
@@ -391,7 +395,13 @@ fun runEstimator(
     if (updatedMethodFilter != null)
         logger.info { "Filtering: class='$classFqnFilter', method ='$methodNameFilter'" }
 
-    val projectToClassFQNs = classesLists.listFiles()!!.associate { it.name to File(it, "list").readLines() }
+    val projectDirs = classesLists.listFiles()!!
+    val projectToClassFQNs = projectDirs.associate {
+        it.name to File(it, "list").readLines()
+    }
+    val projectToExpectedExceptions = projectDirs.associate {
+        it.name to parseExceptionsFile(File(it, "exceptions"))
+    }
 
     val projects = mutableListOf<ProjectToEstimate>()
 
@@ -400,6 +410,7 @@ fun runEstimator(
         val project = ProjectToEstimate(
             name,
             classesFQN,
+            projectToExpectedExceptions.getValue(name),
             File(classpathDir, name).listFiles()!!.filter { it.toString().endsWith("jar") },
             testCandidatesDir,
             unzippedJars
@@ -463,7 +474,8 @@ fun runEstimator(
                             methodNameFilter,
                             statsForProject,
                             compiledTestDir,
-                            classFqn
+                            classFqn,
+                            project.expectedExceptions.getForClass(classFqn)
                         )
                     }
                     catch (e: Throwable) {
@@ -526,6 +538,7 @@ private fun classNamesByJar(jar: File): List<String> {
 class ProjectToEstimate(
     val name: String,
     val classFQNs: List<String>,
+    val expectedExceptions:  ExpectedExceptionsForProject,
     private val jars: List<File>,
     testCandidatesDir: File,
     unzippedJars: File

utbot-junit-contest/src/main/kotlin/org/utbot/contest/ExpectedExceptions.kt

@@ -0,0 +1,54 @@
+package org.utbot.contest
+
+import java.io.File
+
+class ExpectedExceptionsForMethod(
+    val exceptionNames: List<String>
+)
+
+class ExpectedExceptionsForClass {
+
+    private val byMethodName: MutableMap<String, ExpectedExceptionsForMethod> =
+        mutableMapOf()
+
+    fun getForMethod(methodName: String): ExpectedExceptionsForMethod =
+        byMethodName[methodName] ?: ExpectedExceptionsForMethod(listOf())
+
+    fun setForMethod(methodName: String, exceptionNames: List<String>) {
+        byMethodName[methodName] = ExpectedExceptionsForMethod(exceptionNames)
+    }
+}
+
+class ExpectedExceptionsForProject {
+
+    private val byClassName: MutableMap<String, ExpectedExceptionsForClass> =
+        mutableMapOf()
+
+    fun getForClass(className: String): ExpectedExceptionsForClass =
+        byClassName[className] ?: ExpectedExceptionsForClass()
+
+    fun setForClass(className: String, methodName: String, exceptionNames: List<String>) {
+        val forClass = byClassName.getOrPut(className) { ExpectedExceptionsForClass() }
+        forClass.setForMethod(methodName, exceptionNames)
+    }
+}
+
+fun parseExceptionsFile(exceptionsDescriptionFile: File): ExpectedExceptionsForProject {
+    if (!exceptionsDescriptionFile.exists()) {
+        return ExpectedExceptionsForProject()
+    }
+
+    val forProject = ExpectedExceptionsForProject()
+    for (methodDescription in exceptionsDescriptionFile.readLines()) {
+        val methodFqn = methodDescription.substringBefore(':').trim()
+        val classFqn = methodFqn.substringBeforeLast('.')
+        val methodName = methodFqn.substringAfterLast('.')
+        val exceptionFqns = methodDescription.substringAfter(':')
+            .split(' ')
+            .map { it.trim() }
+            .filter { it.isNotBlank() }
+        forProject.setForClass(classFqn, methodName, exceptionFqns)
+    }
+
+    return forProject
+}

utbot-junit-contest/src/main/kotlin/org/utbot/contest/Statistics.kt

@@ -75,9 +75,15 @@ class StatsForProject(val project: String) {
                 else this
             }
 
+    val detectedExceptionsCount: Int
+        get() = statsForClasses.sumOf { it.detectedExceptionsCount }
+    val expectedExceptionsCount: Int
+        get() = statsForClasses.sumOf { it.expectedExceptionsCount }
+
     override fun toString(): String = "\n<StatsForProject($project)> :" +
             "\n\t#classes for generation = $classesForGeneration" +
             "\n\t#tc generated = $testCasesGenerated" +
+            (if (expectedExceptionsCount > 0) "\n\t#detected exceptions = $detectedExceptionsCount/$expectedExceptionsCount" else "") +
             "\n\t#classes without problems = $classesWithoutProblems" +
             "\n\t#classes canceled by timeout = $classesCanceledByTimeout" +
             "\n----------------------------------------" +
@@ -128,6 +134,11 @@ class StatsForClass(val project: String, val className: String) {
     val methodsWithAtLeastOneException: Int get() = statsForMethods.count { it.failReasons.isNotEmpty() }
     val testcasesGenerated: Int get() = statsForMethods.sumOf { it.testsGeneratedCount }
 
+    val detectedExceptionsCount: Int
+        get() = statsForMethods.sumOf { it.detectedExceptionsCount }
+    val expectedExceptionsCount: Int
+        get() = statsForMethods.sumOf { it.expectedExceptionsCount }
+
     var coverage = CoverageInstructionsSet()
     var fuzzedCoverage = CoverageInstructionsSet()
     var concolicCoverage = CoverageInstructionsSet()
@@ -149,23 +160,37 @@ class StatsForClass(val project: String, val className: String) {
             "\n\t#methods with at least one TC = ${statsForMethods.count { it.testsGeneratedCount > 0 }}" +
             "\n\t#methods with exceptions = $methodsWithAtLeastOneException" +
             "\n\t#generated TC = $testcasesGenerated" +
+            (if (expectedExceptionsCount > 0) "\n\t#detected exceptions = $detectedExceptionsCount/$expectedExceptionsCount" else "") +
             "\n\t#total coverage = ${coverage.prettyInfo()}" +
             "\n\t#fuzzed coverage = ${fuzzedCoverage.prettyInfo()}" +
             "\n\t#concolic coverage = ${concolicCoverage.prettyInfo()}"
 }
 
 
-class StatsForMethod(val methodName: String) {
+class StatsForMethod(
+    val methodName: String,
+    val expectedExceptionFqns: List<String>
+) {
     var testsGeneratedCount = 0
 
     val failReasons: MutableMultiset<FailReason> = mutableMultisetOf()
 
+    val detectedExceptionFqns: MutableSet<String> = mutableSetOf()
+
     //generated no TC, nor exception
     val isSuspicious: Boolean get() = failReasons.isEmpty() && testsGeneratedCount == 0
 
+    val detectedExceptionsCount: Int
+        get() = expectedExceptionFqns.toSet().intersect(detectedExceptionFqns).size
+
+    val expectedExceptionsCount: Int =
+        expectedExceptionFqns.size
+
 
     override fun toString(): String = "\n<StatsForMethod> :" + (if (isSuspicious) " SUSPICIOUS" else "") +
-            "\n\t#generatedTC=$testsGeneratedCount\n\t" +
+            "\n\t#generated TC = $testsGeneratedCount" +
+            (if (expectedExceptionsCount > 0) "\n\t#detected exceptions = $detectedExceptionsCount/$expectedExceptionsCount" else "") +
+            "\n\t" +
             (if (failReasons.isEmpty()) "WITH NO EXCEPTIONS"
             else "FAILED ${failReasons.sumOfMultiplicities} time(s) with ${failReasons.size} different exception(s)\"")
 

utbot-junit-contest/src/main/resources/classes/codeforces/exceptions

@@ -0,0 +1,11 @@
+org.utbot.examples.codeforces.error.stackof.Task_70A.solve: java.lang.StackOverflowError
+org.utbot.examples.codeforces.exception.aiobe.Task_70A.solve: java.lang.ArrayIndexOutOfBoundsException
+org.utbot.examples.codeforces.exception.aiobe.Task_71B.solve: java.lang.ArrayIndexOutOfBoundsException
+org.utbot.examples.codeforces.exception.aiobe.Task_1712C.solve: java.lang.ArrayIndexOutOfBoundsException
+org.utbot.examples.codeforces.exception.aiobe.Task_1749C.solve: java.lang.ArrayIndexOutOfBoundsException
+org.utbot.examples.codeforces.exception.arithmetic.Task_1715B.solve: java.lang.ArithmeticException
+org.utbot.examples.codeforces.exception.error.Task_1718A2.solve: java.lang.AssertionError
+org.utbot.examples.codeforces.exception.iobe.Task_1740C.solve: java.lang.IndexOutOfBoundsException
+org.utbot.examples.codeforces.exception.npe.Task_1703D.solve: java.lang.NullPointerException
+org.utbot.examples.codeforces.exception.siobe.Task_75A.solve: java.lang.StringIndexOutOfBoundsException
+org.utbot.examples.codeforces.exception.siobe.Task_1702B.solve: java.lang.StringIndexOutOfBoundsException

utbot-junit-contest/src/main/resources/classes/codeforces/list

@@ -0,0 +1,11 @@
+org.utbot.examples.codeforces.error.stackof.Task_70A
+org.utbot.examples.codeforces.exception.aiobe.Task_70A
+org.utbot.examples.codeforces.exception.aiobe.Task_71B
+org.utbot.examples.codeforces.exception.aiobe.Task_1712C
+org.utbot.examples.codeforces.exception.aiobe.Task_1749C
+org.utbot.examples.codeforces.exception.arithmetic.Task_1715B
+org.utbot.examples.codeforces.exception.error.Task_1718A2
+org.utbot.examples.codeforces.exception.iobe.Task_1740C
+org.utbot.examples.codeforces.exception.npe.Task_1703D
+org.utbot.examples.codeforces.exception.siobe.Task_75A
+org.utbot.examples.codeforces.exception.siobe.Task_1702B