From a11c0ce74d981c6c5a9e7f6a072c9f9a2f59e7bd Mon Sep 17 00:00:00 2001 From: Nikita Stroganov Date: Wed, 25 Jan 2023 15:42:34 +0300 Subject: [PATCH 1/5] Add codeforces examples to contest --- .../main/kotlin/org/utbot/contest/Contest.kt | 24 +++--- .../org/utbot/contest/ContestEstimator.kt | 72 ++++++++++++++++-- .../kotlin/org/utbot/contest/Statistics.kt | 29 ++++++- .../resources/classes/codeforces/exceptions | 11 +++ .../main/resources/classes/codeforces/list | 11 +++ .../codeforces/Codeforces-1.0-SNAPSHOT.jar | Bin 0 -> 14610 bytes 6 files changed, 128 insertions(+), 19 deletions(-) create mode 100644 utbot-junit-contest/src/main/resources/classes/codeforces/exceptions create mode 100644 utbot-junit-contest/src/main/resources/classes/codeforces/list create mode 100644 utbot-junit-contest/src/main/resources/projects/codeforces/Codeforces-1.0-SNAPSHOT.jar diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt index c41a617309..25213cf2ab 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt @@ -14,14 +14,6 @@ import org.utbot.framework.codegen.domain.ForceStaticMocking import org.utbot.framework.codegen.domain.StaticsMocking import org.utbot.framework.codegen.domain.junitByVersion import org.utbot.framework.codegen.CodeGenerator -import org.utbot.framework.plugin.api.CodegenLanguage -import org.utbot.framework.plugin.api.Coverage -import org.utbot.framework.plugin.api.ExecutableId -import org.utbot.framework.plugin.api.MockStrategyApi -import org.utbot.framework.plugin.api.TestCaseGenerator -import org.utbot.framework.plugin.api.UtError -import org.utbot.framework.plugin.api.UtExecution -import org.utbot.framework.plugin.api.UtMethodTestSet import org.utbot.framework.plugin.api.util.UtContext import org.utbot.framework.plugin.api.util.executableId import org.utbot.framework.plugin.api.util.id @@ -59,10 +51,10 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.newSingleThreadContext import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withTimeoutOrNull -import kotlinx.coroutines.yield import org.utbot.framework.SummariesGenerationType import org.utbot.framework.codegen.services.language.CgLanguageAssistant import org.utbot.framework.minimization.minimizeExecutions +import org.utbot.framework.plugin.api.* import org.utbot.framework.plugin.api.util.isSynthetic import org.utbot.framework.util.jimpleBody import org.utbot.summary.summarize @@ -154,7 +146,8 @@ fun main(args: Array) { fuzzingRatio = 0.1, classpathString, runFromEstimator = false, - methodNameFilter = null + methodNameFilter = null, + expectedExceptions = ExpectedExceptionsForClass() ) println("${ContestMessage.READY}") } @@ -187,7 +180,8 @@ fun runGeneration( fuzzingRatio: Double, classpathString: String, runFromEstimator: Boolean, - methodNameFilter: String? = null // For debug purposes you can specify method name + methodNameFilter: String? = null, // For debug purposes you can specify method name + expectedExceptions: ExpectedExceptionsForClass ): StatsForClass = runBlocking { val testsByMethod: MutableMap> = mutableMapOf() val currentContext = utContext @@ -266,7 +260,10 @@ fun runGeneration( val methodJob = currentCoroutineContext().job logger.debug { " ... " } - val statsForMethod = StatsForMethod("${method.classId.simpleName}#${method.name}") + val statsForMethod = StatsForMethod( + "${method.classId.simpleName}#${method.name}", + expectedExceptions.getForMethod(method.name).exceptionNames + ) statsForClass.statsForMethods.add(statsForMethod) @@ -337,6 +334,9 @@ fun runGeneration( val className = Type.getInternalName(method.classId.jClass) logger.debug { "--new testCase collected, to generate: $testMethodName" } statsForMethod.testsGeneratedCount++ + result.result.exceptionOrNull()?.let { exception -> + statsForMethod.detectedExceptionFqns += exception::class.java.name + } result.coverage?.let { statsForClass.updateCoverage( newCoverage = it, diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt index a9fe1a25d7..c2aa44dce5 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt @@ -137,7 +137,8 @@ enum class Tool { methodNameFilter: String?, statsForProject: StatsForProject, compiledTestDir: File, - classFqn: String + classFqn: String, + expectedExceptions: ExpectedExceptionsForClass ) = withUtContext(ContextManager.createNewContext(project.classloader)) { val classStats: StatsForClass = try { runGeneration( @@ -147,7 +148,8 @@ enum class Tool { fuzzingRatio, project.sootClasspathString, runFromEstimator = true, - methodNameFilter + methodNameFilter, + expectedExceptions ) } catch (e: CancellationException) { logger.info { "[$classFqn] finished with CancellationException" } @@ -277,7 +279,8 @@ enum class Tool { methodNameFilter: String?, statsForProject: StatsForProject, compiledTestDir: File, - classFqn: String + classFqn: String, + expectedExceptions: ExpectedExceptionsForClass ) abstract fun moveProducedFilesIfNeeded() @@ -391,7 +394,13 @@ fun runEstimator( if (updatedMethodFilter != null) logger.info { "Filtering: class='$classFqnFilter', method ='$methodNameFilter'" } - val projectToClassFQNs = classesLists.listFiles()!!.associate { it.name to File(it, "list").readLines() } + val projectDirs = classesLists.listFiles()!! + val projectToClassFQNs = projectDirs.associate { + it.name to File(it, "list").readLines() + } + val projectToExpectedExceptions = projectDirs.associate { + it.name to parseExceptionsFile(File(it, "exceptions")) + } val projects = mutableListOf() @@ -400,6 +409,7 @@ fun runEstimator( val project = ProjectToEstimate( name, classesFQN, + projectToExpectedExceptions[name]!!, File(classpathDir, name).listFiles()!!.filter { it.toString().endsWith("jar") }, testCandidatesDir, unzippedJars @@ -463,7 +473,8 @@ fun runEstimator( methodNameFilter, statsForProject, compiledTestDir, - classFqn + classFqn, + project.expectedExceptions.getForClass(classFqn) ) } catch (e: Throwable) { @@ -523,6 +534,57 @@ private fun classNamesByJar(jar: File): List { .toList() } +class ExpectedExceptionsForMethod( + val exceptionNames: List +) + +class ExpectedExceptionsForClass { + + private val byMethodName: MutableMap = + mutableMapOf() + + fun getForMethod(methodName: String): ExpectedExceptionsForMethod = + byMethodName[methodName] ?: ExpectedExceptionsForMethod(listOf()) + + fun setForMethod(methodName: String, exceptionNames: List) { + byMethodName[methodName] = ExpectedExceptionsForMethod(exceptionNames) + } +} + +class ExpectedExceptionsForProject { + + private val byClassName: MutableMap = + mutableMapOf() + + fun getForClass(className: String): ExpectedExceptionsForClass = + byClassName[className] ?: ExpectedExceptionsForClass() + + fun setForClass(className: String, methodName: String, exceptionNames: List) { + val forClass = byClassName.getOrPut(className) { ExpectedExceptionsForClass() } + forClass.setForMethod(methodName, exceptionNames) + } +} + +fun parseExceptionsFile(exceptionsDescriptionFile: File): ExpectedExceptionsForProject { + if (!exceptionsDescriptionFile.exists()) { + return ExpectedExceptionsForProject() + } + + val forProject = ExpectedExceptionsForProject() + for (methodDescription in exceptionsDescriptionFile.readLines()) { + val methodFqn = methodDescription.substringBefore(':').trim() + val classFqn = methodFqn.substringBeforeLast('.') + val methodName = methodFqn.substringAfterLast('.') + val exceptionFqns = methodDescription.substringAfter(':') + .split(' ') + .map { it.trim() } + .filter { it.isNotBlank() } + forProject.setForClass(classFqn, methodName, exceptionFqns) + } + + return forProject +} + class ProjectToEstimate( val name: String, val classFQNs: List, diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Statistics.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Statistics.kt index 16918a8e4c..049947a6db 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Statistics.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Statistics.kt @@ -75,9 +75,15 @@ class StatsForProject(val project: String) { else this } + val detectedExceptionsCount: Int + get() = statsForClasses.sumOf { it.detectedExceptionsCount } + val expectedExceptionsCount: Int + get() = statsForClasses.sumOf { it.expectedExceptionsCount } + override fun toString(): String = "\n :" + "\n\t#classes for generation = $classesForGeneration" + "\n\t#tc generated = $testCasesGenerated" + + (if (expectedExceptionsCount > 0) "\n\t#detected exceptions = $detectedExceptionsCount/$expectedExceptionsCount" else "") + "\n\t#classes without problems = $classesWithoutProblems" + "\n\t#classes canceled by timeout = $classesCanceledByTimeout" + "\n----------------------------------------" + @@ -128,6 +134,11 @@ class StatsForClass(val project: String, val className: String) { val methodsWithAtLeastOneException: Int get() = statsForMethods.count { it.failReasons.isNotEmpty() } val testcasesGenerated: Int get() = statsForMethods.sumOf { it.testsGeneratedCount } + val detectedExceptionsCount: Int + get() = statsForMethods.sumOf { it.detectedExceptionsCount } + val expectedExceptionsCount: Int + get() = statsForMethods.sumOf { it.expectedExceptionsCount } + var coverage = CoverageInstructionsSet() var fuzzedCoverage = CoverageInstructionsSet() var concolicCoverage = CoverageInstructionsSet() @@ -149,23 +160,37 @@ class StatsForClass(val project: String, val className: String) { "\n\t#methods with at least one TC = ${statsForMethods.count { it.testsGeneratedCount > 0 }}" + "\n\t#methods with exceptions = $methodsWithAtLeastOneException" + "\n\t#generated TC = $testcasesGenerated" + + (if (expectedExceptionsCount > 0) "\n\t#detected exceptions = $detectedExceptionsCount/$expectedExceptionsCount" else "") + "\n\t#total coverage = ${coverage.prettyInfo()}" + "\n\t#fuzzed coverage = ${fuzzedCoverage.prettyInfo()}" + "\n\t#concolic coverage = ${concolicCoverage.prettyInfo()}" } -class StatsForMethod(val methodName: String) { +class StatsForMethod( + val methodName: String, + val expectedExceptionFqns: List +) { var testsGeneratedCount = 0 val failReasons: MutableMultiset = mutableMultisetOf() + val detectedExceptionFqns: MutableSet = mutableSetOf() + //generated no TC, nor exception val isSuspicious: Boolean get() = failReasons.isEmpty() && testsGeneratedCount == 0 + val detectedExceptionsCount: Int + get() = expectedExceptionFqns.toSet().intersect(detectedExceptionFqns).size + + val expectedExceptionsCount: Int = + expectedExceptionFqns.size + override fun toString(): String = "\n :" + (if (isSuspicious) " SUSPICIOUS" else "") + - "\n\t#generatedTC=$testsGeneratedCount\n\t" + + "\n\t#generated TC = $testsGeneratedCount" + + (if (expectedExceptionsCount > 0) "\n\t#detected exceptions = $detectedExceptionsCount/$expectedExceptionsCount" else "") + + "\n\t" + (if (failReasons.isEmpty()) "WITH NO EXCEPTIONS" else "FAILED ${failReasons.sumOfMultiplicities} time(s) with ${failReasons.size} different exception(s)\"") diff --git a/utbot-junit-contest/src/main/resources/classes/codeforces/exceptions b/utbot-junit-contest/src/main/resources/classes/codeforces/exceptions new file mode 100644 index 0000000000..9a5c69a095 --- /dev/null +++ b/utbot-junit-contest/src/main/resources/classes/codeforces/exceptions @@ -0,0 +1,11 @@ +org.utbot.examples.codeforces.error.stackof.Task_70A.solve: java.lang.StackOverflowError +org.utbot.examples.codeforces.exception.aiobe.Task_70A.solve: java.lang.ArrayIndexOutOfBoundsException +org.utbot.examples.codeforces.exception.aiobe.Task_71B.solve: java.lang.ArrayIndexOutOfBoundsException +org.utbot.examples.codeforces.exception.aiobe.Task_1712C.solve: java.lang.ArrayIndexOutOfBoundsException +org.utbot.examples.codeforces.exception.aiobe.Task_1749C.solve: java.lang.ArrayIndexOutOfBoundsException +org.utbot.examples.codeforces.exception.arithmetic.Task_1715B.solve: java.lang.ArithmeticException +org.utbot.examples.codeforces.exception.error.Task_1718A2.solve: java.lang.AssertionError +org.utbot.examples.codeforces.exception.iobe.Task_1740C.solve: java.lang.IndexOutOfBoundsException +org.utbot.examples.codeforces.exception.npe.Task_1703D.solve: java.lang.NullPointerException +org.utbot.examples.codeforces.exception.siobe.Task_75A.solve: java.lang.StringIndexOutOfBoundsException +org.utbot.examples.codeforces.exception.siobe.Task_1702B.solve: java.lang.StringIndexOutOfBoundsException \ No newline at end of file diff --git a/utbot-junit-contest/src/main/resources/classes/codeforces/list b/utbot-junit-contest/src/main/resources/classes/codeforces/list new file mode 100644 index 0000000000..2fa0b1a6a1 --- /dev/null +++ b/utbot-junit-contest/src/main/resources/classes/codeforces/list @@ -0,0 +1,11 @@ +org.utbot.examples.codeforces.error.stackof.Task_70A +org.utbot.examples.codeforces.exception.aiobe.Task_70A +org.utbot.examples.codeforces.exception.aiobe.Task_71B +org.utbot.examples.codeforces.exception.aiobe.Task_1712C +org.utbot.examples.codeforces.exception.aiobe.Task_1749C +org.utbot.examples.codeforces.exception.arithmetic.Task_1715B +org.utbot.examples.codeforces.exception.error.Task_1718A2 +org.utbot.examples.codeforces.exception.iobe.Task_1740C +org.utbot.examples.codeforces.exception.npe.Task_1703D +org.utbot.examples.codeforces.exception.siobe.Task_75A +org.utbot.examples.codeforces.exception.siobe.Task_1702B \ No newline at end of file diff --git a/utbot-junit-contest/src/main/resources/projects/codeforces/Codeforces-1.0-SNAPSHOT.jar b/utbot-junit-contest/src/main/resources/projects/codeforces/Codeforces-1.0-SNAPSHOT.jar new file mode 100644 index 0000000000000000000000000000000000000000..8e57691354cdef7038ac035565938ddce6e63333 GIT binary patch literal 14610 zcmbtb1z6SD(+25oM7p~!4T5xcgM@U$rBguaN=pfn(hX7)(%m2}-O>%x@Zs*V?z+3~ z`u~2!z0bp$c~8vDotg8ZAPoTt5BBx(HQ`qK<;`!u;O<^!MO6eDB;~}I6n+~93+8h- zY|ks;o$cM91Mhw?|2RxmP)<@zR9S^lR_sW&uUAHzfpG{)nt^tpZ?Ig6d6acw)t+8z zpo?CbK?oLRB~vX1o34r6wI(S<2}4>**)f^62ooOxJ(^Vs>p4cR(h+9p3H>pmTTPM_ ztAkWiS^FXy2-NX88}u1<2vg@>j8&?rK;kiCzkr2X4cPq*!N7puEd=szhCqAM-`DWh z5nngp>j)=D1EAv%#$o(voUyCEwXK!0!w&=q|6aflXk=^xv^V^rEcw668r$0g?SFty z{~t9tIO-c(0!@CPiTd9)xf&YVI+_D*exQZ%C$#9B0}YISpo{fqblICbnpqn=nj8K= z;m=I&-!`7%CnonV>--OA`1{V=*#2n8erACVf0`BSFD?ahRM2f06LXD3wE^Q08hq>nL2w~_baW-KATlVV zDK6wUTytH{sIUA)1cim6(igy8(Ni7(EW;HHyN~v69v}5@H+a_7JPIzep9=F&>V5oN-~{a{&cO!~>PM;f)6H2FrAkMOP|lZBhjc+Oy$9#rqDuHkN9& zr1SbsLI{Kns(ylkQO}`k^O3_=NAig{q@wv7x8R#Ai%(wp}@eN{QIf@Zi~PA?=R;L-~_M; z|90{!(m*^_9KMN$WyyZXwINAu&a_u-&2phHF(rgCMDv1a>A6KnvjW+aYASO!$iq7i zLo-_P(IX?yzylA};NcPc+p>#XMGOM`Kb5?MLZ>5Fv&zqx;&PdZ64KB&Z#Z>n@Mv^c zC30?cWF8c9lLCGhZObZ*HE0~u^MRWmurI0EP1)CoV|M; zAFC*z80To9%o9z>{aXc62j6X?$#B`$_DLWt(-F1B!Ny zw6ISt6DqdTO+I+ck3sc0sanv)QWas3^klyTsiyY*rNa$n z)Z3$7mMQf|<~^7f>&Xk_lr=@@Cm}Oqq#4rdoh+6@_$};4``F0K)|;n*z*lcBF~tv{ zBrl*l4+wVlf(|i22XtclFgyH_VbvN-H&opML>`EBf)$QNIxXB&*S!6vT=w%9k=_E< zA&7Qr-XgaIa)iX*A!iZgL}c^QyzOWv%SqEFwaJe=X6(dLgn@&FB_{32QIsJ6>X)qs zYTM}$fM&1~h+h9R4B;il;nG^>T|iuT1*UjjkUOcnPOCTgpB-0XD5^E-E>xG^Iq(1M zxNO|_j;lN>IyrvzUHWjw{$`V4yH2dUX1fw*R?+Yx{hg=1WO)d;~%4PLnT=5@LqXO+#W zC#Zppt=t07nDZIS?xm&@TefDwSTDFs_e=1_mm5y=&Y@c=rcZL>UHea9MP<;5dT}1) z^g}&^q+d@N;5bAa4z1w&0=4bnIxClh32hh%GCcz$F!HfkUgwoD0TQF4mh`_TXk*VR zfr#g94$u)%nn#o0G1uxOojj=5BmLlm6@+!1{z92L9c3_e5$fUq9l}-g#u9Of0dO8Z zSJx>aF~CP^fJ+6dND@qYEbkaFuDtGFh#D!X4PRhaxCC9Vl8ccZx=FM6q%zNf3GIWg zmWp~B)Ul!?dU6nVc3&xkh_PAdwHSY*Ap`A0eN3<$0TagpmJatXL*tc1yA}M}i%rk| zZ1+f*X1nuuX-f8;?eqR`A6VZK6-8O#hybc5#$y1xTr~7lc87ngy;-2j=_sipDk@x_ zk`yuncN2IKL6wuCN0F<}3CzCqI>|%EBmvk1h%*X_D+ux@9od18UnW3%P4-r!F%erf%Eoae)g`+#qv}l{itIWdBEta{_X&X0_dvYMN^TN2t5B-v)%Q zjILSR?!x538!rVbxchiemvt8;7j>S(jme0a1`K!T#?2~r(UGWh(7cQbn1`Ua>TZze zjfBRA`|Q)Fd1|hqk$a0Z&5LE)(t#%axz>4G?>W^CswyRM_ShF`}#80u7RvtC_`CKL}7ExvtS4^~_xPJzq3-`s1= z5W^z+gj|P;8&u|3(98G<8r4=Q0C~fiv2BFDX@sc>1e^=R4`0BsE~B~E%`X(Z7d`bnS2&2WCLQOq2{vVlQ*VN;4Jo?Zo6j zKVI#AuIBjO?qz()wq@Sv$HZ6bF!T_;$H`hD>eR)Rhj@QB9EprML4rG@VY&;L|Eu8u zgnr9&lzMGO1Tgt9rU6KH$=FH>4;M)UTONeSB0!1eN=QU1k(wn!y-vmGWRpwc3Qq1J zYLYL>3bE3M5(-3F0DdA(#eYmZ%0MwgZ3Z2F+4G$KahMBr@&u0P%&-dQQxED=YjiV$y zAWd%{%0e`XGuj3X6Yh3NvIK^+`}7s_8&-AN@V+^yYTxnUjny=h*s6|d zSJlS&bk=ANWekhIEd+uSj^YSrbj2d*a6=tuBhwffUmC(uZ zYu`Ys^a)MPP+!H`6YwR^!wY;*9&lET^g%|B5}v{8wkZvK?0$dPz|keRP=jKZ2Rg~Z zs!vF>Vqsqo2uQ^%oFl4SB~2>z0u>lyFIK&Z*R0LlidNX+Edol1_uj94i~*syzOG_M zpk4f8DcdLCh&Af6T||ClqaQ!l1l57ZY==@{Rx;HgS`pEyT(}4pxBp;^O9YPbR_LuA z$ilwU)X4l`hDCPS27=oJ-s-TB90%AvBSRkAWks1Uvcj1+N#gd93O9*j(1x}aoHG(Y z6#RPSyj!aC!o09?!2tP6K)o&)e`W}W+&si7)fwva^ZJv3gMrLSKo@-`z)LN zH|xt%*0RLV!n|=dp^TPckt{T~P;v^L7@fuef-{M86G93XKmlFB<7GOXr;M`MF!G4c z7N7=~eV%AymI@tgQ1>W`cpeKjyUaiT=)Dk~dXi{f3iFKM%ar8~aPR$i)x^}%m&=d| zFpE!lBwxyM0fvl$%GUbGe~hqJp-9!`x<*~V{IXvf@l7s%-o%#Y z+UAO(htCTxp+{M+F#vcM5x~r1d1n|+unYMa5SK?L(*!)zdXa~q- z-%VhcCuEg}hES;kjoG+7)z_ypZM8ANo`osY^H;a2q4!tP58I?BYfpLk43TtYkh}M_ zBt!1&ut8!%@oDaX1K80Le+Ty)un#3MJVCr4){+gU-NK))tPCR_E2&#S=kp?Hx+6=n zX73VPu8=6-ibSuXGTa7ZV=_8GQM@|4Js9pCB~ev#P8=8=QpYVF%oq zwAl~uACDy0gse`mIv}Kotevg{9-nf{a3YZjNtAsoirX~=^b(i9#7}SLkyIq$JGLL! ztUq`2|%p($)ZO#H&6wv08tgfzZLFuu&k zu>j#cDSyS#s23eks<}@AkEEJrXgc~XWP2L#D&T>kDb5DV@@WLbWg9);B3`Ph0SB69 zueoI8d*4_g8~N^*7g8XU?pA>Zvb@BJy3UP$=JuPb4o_PJrXzX@G0)8~k%PHaB2mle z3p9F3dY_Zem&gZViEY!hG$(DPt5@E@{Mq}4pU)!d-FY4@3>X;yPk7(29iV&1;}T@~ zE$*x8DBsn$UUJUS0TeM2=u!sMQszi+!84yj8AK=r1*B(<(-82-8u=iR^pfL20|`v7 z5GunBb98N&!*~p};+s8y^g%>5 z`GoL^T{yQ)JY zK*nu{MmCyaGH}1+QZ?$C{EG{1y2qHnxtg1pZwk_ERW$)kKjqsq82ol`TU z*%^(pBM%d&j2XR1kRmZD1<<~{4EPWVH`!oPn-57(@E#v?r^%jF3O;y?)=Jz z%%m5UrYpUld!P|erB82OR+gn`I$Kaz9s}8ebe)019gJKc3hCYc6EomURk(bF66V5$_ke5J;p&Jc7!(=7roZQ-q#(`oIuWJehC< z*ME9z4o2!4uqP%mq&ay7qayYo(P9M;lJsp=^HIC{$5G+j9@gCJ&d5$vlU|5UNn>rK zykvP)0@ys^<++H{UiT&+T4Z^zTf#Z<^}u6(KY838OE!YE#T#X|8~Q~W#=72*W?J2( zfrN(%!{$f!*9L#e~($#0t` z=<1Z%b@SkvPz()@-851jalUHVR_4tMA?&_DucMM#!eBIRz7R{`^#&v}zT2Qt9a2wA zh*o)(S`IBcVxckKbBo_k_*{Xa5!I)h=3vKn9QZHDNJy4RECwa7zCNHNOl zF&Pr39FsJnigz83Ne^Pj*Qtp?G@;;1c$ubwweX@k_JRIg;X z>#6Sn4Z)v&qL|g}{o-_W%c;FygRqi_AY=Cs@Amggj$eCowig0bqj#x;3^W)R-%kXu zUz)qW#V$7H`wCy>*U*JrI~i@oDPaC^n%Zp0LJ3xy4jRCgrDWj;N3HNQH^U|hs?w?~ zej7TEQ@0|6RrdmcXEc89xFZpLTc*=atn;9_vGoAOar7)jb{edRbGI+$DCN}S?J4)= zB|rZYNM;Zgmbk6hgUgG`;3`1RK{)&)Qx%a&^kJONTG|#I^#j7;m&~DXrAez@ux}lP z)%#79+sOw)Rz1UM!VwUA{B=sN5&bab=>c8JHr)p6J(F0LmAZ@???uZOBC@I0uw_lV z3n!PZL})IzDm$4|94sxxW|^uR^sVDp^9|j^BGHRJyeADBR~yQ|sMG>jRp~tn$*ww> z+$bH2%wrSoa!8hy%4#d0C<~&8DBsB9t1#zrVkv>|HYqZh(_60=gYGVW4kD}B`kXB( zYTa3Fop|TMGS!3j<{5Wqg}&g^09C{F+1nt_e0hG> zo99JAu{{7fOR>r&ElV|qBzH`TzQS$7-nU4GL}aMj`T+ZF_SAF+ zsQ1m9#;`4vIy?O`ZR@97e3r0`Wbb0zSz7e<Ie#+|$urtHh)1M!Co~X#)>lycx(K>JW_e zv=F8gWaehDryd(d9|z1(lBd8gNthc&3ec(9@e#(Gz&p`0_p~)pB;nUc%5%o-TwCM@=H>>IRu{7oV!gcnQE%QBqQ`Teu8Q2b>U<5t zKklV7YP>8*Dy55EAZ2A(gd-Epj?h+ zzf;u($-}rnetV;*_KLa{%qO|+)HRvKQK3F@v@;P?OAHPe`Bo-DvrQ6SaqJPfgpzG= zx<_{-7WWyweIRMH5uRYCWt5YAS!0@A*EZm|(KghuT5>?YLq8%mJd|(&Frf6144hBy z7U9pi4yt%*)yiG269oHn1@6~trQf0jGpopbq9bjEA&zrHsk>L4jAWO@)l7rgL-why zc~KE!Aj3z2n1Uq6f1mbMX}KLosf+p+JCr-QML@3mCU`FG`^BdBuW9`R!e~|VN8$67 zn6T;{_RvrRZp!U&n`}_tv)%UH_TMI)H)`G-S!QON z=xSF`Cto%j8<>lVd^<|OvX50QI}n0RG0{bZa=3I1TV?+4WbCrWh-p^w!&F@r&!u!% z#KtCf&xE`pUWax{GQ~rtH@-Gd%kbd<$j(Ca*&Q$8Of&6pnvx=r~pR`N6(Hg5;ot<2` z=SRA-x257{KPEYJV~Wt&DwoDo9N01$NH&4Rl#Vx8byXQq=%+#p8tZ`2EYNchC#v$7 z4LhYc98$#Lu}Dfasb{IOo=q}Z&&S&1CzSUbXVI~F(=L3GHIc8`BhIF(W0@DvP>O5X zq7^u7s&uv41UV>KOvxb0nlagUj{K==?n@%MG0JF3M|2AKxDqPy3nz`nb?wubkHk#_ z5<69>7PpyDP7{s(q_d!vT7ZA3*9yBL$TGXY;CWgX5NTE0&!^?Q($(@HNYvfTbd%{oLV0yYdbJ<{! zJ%*;&(DpVn_fv@D^d6n?3(#c<^6Z(@g?sEaGQ~gJI}Sd`GI}NzM4g`7;r2>%q%r@U$$^3INz8}UC293fpkVmY^6FT3TUX+z70L@W>Un7x`PhO zpbJ3kqfzP2K|81P4YEyas^84UFfP0TG3p#9*iS}>nTf&z*wP;AUr4aGrq_sEh?})y z6}GW&o~(`T?3p!)C*MRSoUoohNp*gb$?+sz>?X*GKJcUv4o?Gtqm09`Wam@caA4_^ zwn?z@l52^Q42^a#RXRK+!QX=^T37|iEzp^H)xK- zv@f0=knD7(DIlrS5u)q^KEOsF@HOO~p@&h+Yb(U4NjhS57cZb#wR4cJvR{FAK7kX> z6)V`INurfSqayS#f@93H3V}sEmOYau|>Xi6OwxMKM_XdZ7>w)DIhqddy z2_U%4Qbj20g%Q4ZV<-mBep{+P30x~;TBttmF1=rb)H{~Iqj7@dGz!Z%7bo_BoM}2< zbz*s~@nKi}*V7==iKO&e8!p5V&EA0mV@}hW{D%IJEZ^N+T#BdVR$(fhS|ieT(J$Xn zgIh;uap3dLL3cSuoC%I^xay~LZGyaQ==miJpF}sO$@fUC{L#94deGN!?diJsR89@~ zaw?1XZuswNy$v3b_tPd8+bAPFGYn*`)t=T~$$ENwSf-kd{0=8&XxMBkG>+i`oGiK+ zDNeg2TpsMO0x3=_4Jhxno>{nXXd`R}RII4Q7iLvwlq)JdtpfZI0$D+{%e9)$xi!iF z6fA9Il8W)Q`8@079l(|wQ8};^Qv9$qBBc||x58`hA=pu|2#ZXtR2QsYk95hyn^GB; zmB+^TWRLRDVKe;o*)pQtU98wY%f||au*s0U~1XGK_RI&kKy0_jQeW0_ARMO^V4K^-9o!))m=jo z&n)3WanHx;(3Yb^Jnsv{_~7Bk{Tft|Inf6q@!=`hwX4YETs>4GX8%_>%~b9RbFS}(I1&~mwRF#U17R={nO+th(G5M)(J#KJ$GeC-Cf!7GmY{;cmgN;JrzVn z+F@Gd&JXCd!nf*Vrd8}{6o$uec+m>m z>n4fzGS&B0it4+;g)U~@DoLR2La7P(;|9eetyn7CqNVCBlfie78}cew#HuoQ%LP-} zJFRUCDsl6-w8E|3W0loE2Om$(4`|2J!ME|{luKqt*VuDEt6}2InvDGNLGwtbSJzZP zX|kWaH#Rvnfb2@>knyPa2v!&97xf5Iw_JO0?VNYa_JJ@h8@-4^C#mH#y!`XBA>F9i zeGMM?&|+@BFAaK7o%k08i_(G+=An8j;()NNY7%CiJ+6ke+OpcrtJslaG4$&AwjwqY zKwJl}l=*Z212!8&lCW2$Z$!BFJ=2#**l0BjI-kuJ!y&4~K^5QdQ%-g~;p{!R%wHs8 z7IlQfx!f&^<&TOpA7QO(zYRLxd+HVPcrdPV zu=#Bl0Uz6JS)zrd8$%QA8h5hbJN45GK60KO-BYba3~}}Jp{U&?qJUTlVTD7XU1qC| zE+xau%CemTr`5yw4XFub6+dJYs?}beCkSFCq|7yY@3&=hiY^3)>>e7my3D}voSBf* zhL7wzK@A&EtJ&lUf57B+J+lvZB6dtpFIC%pE-Mz)-WqnLywu}Y%tLyW*i*pD_!&hw z#wbU*{JH9=o%+Iz_onPZUt{MY1LA=&#ufA#$s&RPw^;uQZFNVESHKyCBh1@W5i^-Z z_8v#f7t37DPN-A>LW+>JkGT*YQ*C63vPRH^f)j+oA$-Czy~8nldI?=~wRY}t+|*_m zL%4%tmMwvup=ax>1Z^$??F;Fa{RHGLe(+GU5^bSv^Ip0z*OI~r$q>^xY!#b^N$eFG z^VsBOAZOwmMF|qMkOn(OxYhZ9>rSBs#)IdN^y66PQIKqwp*GDKewo ztu97OFi9=)Z<>_r!sqRv2M4CqYiRHgO{=o zB5_Z{QC=z9_5lqb?tscLtcg#%2UznMU_l!%%za>z9uZrtuRSEx#on->aG4udt%)}qd{^P;hE=wgzxpn66i$<<#+^lDxlyT&JHJMfT9<)SPw z9$1;1Zqz!s2ivu(5u=TSHbN+w1QG254*)%$v65w;zD6HP)8mRJEGwSC`?3roT-ke+ z??1wgg)=9J#s}HcWpjuCF-;jO&VlDEoE;r%XnhY*&TnHt2APUjql0!BY$9-Tt(1{f zZ24moSSK3_Te`1>OZ3Bvn@JlvU6nz86t2=Cq$WwOhBIyp&joOsA}6>7>(M5ccDMEI zoNgudEgB(!An13H(wBUZT(s(8*z7n)s$0?@(}`Vq4-19?<&d1&+oX?;U9i^QV>yjc zG^`iD1TB-C$;&0dpuhwxK;as#GTC`|5#G%BEN4GwO{GHSAas2+vXq0u=UIEs!X&K{ zW0-QR%{xA;&){95#C1E~;=`SK(lqFP)420CRaqtU7+)gxaz`0voQc8f7BB)9LL`dEn$ z-2uT~DdZ&TNLgv*FeFMj#H|KBM2ckuJzDw&ZE~c4uTn@b>(P-F(g~uY-!(pjFW;50 z>sM+t_ybIce`a~#E%WX{1=|t)`ue+Ze-rzA*7w)JU!P1Ls=MF6i~WHE{!PPQIoEft z^xN|lxclz?R|fWvBz|4MSD~*GTfa%%bGE-N@q7OFSJw8g&sRB&J3s!4^sC%`MEM~w z;l2DXz#@OI;7=jxpC?29CtWZ8Ro6Y$_@6a?WeETJd|h74f7E!-%>B`Ze`S4t(~Njm z{QS+v{bAk4zq3Dor1dKg`I}aypU`^G3IFH4_{s|ZraR?lbbq@}|L>FE`w4)5Kgh2i zecvYk_%6f#uiO0_Tl=5)<fiI4zu}`b`{>kQl!up+<`90S6^tW$V zt%&~<*6(Dv?~%UeOMOGSLjK>7?%7k{BYjVi`G(|x_7h0oJpBIN{+=-NJ>d7OfNy{Y zcmMA6Uk~gbI0D~;eSeGh4XpUle+T>9JH79rzP|zZ1~p6Y-=Y5QKHz(#@4IE+knqU< zJJKII1^`ax``+32Sl{;}zF|>N{!duHcPG9f{e4yQD-tT$?X4v3A2Oi(8s|Q;{Z;sn zq5k&P`LDuvHul4s=bu%7t$M!AtNCYD|Djs>w)(%Xg6?rLeIC;NW%WN^z#mY4tBL*y v_O+1vJ`eu}_TX;weor=k-RhqL`<9g}NJHJldN44eyDuCBFtC+hj^+OVumgF@ literal 0 HcmV?d00001 From 31470ac2f4e86cae20d7f35d4c8c240ff7446ea9 Mon Sep 17 00:00:00 2001 From: Nikita Stroganov Date: Wed, 25 Jan 2023 16:32:32 +0300 Subject: [PATCH 2/5] Update documentation --- utbot-junit-contest/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utbot-junit-contest/README.md b/utbot-junit-contest/README.md index 6dbdf99013..63717d7d24 100644 --- a/utbot-junit-contest/README.md +++ b/utbot-junit-contest/README.md @@ -28,6 +28,9 @@ The projects are provided to Contest estimator in advance. All available projects are placed in the [resources][resources] folder, which contains: - [projects][projects] consisting of the folders with the project JAR files in them. - [classes][classes] consisting of the folders — each named after the project and containing the `list` file with the fully qualified class names. +It also may contain an `exceptions` file with the description of the expected exceptions, that utbot should find. +Description is presented in the format: `.: ...`. +For example, see this [file](src/main/resources/classes/codeforces/exceptions). ### How to add a new project You should add both the JAR files to the `projects` folder and the file with a list of classes to the `classes` folder. From b99eb5ecad1ea8d40be80ef821812de95f60b856 Mon Sep 17 00:00:00 2001 From: Nikita Stroganov Date: Wed, 25 Jan 2023 16:35:03 +0300 Subject: [PATCH 3/5] Fix compilation --- .../src/main/kotlin/org/utbot/contest/ContestEstimator.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt index c2aa44dce5..c594f49a23 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt @@ -208,7 +208,8 @@ enum class Tool { methodNameFilter: String?, statsForProject: StatsForProject, compiledTestDir: File, - classFqn: String + classFqn: String, + expectedExceptions: ExpectedExceptionsForClass ) { // EvoSuite has several phases, the variable below is responsible for assert generation // timeout. We want to give 10s for a big time budgets and timeLimit / 5 for small budgets. From 6ce7c0f3e591e82e284f42758ba36421a1ae9974 Mon Sep 17 00:00:00 2001 From: Nikita Stroganov Date: Wed, 25 Jan 2023 18:29:33 +0300 Subject: [PATCH 4/5] Fix compilation again --- .../src/main/kotlin/org/utbot/contest/ContestEstimator.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt index c594f49a23..119ceb16d9 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt @@ -589,6 +589,7 @@ fun parseExceptionsFile(exceptionsDescriptionFile: File): ExpectedExceptionsForP class ProjectToEstimate( val name: String, val classFQNs: List, + val expectedExceptions: ExpectedExceptionsForProject, private val jars: List, testCandidatesDir: File, unzippedJars: File From 4bc3566ae7f7958e087bce73acdc6790350c970e Mon Sep 17 00:00:00 2001 From: Nikita Stroganov Date: Fri, 27 Jan 2023 13:42:37 +0300 Subject: [PATCH 5/5] Refactor after review --- .../main/kotlin/org/utbot/contest/Contest.kt | 8 +-- .../org/utbot/contest/ContestEstimator.kt | 57 +------------------ .../org/utbot/contest/ExpectedExceptions.kt | 54 ++++++++++++++++++ 3 files changed, 61 insertions(+), 58 deletions(-) create mode 100644 utbot-junit-contest/src/main/kotlin/org/utbot/contest/ExpectedExceptions.kt diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt index 25213cf2ab..cba7acaef4 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt @@ -146,8 +146,8 @@ fun main(args: Array) { fuzzingRatio = 0.1, classpathString, runFromEstimator = false, - methodNameFilter = null, - expectedExceptions = ExpectedExceptionsForClass() + expectedExceptions = ExpectedExceptionsForClass(), + methodNameFilter = null ) println("${ContestMessage.READY}") } @@ -180,8 +180,8 @@ fun runGeneration( fuzzingRatio: Double, classpathString: String, runFromEstimator: Boolean, - methodNameFilter: String? = null, // For debug purposes you can specify method name - expectedExceptions: ExpectedExceptionsForClass + expectedExceptions: ExpectedExceptionsForClass, + methodNameFilter: String? = null // For debug purposes you can specify method name ): StatsForClass = runBlocking { val testsByMethod: MutableMap> = mutableMapOf() val currentContext = utContext diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt index 119ceb16d9..199bae753b 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt @@ -148,8 +148,8 @@ enum class Tool { fuzzingRatio, project.sootClasspathString, runFromEstimator = true, - methodNameFilter, - expectedExceptions + expectedExceptions, + methodNameFilter ) } catch (e: CancellationException) { logger.info { "[$classFqn] finished with CancellationException" } @@ -410,7 +410,7 @@ fun runEstimator( val project = ProjectToEstimate( name, classesFQN, - projectToExpectedExceptions[name]!!, + projectToExpectedExceptions.getValue(name), File(classpathDir, name).listFiles()!!.filter { it.toString().endsWith("jar") }, testCandidatesDir, unzippedJars @@ -535,57 +535,6 @@ private fun classNamesByJar(jar: File): List { .toList() } -class ExpectedExceptionsForMethod( - val exceptionNames: List -) - -class ExpectedExceptionsForClass { - - private val byMethodName: MutableMap = - mutableMapOf() - - fun getForMethod(methodName: String): ExpectedExceptionsForMethod = - byMethodName[methodName] ?: ExpectedExceptionsForMethod(listOf()) - - fun setForMethod(methodName: String, exceptionNames: List) { - byMethodName[methodName] = ExpectedExceptionsForMethod(exceptionNames) - } -} - -class ExpectedExceptionsForProject { - - private val byClassName: MutableMap = - mutableMapOf() - - fun getForClass(className: String): ExpectedExceptionsForClass = - byClassName[className] ?: ExpectedExceptionsForClass() - - fun setForClass(className: String, methodName: String, exceptionNames: List) { - val forClass = byClassName.getOrPut(className) { ExpectedExceptionsForClass() } - forClass.setForMethod(methodName, exceptionNames) - } -} - -fun parseExceptionsFile(exceptionsDescriptionFile: File): ExpectedExceptionsForProject { - if (!exceptionsDescriptionFile.exists()) { - return ExpectedExceptionsForProject() - } - - val forProject = ExpectedExceptionsForProject() - for (methodDescription in exceptionsDescriptionFile.readLines()) { - val methodFqn = methodDescription.substringBefore(':').trim() - val classFqn = methodFqn.substringBeforeLast('.') - val methodName = methodFqn.substringAfterLast('.') - val exceptionFqns = methodDescription.substringAfter(':') - .split(' ') - .map { it.trim() } - .filter { it.isNotBlank() } - forProject.setForClass(classFqn, methodName, exceptionFqns) - } - - return forProject -} - class ProjectToEstimate( val name: String, val classFQNs: List, diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ExpectedExceptions.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ExpectedExceptions.kt new file mode 100644 index 0000000000..ebf41a05a5 --- /dev/null +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ExpectedExceptions.kt @@ -0,0 +1,54 @@ +package org.utbot.contest + +import java.io.File + +class ExpectedExceptionsForMethod( + val exceptionNames: List +) + +class ExpectedExceptionsForClass { + + private val byMethodName: MutableMap = + mutableMapOf() + + fun getForMethod(methodName: String): ExpectedExceptionsForMethod = + byMethodName[methodName] ?: ExpectedExceptionsForMethod(listOf()) + + fun setForMethod(methodName: String, exceptionNames: List) { + byMethodName[methodName] = ExpectedExceptionsForMethod(exceptionNames) + } +} + +class ExpectedExceptionsForProject { + + private val byClassName: MutableMap = + mutableMapOf() + + fun getForClass(className: String): ExpectedExceptionsForClass = + byClassName[className] ?: ExpectedExceptionsForClass() + + fun setForClass(className: String, methodName: String, exceptionNames: List) { + val forClass = byClassName.getOrPut(className) { ExpectedExceptionsForClass() } + forClass.setForMethod(methodName, exceptionNames) + } +} + +fun parseExceptionsFile(exceptionsDescriptionFile: File): ExpectedExceptionsForProject { + if (!exceptionsDescriptionFile.exists()) { + return ExpectedExceptionsForProject() + } + + val forProject = ExpectedExceptionsForProject() + for (methodDescription in exceptionsDescriptionFile.readLines()) { + val methodFqn = methodDescription.substringBefore(':').trim() + val classFqn = methodFqn.substringBeforeLast('.') + val methodName = methodFqn.substringAfterLast('.') + val exceptionFqns = methodDescription.substringAfter(':') + .split(' ') + .map { it.trim() } + .filter { it.isNotBlank() } + forProject.setForClass(classFqn, methodName, exceptionFqns) + } + + return forProject +}