Path selector quality analysis

utbot-analytics/src/main/kotlin/org/utbot/ClassifierTrainer.kt

@@ -6,7 +6,7 @@ import org.utbot.metrics.ClassificationMetrics
 import org.utbot.models.ClassifierModel
 import org.utbot.models.loadModelFromJson
 import org.utbot.models.save
-import org.utbot.visual.ClassificationHtmlReport
+import org.utbot.visual.ClassificationHTMLReport
 import smile.classification.Classifier
 import smile.data.CategoricalEncoder
 import smile.data.DataFrame
@@ -20,7 +20,7 @@ private const val dataPath = "logs/stats.txt"
 private const val logDir = "logs"
 
 class ClassifierTrainer(data: DataFrame, val classifierModel: ClassifierModel = ClassifierModel.GBM) :
-        AbstractTrainer(data, savePcaVariance = true) {
+    AbstractTrainer(data, savePcaVariance = true) {
     private lateinit var metrics: ClassificationMetrics
     lateinit var model: Classifier<DoubleArray>
     val properties = Properties()
@@ -62,19 +62,27 @@ class ClassifierTrainer(data: DataFrame, val classifierModel: ClassifierModel =
         val xFrame = Formula.lhs(targetColumn).x(validationData)
         val x = xFrame.toArray(false, CategoricalEncoder.LEVEL)
 
-        metrics = ClassificationMetrics(classifierModel.name, model, Compose(transforms), actualLabel.map { it.toInt() }.toIntArray(), x)
+        metrics = ClassificationMetrics(
+            classifierModel.name,
+            model,
+            Compose(transforms),
+            actualLabel.map { it.toInt() }.toIntArray(),
+            x
+        )
     }
 
     override fun visualize() {
-        val report = ClassificationHtmlReport()
+        val report = ClassificationHTMLReport()
         report.run {
             addHeader(classifierModel.name, properties)
             addDataDistribution(formula.y(data).toDoubleArray())
             addClassDistribution(classSizesBeforeResampling)
             addClassDistribution(classSizesAfterResampling, before = false)
             addPCAPlot(pcaVarianceProportion, pcaCumulativeVarianceProportion)
-            addMetrics(metrics.acc, metrics.f1Macro, metrics.avgPredTime, metrics.precision.toDoubleArray(),
-                    metrics.recall.toDoubleArray())
+            addMetrics(
+                metrics.acc, metrics.f1Macro, metrics.avgPredTime, metrics.precision.toDoubleArray(),
+                metrics.recall.toDoubleArray()
+            )
             addConfusionMatrix(metrics.getNormalizedConfusionMatrix())
             save()
         }

utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysis.kt

@@ -0,0 +1,182 @@
+package org.utbot
+
+import org.jsoup.Jsoup
+import org.jsoup.nodes.Document
+import org.jsoup.select.Elements
+import org.utbot.visual.FigureBuilders
+import org.utbot.visual.HtmlBuilder
+import java.io.File
+import java.nio.file.Paths
+
+
+data class Coverage(
+    val misInstructions: Int,
+    val covInstruction: Int,
+    val misBranches: Int,
+    val covBranches: Int,
+    val time: Double = 0.0
+) {
+    fun getInstructionCoverage(): Double = when {
+        (this.misInstructions == 0 && this.covInstruction == 0) -> 0.0
+        (this.misInstructions == 0) -> 1.0
+        else -> this.covInstruction.toDouble() / (this.covInstruction + this.misInstructions)
+    }
+
+    fun getBranchesCoverage(): Double = when {
+        (this.misBranches == 0 && this.covBranches == 0) -> 0.0
+        (this.misBranches == 0) -> 1.0
+        else -> this.covBranches.toDouble() / (this.covBranches + this.misBranches)
+    }
+
+    fun getInstructions() = misInstructions + covInstruction
+}
+
+
+fun parseJacocoReport(path: String, classes: Set<String>): Pair<Map<String, Coverage>, Map<String, Coverage>> {
+    val perClassResult = mutableMapOf<String, Coverage>()
+    val perMethodResult = mutableMapOf<String, Coverage>()
+    val contestDocument: Document = Jsoup.parse(File("$path/index.html"), null)
+    val pkgRows: Elements = contestDocument.select("table")[0].select("tr")
+
+    for (i in 2 until pkgRows.size) {
+        val pkgHref = pkgRows[i].select("td")[0].select("a").attr("href")
+        val packageDocument: Document = Jsoup.parse(File("$path/$pkgHref"), null)
+        val classRows: Elements = packageDocument.select("table")[0].select("tr")
+
+        for (j in 2 until classRows.size) {
+            val classHref = classRows[j].select("td")[0].select("a").attr("href")
+            val className = pkgHref.replace("/index.html", "") + "." + classHref.split(".")[0]
+            if (!classes.contains(className)) {
+                continue
+            }
+
+            val classDocument: Document = Jsoup.parse(File("$path/${pkgHref.replace("index.html", classHref)}"), null)
+            val methodRows: Elements = classDocument.select("table")[0].select("tr")
+
+            val coverageInfos = methodRows[1].select("td")
+            val (misInstructions, covInstructions) = coverageInfos[1].text()?.replace(",", "")?.split(" of ")?.let {
+                val misInstructions = it[0].toInt()
+                val allInstructions = it[1].toInt()
+                misInstructions to (allInstructions - misInstructions)
+            } ?: (0 to 0)
+
+            val (misBranches, covBranches) = coverageInfos[3].text()?.replace(",", "")?.split(" of ")?.let {
+                val misBranches = it[0].toInt()
+                val allBranches = it[1].toInt()
+                misBranches to (allBranches - misBranches)
+            } ?: (0 to 0)
+
+            val name = pkgHref.replace("/index.html", "") + "." + classHref.split(".")[0]
+            perClassResult[name] = Coverage(misInstructions, covInstructions, misBranches, covBranches)
+
+            for (k in 2 until methodRows.size) {
+
+                val cols = methodRows[k].select("td")
+                val methodHref = methodRows[k].select("td")[0].select("a").attr("href")
+                val methodName = classHref.replace("/index.html", "." + methodHref.replace("html", cols[0].text()))
+
+                val instructions = cols[1].select("img")
+                val methodMisInstructions =
+                    instructions.getOrNull(0)?.attr("title")?.toString()?.replace(",", "")?.toInt()
+                        ?: 0
+                val methodCovInstructions =
+                    instructions.getOrNull(1)?.attr("title")?.toString()?.replace(",", "")?.toInt()
+                        ?: 0
+
+                val branches = cols[3].select("img")
+                val methodMisBranches = branches.getOrNull(0)?.attr("title")?.toString()?.replace(",", "")?.toInt() ?: 0
+                val methodCovBranches = branches.getOrNull(1)?.attr("title")?.toString()?.replace(",", "")?.toInt() ?: 0
+
+                if (methodMisInstructions == 0 && methodCovBranches == 0 && methodCovInstructions == 0 && methodMisBranches == 0) continue
+                perMethodResult[methodName] =
+                    Coverage(methodMisInstructions, methodCovInstructions, methodMisBranches, methodCovBranches)
+            }
+        }
+    }
+
+    return perClassResult to perMethodResult
+}
+
+
+fun main() {
+    val htmlBuilder = HtmlBuilder()
+
+    val classes = mutableSetOf<String>()
+    File(QualityAnalysisConfig.classesList).inputStream().bufferedReader().forEachLine { classes.add(it) }
+
+    // Parse data
+    val jacocoCoverage = QualityAnalysisConfig.selectors.map {
+        it to parseJacocoReport("eval/jacoco/${QualityAnalysisConfig.project}/${it}", classes).first
+    }
+
+    // Instruction coverage report (sum coverages percentages / classNum)
+    val instructionMetrics = jacocoCoverage.map { jacoco ->
+        jacoco.first to jacoco.second.map { it.value.getInstructionCoverage() }
+    }
+    htmlBuilder.addHeader("Instructions coverage (sum coverages percentages / classNum)")
+    instructionMetrics.forEach {
+        htmlBuilder.addText("Mean(Instruction (${it.first} model)) =${it.second.sum() / it.second.size}")
+    }
+    htmlBuilder.addFigure(
+        FigureBuilders.buildBoxPlot(
+            instructionMetrics.map { it.second.map { _ -> "Instructions (${it.first} model)" } }.flatten()
+                .toTypedArray(),
+            instructionMetrics.map { it.second }.flatten().toDoubleArray(),
+            title = "Coverage",
+            xLabel = "Instructions",
+            yLabel = "Count"
+        )
+    )
+
+    // Instruction coverage report (sum covered instructions / sum instructions)
+    htmlBuilder.addHeader("Instructions coverage(sum covered instructions / sum instructions)")
+    val covInstruction = jacocoCoverage.map { jacoco ->
+        jacoco.first to jacoco.second.map { it.value.covInstruction }
+    }
+    val instructions = jacocoCoverage.map { jacoco ->
+        jacoco.first to jacoco.second.map { it.value.getInstructions() }
+    }.toMap()
+    covInstruction.forEach {
+        htmlBuilder.addText(
+            "Mean(Instruction (${it.first} model)) =${
+                it.second.sum().toDouble() / (instructions[it.first]?.sum()?.toDouble() ?: 0.0)
+            }"
+        )
+    }
+    htmlBuilder.addFigure(
+        FigureBuilders.buildBoxPlot(
+            covInstruction.map { it.second.map { _ -> "Instructions (${it.first} model)" } }.flatten().toTypedArray(),
+            covInstruction.map { it.second }.flatten().map { it.toDouble() }.toDoubleArray(),
+            title = "Coverage",
+            xLabel = "Instructions",
+            yLabel = "Count"
+        )
+    )
+
+    // Branches coverage report
+    val branchesMetrics = jacocoCoverage.map { jacoco ->
+        jacoco.first to jacoco.second.map { it.value.getBranchesCoverage() }
+    }
+    htmlBuilder.addHeader("Branches coverage")
+    branchesMetrics.forEach {
+        htmlBuilder.addText("Mean(Branches (${it.first} model)) =${it.second.sum() / it.second.size}")
+    }
+    htmlBuilder.addFigure(
+        FigureBuilders.buildBoxPlot(
+            branchesMetrics.map { it.second.map { it2 -> "Branches (${it.first} model)" } }.flatten().toTypedArray(),
+            branchesMetrics.map { it.second }.flatten().toDoubleArray(),
+            title = "Coverage",
+            xLabel = "Branches",
+            yLabel = "Count"
+        )
+    )
+
+    // Save report
+    htmlBuilder.saveHTML(
+        Paths.get(
+            QualityAnalysisConfig.outputDir,
+            QualityAnalysisConfig.project,
+            "test.html"
+        ).toFile().absolutePath
+    )
+}

utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysisConfig.kt

@@ -0,0 +1,21 @@
+package org.utbot
+
+import java.io.FileInputStream
+import java.util.Properties
+
+object QualityAnalysisConfig {
+
+    private const val configPath = "utbot-analytics/src/main/resources/config.properties"
+
+    private val properties = Properties().also { props ->
+        FileInputStream(configPath).use { inputStream ->
+            props.load(inputStream)
+        }
+    }
+
+    val project: String = properties.getProperty("project")
+    val selectors: List<String> = properties.getProperty("selectors").split(",")
+    val covStatistics: List<String> = properties.getProperty("covStatistics").split(",")
+    val outputDir: String = "eval/res"
+    val classesList: String = "utbot-junit-contest/src/main/resources/classes/${project}/list"
+}

utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysisV2.kt

@@ -0,0 +1,70 @@
+package org.utbot
+
+import org.jsoup.Jsoup
+import org.jsoup.nodes.Document
+import org.jsoup.select.Elements
+import org.utbot.visual.FigureBuilders
+import org.utbot.visual.HtmlBuilder
+import java.io.File
+import java.nio.file.Paths
+
+
+fun parseReport(path: String, classes: Set<String>): Map<String, Double> {
+    val result = mutableMapOf<String, Double>()
+    val contestDocument: Document = Jsoup.parse(File("$path\\index.html"), null)
+    val packageRows: Elements = contestDocument.select("table")[1].select("tr")
+    for (i in 1 until packageRows.size) {
+        val packageHref = packageRows[i].select("td")[0].select("a").attr("href")
+        val packageDocument: Document = Jsoup.parse(File("$path\\$packageHref"), null)
+        val clsRows: Elements = packageDocument.select("table")[1].select("tr")
+
+        for (j in 1 until clsRows.size) {
+            val clsName =
+                clsRows[j].select("td")[0].select("a").attr("href").replace(".classes/", "").replace(".html", "")
+            val fullName = packageHref.replace("/index.html", ".$clsName")
+
+            if (classes.contains(fullName)) {
+                result.put(fullName, clsRows[j].select("td")[3].select("span")[0].text().replace("%", "").toDouble())
+            }
+        }
+    }
+
+    return result
+}
+
+
+fun main() {
+    val htmlBuilder = HtmlBuilder()
+
+    val classes = mutableSetOf<String>()
+    File(QualityAnalysisConfig.classesList).inputStream().bufferedReader().forEachLine { classes.add(it) }
+
+    // Parse data
+    val jacocoCoverage = QualityAnalysisConfig.selectors.map {
+        it to parseReport("eval/jacoco/${QualityAnalysisConfig.project}/${it}", classes)
+    }
+
+    // Coverage report
+    htmlBuilder.addHeader("Line coverage")
+    jacocoCoverage.forEach {
+        htmlBuilder.addText("Mean(Line (${it.first} model)) =" + it.second.map { it.value }.sum() / it.second.size)
+    }
+    htmlBuilder.addFigure(
+        FigureBuilders.buildBoxPlot(
+            jacocoCoverage.map { it.second.map { it2 -> "Line (${it.first} model)" } }.flatten().toTypedArray(),
+            jacocoCoverage.map { it.second.map { it.value } }.flatten().toDoubleArray(),
+            title = "Coverage",
+            xLabel = "Instructions",
+            yLabel = "Count"
+        )
+    )
+
+    // Save report
+    htmlBuilder.saveHTML(
+        Paths.get(
+            QualityAnalysisConfig.outputDir,
+            QualityAnalysisConfig.project,
+            QualityAnalysisConfig.selectors.joinToString("_")
+        ).toFile().absolutePath
+    )
+}