diff --git a/utbot-analytics/src/main/kotlin/org/utbot/ClassifierTrainer.kt b/utbot-analytics/src/main/kotlin/org/utbot/ClassifierTrainer.kt index c3bba8ff70..b99d3e9838 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/ClassifierTrainer.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/ClassifierTrainer.kt @@ -6,7 +6,7 @@ import org.utbot.metrics.ClassificationMetrics import org.utbot.models.ClassifierModel import org.utbot.models.loadModelFromJson import org.utbot.models.save -import org.utbot.visual.ClassificationHtmlReport +import org.utbot.visual.ClassificationHTMLReport import smile.classification.Classifier import smile.data.CategoricalEncoder import smile.data.DataFrame @@ -20,7 +20,7 @@ private const val dataPath = "logs/stats.txt" private const val logDir = "logs" class ClassifierTrainer(data: DataFrame, val classifierModel: ClassifierModel = ClassifierModel.GBM) : - AbstractTrainer(data, savePcaVariance = true) { + AbstractTrainer(data, savePcaVariance = true) { private lateinit var metrics: ClassificationMetrics lateinit var model: Classifier val properties = Properties() @@ -62,19 +62,27 @@ class ClassifierTrainer(data: DataFrame, val classifierModel: ClassifierModel = val xFrame = Formula.lhs(targetColumn).x(validationData) val x = xFrame.toArray(false, CategoricalEncoder.LEVEL) - metrics = ClassificationMetrics(classifierModel.name, model, Compose(transforms), actualLabel.map { it.toInt() }.toIntArray(), x) + metrics = ClassificationMetrics( + classifierModel.name, + model, + Compose(transforms), + actualLabel.map { it.toInt() }.toIntArray(), + x + ) } override fun visualize() { - val report = ClassificationHtmlReport() + val report = ClassificationHTMLReport() report.run { addHeader(classifierModel.name, properties) addDataDistribution(formula.y(data).toDoubleArray()) addClassDistribution(classSizesBeforeResampling) addClassDistribution(classSizesAfterResampling, before = false) addPCAPlot(pcaVarianceProportion, pcaCumulativeVarianceProportion) - addMetrics(metrics.acc, metrics.f1Macro, metrics.avgPredTime, metrics.precision.toDoubleArray(), - metrics.recall.toDoubleArray()) + addMetrics( + metrics.acc, metrics.f1Macro, metrics.avgPredTime, metrics.precision.toDoubleArray(), + metrics.recall.toDoubleArray() + ) addConfusionMatrix(metrics.getNormalizedConfusionMatrix()) save() } diff --git a/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysis.kt b/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysis.kt new file mode 100644 index 0000000000..c55db51e19 --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysis.kt @@ -0,0 +1,182 @@ +package org.utbot + +import org.jsoup.Jsoup +import org.jsoup.nodes.Document +import org.jsoup.select.Elements +import org.utbot.visual.FigureBuilders +import org.utbot.visual.HtmlBuilder +import java.io.File +import java.nio.file.Paths + + +data class Coverage( + val misInstructions: Int, + val covInstruction: Int, + val misBranches: Int, + val covBranches: Int, + val time: Double = 0.0 +) { + fun getInstructionCoverage(): Double = when { + (this.misInstructions == 0 && this.covInstruction == 0) -> 0.0 + (this.misInstructions == 0) -> 1.0 + else -> this.covInstruction.toDouble() / (this.covInstruction + this.misInstructions) + } + + fun getBranchesCoverage(): Double = when { + (this.misBranches == 0 && this.covBranches == 0) -> 0.0 + (this.misBranches == 0) -> 1.0 + else -> this.covBranches.toDouble() / (this.covBranches + this.misBranches) + } + + fun getInstructions() = misInstructions + covInstruction +} + + +fun parseJacocoReport(path: String, classes: Set): Pair, Map> { + val perClassResult = mutableMapOf() + val perMethodResult = mutableMapOf() + val contestDocument: Document = Jsoup.parse(File("$path/index.html"), null) + val pkgRows: Elements = contestDocument.select("table")[0].select("tr") + + for (i in 2 until pkgRows.size) { + val pkgHref = pkgRows[i].select("td")[0].select("a").attr("href") + val packageDocument: Document = Jsoup.parse(File("$path/$pkgHref"), null) + val classRows: Elements = packageDocument.select("table")[0].select("tr") + + for (j in 2 until classRows.size) { + val classHref = classRows[j].select("td")[0].select("a").attr("href") + val className = pkgHref.replace("/index.html", "") + "." + classHref.split(".")[0] + if (!classes.contains(className)) { + continue + } + + val classDocument: Document = Jsoup.parse(File("$path/${pkgHref.replace("index.html", classHref)}"), null) + val methodRows: Elements = classDocument.select("table")[0].select("tr") + + val coverageInfos = methodRows[1].select("td") + val (misInstructions, covInstructions) = coverageInfos[1].text()?.replace(",", "")?.split(" of ")?.let { + val misInstructions = it[0].toInt() + val allInstructions = it[1].toInt() + misInstructions to (allInstructions - misInstructions) + } ?: (0 to 0) + + val (misBranches, covBranches) = coverageInfos[3].text()?.replace(",", "")?.split(" of ")?.let { + val misBranches = it[0].toInt() + val allBranches = it[1].toInt() + misBranches to (allBranches - misBranches) + } ?: (0 to 0) + + val name = pkgHref.replace("/index.html", "") + "." + classHref.split(".")[0] + perClassResult[name] = Coverage(misInstructions, covInstructions, misBranches, covBranches) + + for (k in 2 until methodRows.size) { + + val cols = methodRows[k].select("td") + val methodHref = methodRows[k].select("td")[0].select("a").attr("href") + val methodName = classHref.replace("/index.html", "." + methodHref.replace("html", cols[0].text())) + + val instructions = cols[1].select("img") + val methodMisInstructions = + instructions.getOrNull(0)?.attr("title")?.toString()?.replace(",", "")?.toInt() + ?: 0 + val methodCovInstructions = + instructions.getOrNull(1)?.attr("title")?.toString()?.replace(",", "")?.toInt() + ?: 0 + + val branches = cols[3].select("img") + val methodMisBranches = branches.getOrNull(0)?.attr("title")?.toString()?.replace(",", "")?.toInt() ?: 0 + val methodCovBranches = branches.getOrNull(1)?.attr("title")?.toString()?.replace(",", "")?.toInt() ?: 0 + + if (methodMisInstructions == 0 && methodCovBranches == 0 && methodCovInstructions == 0 && methodMisBranches == 0) continue + perMethodResult[methodName] = + Coverage(methodMisInstructions, methodCovInstructions, methodMisBranches, methodCovBranches) + } + } + } + + return perClassResult to perMethodResult +} + + +fun main() { + val htmlBuilder = HtmlBuilder() + + val classes = mutableSetOf() + File(QualityAnalysisConfig.classesList).inputStream().bufferedReader().forEachLine { classes.add(it) } + + // Parse data + val jacocoCoverage = QualityAnalysisConfig.selectors.map { + it to parseJacocoReport("eval/jacoco/${QualityAnalysisConfig.project}/${it}", classes).first + } + + // Instruction coverage report (sum coverages percentages / classNum) + val instructionMetrics = jacocoCoverage.map { jacoco -> + jacoco.first to jacoco.second.map { it.value.getInstructionCoverage() } + } + htmlBuilder.addHeader("Instructions coverage (sum coverages percentages / classNum)") + instructionMetrics.forEach { + htmlBuilder.addText("Mean(Instruction (${it.first} model)) =${it.second.sum() / it.second.size}") + } + htmlBuilder.addFigure( + FigureBuilders.buildBoxPlot( + instructionMetrics.map { it.second.map { _ -> "Instructions (${it.first} model)" } }.flatten() + .toTypedArray(), + instructionMetrics.map { it.second }.flatten().toDoubleArray(), + title = "Coverage", + xLabel = "Instructions", + yLabel = "Count" + ) + ) + + // Instruction coverage report (sum covered instructions / sum instructions) + htmlBuilder.addHeader("Instructions coverage(sum covered instructions / sum instructions)") + val covInstruction = jacocoCoverage.map { jacoco -> + jacoco.first to jacoco.second.map { it.value.covInstruction } + } + val instructions = jacocoCoverage.map { jacoco -> + jacoco.first to jacoco.second.map { it.value.getInstructions() } + }.toMap() + covInstruction.forEach { + htmlBuilder.addText( + "Mean(Instruction (${it.first} model)) =${ + it.second.sum().toDouble() / (instructions[it.first]?.sum()?.toDouble() ?: 0.0) + }" + ) + } + htmlBuilder.addFigure( + FigureBuilders.buildBoxPlot( + covInstruction.map { it.second.map { _ -> "Instructions (${it.first} model)" } }.flatten().toTypedArray(), + covInstruction.map { it.second }.flatten().map { it.toDouble() }.toDoubleArray(), + title = "Coverage", + xLabel = "Instructions", + yLabel = "Count" + ) + ) + + // Branches coverage report + val branchesMetrics = jacocoCoverage.map { jacoco -> + jacoco.first to jacoco.second.map { it.value.getBranchesCoverage() } + } + htmlBuilder.addHeader("Branches coverage") + branchesMetrics.forEach { + htmlBuilder.addText("Mean(Branches (${it.first} model)) =${it.second.sum() / it.second.size}") + } + htmlBuilder.addFigure( + FigureBuilders.buildBoxPlot( + branchesMetrics.map { it.second.map { it2 -> "Branches (${it.first} model)" } }.flatten().toTypedArray(), + branchesMetrics.map { it.second }.flatten().toDoubleArray(), + title = "Coverage", + xLabel = "Branches", + yLabel = "Count" + ) + ) + + // Save report + htmlBuilder.saveHTML( + Paths.get( + QualityAnalysisConfig.outputDir, + QualityAnalysisConfig.project, + "test.html" + ).toFile().absolutePath + ) +} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysisConfig.kt b/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysisConfig.kt new file mode 100644 index 0000000000..d7e049f93d --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysisConfig.kt @@ -0,0 +1,21 @@ +package org.utbot + +import java.io.FileInputStream +import java.util.Properties + +object QualityAnalysisConfig { + + private const val configPath = "utbot-analytics/src/main/resources/config.properties" + + private val properties = Properties().also { props -> + FileInputStream(configPath).use { inputStream -> + props.load(inputStream) + } + } + + val project: String = properties.getProperty("project") + val selectors: List = properties.getProperty("selectors").split(",") + val covStatistics: List = properties.getProperty("covStatistics").split(",") + val outputDir: String = "eval/res" + val classesList: String = "utbot-junit-contest/src/main/resources/classes/${project}/list" +} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysisV2.kt b/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysisV2.kt new file mode 100644 index 0000000000..4b53d78afa --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/QualityAnalysisV2.kt @@ -0,0 +1,70 @@ +package org.utbot + +import org.jsoup.Jsoup +import org.jsoup.nodes.Document +import org.jsoup.select.Elements +import org.utbot.visual.FigureBuilders +import org.utbot.visual.HtmlBuilder +import java.io.File +import java.nio.file.Paths + + +fun parseReport(path: String, classes: Set): Map { + val result = mutableMapOf() + val contestDocument: Document = Jsoup.parse(File("$path\\index.html"), null) + val packageRows: Elements = contestDocument.select("table")[1].select("tr") + for (i in 1 until packageRows.size) { + val packageHref = packageRows[i].select("td")[0].select("a").attr("href") + val packageDocument: Document = Jsoup.parse(File("$path\\$packageHref"), null) + val clsRows: Elements = packageDocument.select("table")[1].select("tr") + + for (j in 1 until clsRows.size) { + val clsName = + clsRows[j].select("td")[0].select("a").attr("href").replace(".classes/", "").replace(".html", "") + val fullName = packageHref.replace("/index.html", ".$clsName") + + if (classes.contains(fullName)) { + result.put(fullName, clsRows[j].select("td")[3].select("span")[0].text().replace("%", "").toDouble()) + } + } + } + + return result +} + + +fun main() { + val htmlBuilder = HtmlBuilder() + + val classes = mutableSetOf() + File(QualityAnalysisConfig.classesList).inputStream().bufferedReader().forEachLine { classes.add(it) } + + // Parse data + val jacocoCoverage = QualityAnalysisConfig.selectors.map { + it to parseReport("eval/jacoco/${QualityAnalysisConfig.project}/${it}", classes) + } + + // Coverage report + htmlBuilder.addHeader("Line coverage") + jacocoCoverage.forEach { + htmlBuilder.addText("Mean(Line (${it.first} model)) =" + it.second.map { it.value }.sum() / it.second.size) + } + htmlBuilder.addFigure( + FigureBuilders.buildBoxPlot( + jacocoCoverage.map { it.second.map { it2 -> "Line (${it.first} model)" } }.flatten().toTypedArray(), + jacocoCoverage.map { it.second.map { it.value } }.flatten().toDoubleArray(), + title = "Coverage", + xLabel = "Instructions", + yLabel = "Count" + ) + ) + + // Save report + htmlBuilder.saveHTML( + Paths.get( + QualityAnalysisConfig.outputDir, + QualityAnalysisConfig.project, + QualityAnalysisConfig.selectors.joinToString("_") + ).toFile().absolutePath + ) +} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/StmtCoverageReport.kt b/utbot-analytics/src/main/kotlin/org/utbot/StmtCoverageReport.kt new file mode 100644 index 0000000000..b983c30e9b --- /dev/null +++ b/utbot-analytics/src/main/kotlin/org/utbot/StmtCoverageReport.kt @@ -0,0 +1,180 @@ +package org.utbot + +import org.apache.commons.io.FileUtils +import org.utbot.visual.FigureBuilders +import org.utbot.visual.HtmlBuilder +import smile.read +import java.io.File +import java.nio.file.Paths +import kotlin.random.Random + + +data class Statistics( + val perMethod: Map>, + val perClass: Map>, + val perProject: List +) + + +fun getStatistics(path: String, classes: Set): Statistics { + var projectTotalStmts = 0.0 + var projectMinStartTime = Double.MAX_VALUE + + val rawStatistics = File(path).listFiles()?.filter { it.extension == "txt" && it.readLines().size > 1 }?.map { + val data = read.csv(it.absolutePath) + it.nameWithoutExtension to data.toArray() + }?.toMap() ?: emptyMap() + + val statisticsPerMethod = rawStatistics.map { methodData -> + val startTime = methodData.value[0][0] + val value = methodData.value.map { + doubleArrayOf(it[0] - startTime, it[1] / it[2]) // time, percent + }.sortedBy { it[0] } + + methodData.key to value + }.toMap() + + val statisticsPerClass = classes.map { cls -> + var minStartTime = Double.MAX_VALUE + var totalStmts = 0.0 + val filteredStatistics = rawStatistics.filter { it.key.contains(cls) } + + filteredStatistics.forEach { + minStartTime = minOf(minStartTime, it.value[0][0]) + totalStmts += it.value.getOrNull(1)?.get(2) ?: 0.0 + } + projectTotalStmts += totalStmts + projectMinStartTime = minOf(minStartTime, projectMinStartTime) + + var prevCov = 0.0 + cls to filteredStatistics.toList().sortedBy { it.second[0][0] }.flatMap { methodData -> + val value = methodData.second.map { + doubleArrayOf(it[0] - minStartTime, (it[1] + prevCov) / totalStmts) + } + + prevCov += methodData.second.last()[1] + value + }.sortedBy { it[0] } + }.toMap() + + var prevCov = 0.0 + val statisticsPerProject = rawStatistics + .filter { classes.any { it2 -> it.key.contains(it2) } } + .toList() + .sortedBy { it.second[0][0] } + .flatMap { methodData -> + val value = methodData.second.map { + doubleArrayOf(it[0] - projectMinStartTime, (it[1] + prevCov) / projectTotalStmts) // time, percent + } + + prevCov += methodData.second.last()[1] + value + }.sortedBy { it[0] } + + return Statistics(statisticsPerMethod, statisticsPerClass, statisticsPerProject) +} + +fun main() { + // Processed classes + val classes = mutableSetOf() + File(QualityAnalysisConfig.classesList).inputStream().bufferedReader().forEachLine { classes.add(it) } + + // Prepare folder + val projectDataPath = "${QualityAnalysisConfig.outputDir}/report/${QualityAnalysisConfig.project}/${ + QualityAnalysisConfig.selectors.joinToString("_") + }" + File(projectDataPath).deleteRecursively() + classes.forEach { File(projectDataPath, it).mkdirs() } + File(projectDataPath, "css").mkdirs() + listOf("css/coverage.css", "css/highlight-idea.css").forEach { + FileUtils.copyInputStreamToFile( + Statistics::class.java.classLoader.getResourceAsStream(it), + Paths.get(projectDataPath, it).toFile() + ) + } + + // Parse statistics + val selectors = QualityAnalysisConfig.selectors + val statistics = QualityAnalysisConfig.covStatistics.map { getStatistics(it, classes) } + val colors = QualityAnalysisConfig.selectors.map { + val rnd = Random.Default + "rgb(${rnd.nextInt(256)}, ${rnd.nextInt(256)}, ${rnd.nextInt(256)})" + } + + val htmlBuilderForProject = HtmlBuilder( + pathToStyle = listOf("css/coverage.css", "css/highlight-idea.css"), + pathToJs = listOf("https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.5.1/highlight.min.js") + ) + htmlBuilderForProject.addTable( + statistics.map { it.perClass.map { it.key to (it.value.lastOrNull()?.lastOrNull() ?: 0.0) * 100 }.toMap() }, + selectors, + colors, + "class" + ) + htmlBuilderForProject.addFigure( + FigureBuilders.buildSeveralLinesPlot( + statistics.map { it.perProject.map { it[0] / 1e9 }.toDoubleArray() }, + statistics.map { it.perProject.map { it[1] * 100 }.toDoubleArray() }, + colors, + selectors, + xLabel = "Time (sec)", + yLabel = "Coverage (%)", + title = QualityAnalysisConfig.project + ) + ) + htmlBuilderForProject.saveHTML(File(projectDataPath, "index.html").toString()) + + + classes.forEach { cls -> + val outputDir = File(projectDataPath, cls) + val htmlBuilderForCls = HtmlBuilder( + pathToStyle = listOf("../css/coverage.css", "../css/highlight-idea.css"), + pathToJs = listOf("https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.5.1/highlight.min.js") + ) + val filteredStatistics = statistics.map { + it.perMethod.filter { it.key.contains(cls) } + } + + htmlBuilderForCls.addTable( + filteredStatistics.map { it.map { it.key to it.value.last().last() * 100 }.toMap() }, + selectors, + colors, + "method" + ) + htmlBuilderForCls.addFigure( + FigureBuilders.buildSeveralLinesPlot( + statistics.map { it.perClass[cls]?.map { it[0] / 1e9 }?.toDoubleArray() ?: doubleArrayOf() }, + statistics.map { it.perClass[cls]?.map { it[1] * 100 }?.toDoubleArray() ?: doubleArrayOf() }, + colors, + selectors, + xLabel = "Time (sec)", + yLabel = "Coverage (%)", + title = cls + ) + ) + + File(outputDir.toString()).mkdir() + htmlBuilderForCls.saveHTML(File(outputDir, "index.html").toString()) + + filteredStatistics.first().keys.forEach { method -> + val htmlBuilderForMethod = HtmlBuilder( + pathToStyle = listOf("../../css/coverage.css", "../../css/highlight-idea.css"), + pathToJs = listOf("https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.5.1/highlight.min.js") + ) + htmlBuilderForMethod.addFigure( + FigureBuilders.buildSeveralLinesPlot( + filteredStatistics.map { it[method]?.map { it[0] / 1e9 }?.toDoubleArray() ?: doubleArrayOf() }, + filteredStatistics.map { it[method]?.map { it[1] * 100 }?.toDoubleArray() ?: doubleArrayOf() }, + colors, + selectors, + xLabel = "Time (sec)", + yLabel = "Coverage (%)", + title = method + ) + ) + + File(outputDir, method).mkdirs() + htmlBuilderForMethod.saveHTML(File(outputDir, "${method}/index.html").toString()) + } + } +} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/visual/AbstractHtmlReport.kt b/utbot-analytics/src/main/kotlin/org/utbot/visual/AbstractHtmlReport.kt index 9f1673a57e..da505812b0 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/visual/AbstractHtmlReport.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/visual/AbstractHtmlReport.kt @@ -13,7 +13,7 @@ abstract class AbstractHtmlReport(bodyWidth: Int = 600) { "logs/Report_" + dateTimeFormatter.format(LocalDateTime.now()) + ".html" fun save(filename: String = nameWithDate()) { - builder.saveHtml(filename) + builder.saveHTML(filename) } } diff --git a/utbot-analytics/src/main/kotlin/org/utbot/visual/ClassificationHtmlReport.kt b/utbot-analytics/src/main/kotlin/org/utbot/visual/ClassificationHtmlReport.kt index 9e2784093b..9676fbc543 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/visual/ClassificationHtmlReport.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/visual/ClassificationHtmlReport.kt @@ -2,50 +2,34 @@ package org.utbot.visual import java.time.LocalDateTime import java.time.format.DateTimeFormatter -import java.util.Properties -import tech.tablesaw.plotly.components.Figure +import java.util.* -class ClassificationHtmlReport : AbstractHtmlReport() { - private var figuresNum = 0 +class ClassificationHTMLReport { + private val builder = HtmlBuilder() fun addHeader(modelName: String, properties: Properties) { - val currentDateTime = LocalDateTime.now().format(DateTimeFormatter.ofPattern("dd-MM-yyyy HH:mm:ss")) - builder.addRawHTML("

Model : ${modelName}

") - builder.addRawHTML("

$currentDateTime

") - builder.addRawHTML("

Hyperparameters:

") - for (property in properties) { - builder.addText("${property.key} : ${property.value}") - } - } - - private fun addFigure(figure: Figure) { - builder.addRawHTML("
") - builder.addRawHTML(figure.asJavascript("plot$figuresNum")) - builder.addRawHTML("
") - figuresNum++ + builder.addHeader(modelName, properties) } fun addDataDistribution(y: DoubleArray, threshold: Double = 1000.0) { - val (data, notPresented) = y.partition { x -> x <= threshold } val rawHistogram = FigureBuilders.buildHistogram( - data.toDoubleArray(), + y.filter { x -> x <= threshold }.toDoubleArray(), xLabel = "ms", yLabel = "Number of samples", title = "Raw data distribution" ) - addFigure(rawHistogram) + builder.addFigure(rawHistogram) builder.addText("Number of samples: ${y.size}") - if (notPresented.isNotEmpty()) { - builder.addText( - "And ${notPresented.size} more samples longer than $threshold ms are not presented " + - "in the raw data distribution plot \n\n" - ) - } + builder.addText( + "And ${ + y.filter { x -> x > threshold }.size + } more samples longer than $threshold ms are not presented in the raw data distribution plot \n\n" + ) } fun addConfusionMatrix(confusionMatrix: Array) { - addFigure( + builder.addFigure( FigureBuilders.buildHeatmap( Array(confusionMatrix.size) { it }, Array(confusionMatrix.size) { it }, @@ -61,7 +45,7 @@ class ClassificationHtmlReport : AbstractHtmlReport() { var title = "Class distribution after resampling" if (before) title = "Class distribution before resampling" - addFigure( + builder.addFigure( FigureBuilders.buildBarPlot( Array(classSizes.size) { it }, classSizes, @@ -73,7 +57,7 @@ class ClassificationHtmlReport : AbstractHtmlReport() { } fun addPCAPlot(variance: DoubleArray, cumulativeVariance: DoubleArray) { - addFigure( + builder.addFigure( FigureBuilders.buildTwoLinesPlot( variance, cumulativeVariance, @@ -99,4 +83,17 @@ class ClassificationHtmlReport : AbstractHtmlReport() { ) } } + + fun save(filename: String = "default") { + if (filename == "default") { + val filename = "logs/Classification_Report_" + + LocalDateTime.now() + .format(DateTimeFormatter.ofPattern("dd-MM-yyyy_HH-mm-ss")) + + ".html" + builder.saveHTML(filename) + } else { + builder.saveHTML(filename) + } + } + } \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/visual/FigureBuilders.kt b/utbot-analytics/src/main/kotlin/org/utbot/visual/FigureBuilders.kt index 445934e513..8cfc0ddd15 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/visual/FigureBuilders.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/visual/FigureBuilders.kt @@ -10,37 +10,37 @@ import tech.tablesaw.plotly.traces.* class FigureBuilders { companion object { private fun getXYLayout( - xLabel: String = "X", - yLabel: String = "Y", - title: String = "Plot" + xLabel: String = "X", + yLabel: String = "Y", + title: String = "Plot" ): Layout { return Layout.builder() - .title(title) - .xAxis(Axis.builder().title(xLabel).build()) - .yAxis(Axis.builder().title(yLabel).build()) - .build() + .title(title) + .xAxis(Axis.builder().title(xLabel).build()) + .yAxis(Axis.builder().title(yLabel).build()) + .build() } private fun getXYZLayout( - xLabel: String = "X", - yLabel: String = "Y", - zLabel: String = "Y", - title: String = "Plot" + xLabel: String = "X", + yLabel: String = "Y", + zLabel: String = "Y", + title: String = "Plot" ): Layout { return Layout.builder() - .title(title) - .xAxis(Axis.builder().title(xLabel).build()) - .yAxis(Axis.builder().title(yLabel).build()) - .zAxis(Axis.builder().title(zLabel).build()) - .build() + .title(title) + .xAxis(Axis.builder().title(xLabel).build()) + .yAxis(Axis.builder().title(yLabel).build()) + .zAxis(Axis.builder().title(zLabel).build()) + .build() } fun buildScatterPlot( - x: DoubleArray, - y: DoubleArray, - xLabel: String = "X", - yLabel: String = "Y", - title: String = "Scatter plot" + x: DoubleArray, + y: DoubleArray, + xLabel: String = "X", + yLabel: String = "Y", + title: String = "Scatter plot" ): Figure { val layout = getXYLayout(xLabel, yLabel, title) val trace: Trace = ScatterTrace.builder(x, y).build() @@ -49,10 +49,10 @@ class FigureBuilders { } fun buildHistogram( - data: DoubleArray, - xLabel: String = "X", - yLabel: String = "Y", - title: String = "Histogram" + data: DoubleArray, + xLabel: String = "X", + yLabel: String = "Y", + title: String = "Histogram" ): Figure { val layout = getXYLayout(xLabel, yLabel, title) val trace: Trace = HistogramTrace.builder(data).build() @@ -61,11 +61,11 @@ class FigureBuilders { } fun build2DHistogram( - x: DoubleArray, - y: DoubleArray, - xLabel: String = "X", - yLabel: String = "Y", - title: String = "Histogram 2D" + x: DoubleArray, + y: DoubleArray, + xLabel: String = "X", + yLabel: String = "Y", + title: String = "Histogram 2D" ): Figure { val layout = getXYLayout(xLabel, yLabel, title) val trace: Trace = Histogram2DTrace.builder(x, y).build() @@ -74,11 +74,11 @@ class FigureBuilders { } fun buildBarPlot( - x: Array, - y: DoubleArray, - xLabel: String = "X", - yLabel: String = "Y", - title: String = "BarPlot" + x: Array, + y: DoubleArray, + xLabel: String = "X", + yLabel: String = "Y", + title: String = "BarPlot" ): Figure { val layout = getXYLayout(xLabel, yLabel, title) val trace: Trace = BarTrace.builder(x, y).build() @@ -86,12 +86,12 @@ class FigureBuilders { } fun buildHeatmap( - x: Array, - y: Array, - z: Array, - xLabel: String = "X", - yLabel: String = "Y", - title: String = "Heatmap" + x: Array, + y: Array, + z: Array, + xLabel: String = "X", + yLabel: String = "Y", + title: String = "Heatmap" ): Figure { val layout = getXYLayout(xLabel, yLabel, title) val trace: Trace = HeatmapTrace.builder(x, y, z).build() @@ -99,11 +99,11 @@ class FigureBuilders { } fun buildLinePlot( - x: DoubleArray, - y: DoubleArray, - xLabel: String = "X", - yLabel: String = "Y", - title: String = "Line plot" + x: DoubleArray, + y: DoubleArray, + xLabel: String = "X", + yLabel: String = "Y", + title: String = "Line plot" ): Figure { val layout = getXYLayout(xLabel, yLabel, title) val trace: Trace = ScatterTrace.builder(x, y).mode(ScatterTrace.Mode.LINE_AND_MARKERS).build() @@ -112,35 +112,55 @@ class FigureBuilders { } fun buildTwoLinesPlot( - y1: DoubleArray, - y2: DoubleArray, - xLabel: String = "X", - yLabel: String = "Y", - title: String = "Two lines plot" + y1: DoubleArray, + y2: DoubleArray, + xLabel: String = "X", + yLabel: String = "Y", + title: String = "Two lines plot" ): Figure { val layout = getXYLayout(xLabel, yLabel, title) - val trace1: Trace = ScatterTrace.builder(DoubleArray(y1.size) { it.toDouble() }, y1).mode(ScatterTrace.Mode.LINE_AND_MARKERS).build() - val trace2: Trace = ScatterTrace.builder(DoubleArray(y2.size) { it.toDouble() }, y2).mode(ScatterTrace.Mode.LINE_AND_MARKERS).build() + val trace1: Trace = ScatterTrace.builder(DoubleArray(y1.size) { it.toDouble() }, y1) + .mode(ScatterTrace.Mode.LINE_AND_MARKERS).build() + val trace2: Trace = ScatterTrace.builder(DoubleArray(y2.size) { it.toDouble() }, y2) + .mode(ScatterTrace.Mode.LINE_AND_MARKERS).build() return Figure(layout, trace1, trace2) } - fun buildLinePlotSmoothed( - x: DoubleArray, - y: DoubleArray, - smoothing: Double = 1.2, - xLabel: String = "X", - yLabel: String = "Y", - title: String = "Line plot" + fun buildSeveralLinesPlot( + x: List, + y: List, + colors: List, + names: List, + xLabel: String = "X", + yLabel: String = "Y", + title: String = "Line plot" ): Figure { val layout = getXYLayout(xLabel, yLabel, title) - val trace: Trace = ScatterTrace.builder(x, y) + val traces = x.indices.map { + ScatterTrace.builder(x[it], y[it]) .mode(ScatterTrace.Mode.LINE_AND_MARKERS) - .line(Line.builder().shape(Line.Shape.SPLINE).smoothing(smoothing).build()) + .line(Line.builder().shape(Line.Shape.LINEAR).color(colors[it]).build()) + .name(names[it]) + .showLegend(true) .build() + } + + return Figure(layout, *traces.toTypedArray()) + } + fun buildBoxPlot( + x: Array, + y: DoubleArray, + xLabel: String = "X", + yLabel: String = "Y", + title: String = "Box plot" + ): Figure { + val layout = getXYLayout(xLabel, yLabel, title) + val trace: BoxTrace = BoxTrace.builder(x, y).build() return Figure(layout, trace) } + } } \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/visual/HtmlBuilder.kt b/utbot-analytics/src/main/kotlin/org/utbot/visual/HtmlBuilder.kt index 98037c2186..0b85d21cdf 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/visual/HtmlBuilder.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/visual/HtmlBuilder.kt @@ -1,11 +1,16 @@ package org.utbot.visual +import tech.tablesaw.plotly.components.Figure import java.io.File import java.time.LocalDateTime import java.time.format.DateTimeFormatter -import java.util.* +import java.util.Properties -class HtmlBuilder(bodyMaxWidth: Int = 600) { +class HtmlBuilder( + bodyMaxWidth: Int = 600, + pathToStyle: List = listOf(), + pathToJs: List = listOf() +) { private val pageTop = ("" + System.lineSeparator() + "" @@ -13,55 +18,80 @@ class HtmlBuilder(bodyMaxWidth: Int = 600) { + " Multi-plot test" + System.lineSeparator() + " " - + System.lineSeparator() - + "" + + pathToJs.joinToString("") { "" } + System.lineSeparator() + "" + System.lineSeparator() - + "" + + "" + System.lineSeparator()) private val pageBottom = "" + System.lineSeparator() + "" - private var pageBuilder = StringBuilder(pageTop).appendLine() + private var pageBuilder = StringBuilder(pageTop).append(System.lineSeparator()) + private var figres_num = 0 - fun saveHtml(fileName: String = "Report.html") { - File(fileName).writeText(pageBuilder.toString() + pageBottom) + fun addFigure(figure: Figure) { + figure.asJavascript("plot${this.figres_num}") + pageBuilder.append("
").append(System.lineSeparator()) + .append(figure.asJavascript("plot${this.figres_num}")).append(System.lineSeparator()).append("
") + this.figres_num += 1 } - fun addText(text: String) { - pageBuilder.append("
$text
") + fun saveHTML(fileName: String = "Report.html") { + File(fileName).writeText(pageBuilder.toString() + pageBottom) } - fun addRawHTML(HTMLCode: String) { - pageBuilder.append(HTMLCode) - } + fun addHeader(ModelName: String, properties: Properties) { + val currentDateTime = LocalDateTime.now().format(DateTimeFormatter.ofPattern("dd-MM-yyyy HH:mm:ss")) + pageBuilder.append("

Model : ${ModelName}

").append("

$currentDateTime

") + .append("

Hyperparameters:

") + for (property in properties) { + pageBuilder.append("
${property.key} : ${property.value}
").append(System.lineSeparator()) + } - fun addBreak() { - addText("
") } - fun addHeader1(header: String){ - addText("

$header

") + fun addHeader(header: String, h: String = "h1") { + pageBuilder.append("<$h>${header}") } - fun addHeader2(header: String){ - addText("

$header

") + fun addTable( + statistics: List>, + selectorNames: List, + colors: List, + scope: String + ) { + pageBuilder.append("") + pageBuilder.append("\n") + + selectorNames.forEach { pageBuilder.append("\n") } + pageBuilder.append("") + + statistics.first().keys.forEach { key -> + pageBuilder.append("\n") + + for (i in statistics.indices) { + pageBuilder.append( + "\n" + ) + } + pageBuilder.append("") + } + pageBuilder.append("
$scope${it}
${key}\n" + + "\n" + + "

${String.format("%.0f", statistics[i][key])}

\n" + + "\n" + + "
") } - fun addHeader3(header: String){ - addText("

$header

") + fun addText(text: String) { + pageBuilder.append("
$text
") } + fun addRawHTML(HTMLCode: String) { + pageBuilder.append(HTMLCode) + } } diff --git a/utbot-analytics/src/main/resources/css/coverage.css b/utbot-analytics/src/main/resources/css/coverage.css new file mode 100644 index 0000000000..c85f237112 --- /dev/null +++ b/utbot-analytics/src/main/resources/css/coverage.css @@ -0,0 +1,168 @@ +/* + * Copyright 2000-2021 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +* { + margin: 0; + padding: 0; +} + +body { + background-color: #fff; + font-family: helvetica neue, tahoma, arial, sans-serif; + font-size: 82%; + color: #151515; +} + +h1 { + margin: 0.5em 0; + color: #010101; + font-weight: normal; + font-size: 18px; +} + +h2 { + margin: 0.5em 0; + color: #010101; + font-weight: normal; + font-size: 16px; +} + +a { + color: #1564C2; + text-decoration: none; +} + +a:hover { + text-decoration: underline; +} + +i { + background-color: #eee; +} + +span.separator { + color: #9BA9BA; + padding-left: 5px; + padding-right: 5px; +} + +div.content { + width: 99%; +} + +table.coverageStats { + width: 100%; + border-collapse: collapse; +} + +table.overallStats { + width: 20%; +} + +table.coverageStats td, table.coverageStats th { + padding: 4px 2px; + border-bottom: 1px solid #ccc; +} + +table.coverageStats th { + background-color: #959BA4; + border: none; + font-weight: bold; + text-align: left; + color: #FFF; +} + +table.coverageStats th.coverageStat { + width: 20%; +} + +table.coverageStats th a { + color: #FFF; +} + +table.coverageStats th a:hover { + text-decoration: none; +} + +table.coverageStats th.sortedDesc a { + background: url(../img/arrowDown.gif) no-repeat 100% 2px; + padding-right: 20px; +} + +table.coverageStats th.sortedAsc a { + background: url(../img/arrowUp.gif) no-repeat 100% 2px; + padding-right: 20px; +} + +div.footer { + margin: 2em .5em; + font-size: 85%; + text-align: left; + line-height: 140%; +} + +div.sourceCode { + width: 100%; + border: 1px solid #ccc; + font: normal 12px 'Menlo', 'Bitstream Vera Sans Mono', 'Courier New', 'Courier', monospace; + white-space: pre; +} + +div.sourceCode b { + font-weight: normal; +} + +div.sourceCode i { + display: block; + float: left; + width: 3em; + padding-right: 3px; + border-right: 1px solid #ccc; + font-style: normal; + text-align: right; +} + +div.sourceCode i.no-highlight span.number { + color: #151515; +} + +div.sourceCode .fc, div.sourceCode .fc i { + background-color: #cfc; +} + +div.sourceCode .pc, div.sourceCode .pc i { + background-color: #ffc; +} + +div.sourceCode .nc, div.sourceCode .nc i { + background-color: #fcc; +} + +.percent, .absValue { + font-size: 90%; +} + +.percent .green, .absValue .green { + color: #32cc32; +} + +.percent .red, .absValue .red { + color: #f00; +} + +.percent .totalDiff { + color: #3f3f3f; +} diff --git a/utbot-analytics/src/main/resources/css/highlight-idea.css b/utbot-analytics/src/main/resources/css/highlight-idea.css new file mode 100644 index 0000000000..e82d7e2cb8 --- /dev/null +++ b/utbot-analytics/src/main/resources/css/highlight-idea.css @@ -0,0 +1,153 @@ +/* + * Copyright 2000-2021 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + +Intellij Idea-like styling (c) Vasily Polovnyov + +*/ + +pre code { + display: block; padding: 0.5em; + color: #000; + background: #fff; +} + +pre .subst, +pre .title { + font-weight: normal; + color: #000; +} + +pre .comment, +pre .template_comment, +pre .javadoc, +pre .diff .header { + color: #808080; + font-style: italic; +} + +pre .annotation, +pre .decorator, +pre .preprocessor, +pre .doctype, +pre .pi, +pre .chunk, +pre .shebang, +pre .apache .cbracket, +pre .input_number, +pre .http .title { + color: #808000; +} + +pre .tag, +pre .pi { + background: #efefef; +} + +/* leonid.khachaturov: redefine background as it conflicts with change highlighting we apply on top of source highlighting */ +pre .changeAdded .tag, +pre .changeRemoved .tag, +pre .changeAdded .pi, +pre .changeRemoved .pi { + background: transparent; +} + +/* leonid.khachaturov: redefine .comment from main.css */ +pre .comment { + margin: 0; + padding: 0; + font-size: 100%; +} + +pre .tag .title, +pre .id, +pre .attr_selector, +pre .pseudo, +pre .literal, +pre .keyword, +pre .hexcolor, +pre .css .function, +pre .ini .title, +pre .css .class, +pre .list .title, +pre .nginx .title, +pre .tex .command, +pre .request, +pre .status { + font-weight: bold; + color: #000080; +} + +pre .attribute, +pre .rules .keyword, +pre .number, +pre .date, +pre .regexp, +pre .tex .special { + font-weight: bold; + color: #0000ff; +} + +pre .number, +pre .regexp { + font-weight: normal; +} + +pre .string, +pre .value, +pre .filter .argument, +pre .css .function .params, +pre .apache .tag { + color: #008000; + font-weight: bold; +} + +pre .symbol, +pre .ruby .symbol .string, +pre .ruby .symbol .keyword, +pre .ruby .symbol .keymethods, +pre .char, +pre .tex .formula { + color: #000; + background: #d0eded; + font-style: italic; +} + +pre .phpdoc, +pre .yardoctag, +pre .javadoctag { + text-decoration: underline; +} + +pre .variable, +pre .envvar, +pre .apache .sqbracket, +pre .nginx .built_in { + color: #660e7a; +} + +pre .addition { + background: #baeeba; +} + +pre .deletion { + background: #ffc8bd; +} + +pre .diff .change { + background: #bccff9; +} diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt index 76d1ed0c97..93785ab0ec 100644 --- a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt +++ b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt @@ -330,6 +330,10 @@ object UtSettings { */ var testCounter by getIntProperty(0) + var collectCoverage by getBooleanProperty(false) + + var coverageStatisticsDir by getStringProperty("logs/covStatistics") + /** * Flag for Subpath and NN selectors whether they are combined (Subpath use several indexes, NN use several models) */ diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/CoverageStatistics.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/CoverageStatistics.kt new file mode 100644 index 0000000000..5ddb2ea95c --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/analytics/CoverageStatistics.kt @@ -0,0 +1,49 @@ +package org.utbot.analytics + +import mu.KotlinLogging +import org.utbot.engine.ExecutionState +import org.utbot.engine.InterProceduralUnitGraph +import org.utbot.engine.selectors.strategies.TraverseGraphStatistics +import org.utbot.engine.stmts +import org.utbot.framework.UtSettings +import java.io.File +import java.io.FileOutputStream + + +private val logger = KotlinLogging.logger {} + + +class CoverageStatistics( + private val method: String, + private val globalGraph: InterProceduralUnitGraph +) : TraverseGraphStatistics(globalGraph) { + + private val outputFile: String = "${UtSettings.coverageStatisticsDir}/$method.txt" + + init { + File(outputFile).printWriter().use { out -> + out.println("TIME,COV_TARGET_STMT,TOTAL_TARGET_STMT,COV_ALL_STMT,TOTAL_ALL_STMT") + out.println("${System.nanoTime()}" + "," + getStatistics()) + } + } + + override fun onTraversed(executionState: ExecutionState) { + runCatching { + FileOutputStream(outputFile, true).bufferedWriter() + .use { out -> + out.write(System.nanoTime().toString() + "," + getStatistics()) + out.newLine() + } + }.onFailure { + logger.warn { "Failed to save statistics: ${it.message}" } + } + } + + fun getStatistics() = with(globalGraph) { + val allStmts = this.graphs.flatMap { it.stmts } + val graphStmts = this.graphs.first().stmts + + "${graphStmts.filter { this.isCovered(it) }.size},${graphStmts.size}," + + "${allStmts.filter { this.isCovered(it) }.size},${allStmts.size}" + } +} \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt index 9d203ca991..ea14fee6a4 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt @@ -17,6 +17,7 @@ import kotlinx.coroutines.flow.onStart import kotlinx.coroutines.isActive import kotlinx.coroutines.yield import mu.KotlinLogging +import org.utbot.analytics.CoverageStatistics import org.utbot.analytics.EngineAnalyticsContext import org.utbot.analytics.FeatureProcessor import org.utbot.analytics.Predictors @@ -406,6 +407,7 @@ class UtBotSymbolicEngine( require(trackableResources.isEmpty()) if (useDebugVisualization) GraphViz(globalGraph, pathSelector) + if (UtSettings.collectCoverage) CoverageStatistics(methodUnderTest.toString(), globalGraph) val initStmt = graph.head val initState = ExecutionState(