From a494d22a26ca36c77c39d6b0f8c86f07c53b14ed Mon Sep 17 00:00:00 2001
From: Rustam Sadykov <snob-1432e@mail.ru>
Date: Mon, 24 Oct 2022 11:22:52 +0300
Subject: [PATCH 01/18] provide fuzzing ratio property reformat serialization

---
 .../org/utbot/common/AbstractSettings.kt      |   5 +-
 utbot-junit-contest/build.gradle              |   4 +
 .../main/kotlin/org/utbot/contest/Contest.kt  |  16 +-
 .../org/utbot/contest/ContestEstimator.kt     |  22 ++-
 .../kotlin/org/utbot/contest/Statistics.kt    |  16 +-
 .../org/utbot/monitoring/MonitoringReport.kt  | 118 +++++++++++++
 .../utbot/monitoring/MonitoringSettings.kt    |  14 +-
 .../utbot/monitoring/StatisticsMonitoring.kt  | 157 +++++-------------
 8 files changed, 205 insertions(+), 147 deletions(-)
 create mode 100644 utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringReport.kt

diff --git a/utbot-core/src/main/kotlin/org/utbot/common/AbstractSettings.kt b/utbot-core/src/main/kotlin/org/utbot/common/AbstractSettings.kt
index f564d98a20..d3db9a002e 100644
--- a/utbot-core/src/main/kotlin/org/utbot/common/AbstractSettings.kt
+++ b/utbot-core/src/main/kotlin/org/utbot/common/AbstractSettings.kt
@@ -128,6 +128,9 @@ abstract class AbstractSettings(
     protected fun getStringProperty(defaultValue: String) = getProperty(defaultValue) { it }
     protected inline fun <reified T : Enum<T>> getEnumProperty(defaultValue: T) =
         getProperty(defaultValue) { enumValueOf(it) }
-
+    protected fun getListProperty(defaultValue: List<String>) =
+        getProperty(defaultValue) { it.split(';') }
+    protected inline fun <reified T> getListProperty(defaultValue: List<T>, crossinline elementTransform: (String) -> T) =
+        getProperty(defaultValue) { it.split(';').map(elementTransform) }
     override fun toString(): String = container.toString()
 }
\ No newline at end of file
diff --git a/utbot-junit-contest/build.gradle b/utbot-junit-contest/build.gradle
index 720494e9b8..dd40d660f7 100644
--- a/utbot-junit-contest/build.gradle
+++ b/utbot-junit-contest/build.gradle
@@ -1,3 +1,6 @@
+plugins {
+    id 'org.jetbrains.kotlin.plugin.serialization' version '1.7.20'
+}
 apply plugin: 'jacoco'
 
 configurations {
@@ -58,6 +61,7 @@ dependencies {
     implementation group: 'org.apache.commons', name: 'commons-exec', version: '1.2'
     implementation group: 'io.github.microutils', name: 'kotlin-logging', version: kotlinLoggingVersion
     implementation group: 'org.jsoup', name: 'jsoup', version: '1.6.2'
+    implementation 'org.jetbrains.kotlinx:kotlinx-serialization-json:1.4.1'
     // need for tests
     implementation group: 'org.mockito', name: 'mockito-core', version: '4.2.0'
     implementation group: 'org.mockito', name: 'mockito-inline', version: '4.2.0'
diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt
index 2bd9389f7f..4f208cd0e3 100644
--- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt
+++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt
@@ -138,7 +138,15 @@ fun main(args: Array<String>) {
             val timeBudgetSec = cmd[2].toLong()
             val cut = ClassUnderTest(classLoader.loadClass(classUnderTestName).id, outputDir, classfileDir.toFile())
 
-            runGeneration(cut, timeBudgetSec, classpathString, runFromEstimator = false, methodNameFilter = null)
+            runGeneration(
+                project = "Contest",
+                cut,
+                timeBudgetSec,
+                fuzzingRatio = 0.1,
+                classpathString,
+                runFromEstimator = false,
+                methodNameFilter = null
+            )
             println("${ContestMessage.READY}")
         }
     }
@@ -162,8 +170,10 @@ fun setOptions() {
 @ObsoleteCoroutinesApi
 @SuppressWarnings
 fun runGeneration(
+    project: String,
     cut: ClassUnderTest,
     timeLimitSec: Long,
+    fuzzingRatio: Double,
     classpathString: String,
     runFromEstimator: Boolean,
     methodNameFilter: String? = null // For debug purposes you can specify method name
@@ -205,7 +215,7 @@ fun runGeneration(
         logger.error("Seems like classloader for cut not valid (maybe it was backported to system): ${cut.classLoader}")
     }
 
-    val statsForClass = StatsForClass()
+    val statsForClass = StatsForClass(project, cut.fqn)
 
     val codeGenerator = CodeGenerator(
             cut.classId,
@@ -276,6 +286,8 @@ fun runGeneration(
 
                     val budgetForSymbolicExecution = max(0, budgetForMethod - budgetForLastSolverRequestAndConcreteExecutionRemainingStates)
 
+                    UtSettings.utBotGenerationTimeoutInMillis = budgetForMethod
+                    UtSettings.fuzzingTimeoutInMillis = (budgetForMethod * fuzzingRatio).toLong()
 
                     //start controller that will activate symbolic execution
                     GlobalScope.launch {
diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt
index 3cd74bb432..e3af1ccb40 100644
--- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt
+++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt
@@ -83,6 +83,7 @@ enum class Tool {
             project: ProjectToEstimate,
             cut: ClassUnderTest,
             timeLimit: Long,
+            fuzzingRatio: Double,
             methodNameFilter: String?,
             globalStats: GlobalStats,
             compiledTestDir: File,
@@ -91,8 +92,10 @@ enum class Tool {
             val classStats: StatsForClass = try {
                 withUtContext(UtContext(project.classloader)) {
                     runGeneration(
+                        project.name,
                         cut,
                         timeLimit,
+                        fuzzingRatio,
                         project.sootClasspathString,
                         runFromEstimator = true,
                         methodNameFilter
@@ -167,6 +170,7 @@ enum class Tool {
             project: ProjectToEstimate,
             cut: ClassUnderTest,
             timeLimit: Long,
+            fuzzingRatio: Double,
             methodNameFilter: String?,
             globalStats: GlobalStats,
             compiledTestDir: File,
@@ -237,6 +241,7 @@ enum class Tool {
         project: ProjectToEstimate,
         cut: ClassUnderTest,
         timeLimit: Long,
+        fuzzingRatio: Double, // maybe create some specific settings
         methodNameFilter: String?,
         globalStats: GlobalStats,
         compiledTestDir: File,
@@ -257,6 +262,7 @@ fun main(args: Array<String>) {
     if (args.isEmpty() && System.getProperty("os.name")?.run { contains("win", ignoreCase = true) } == true) {
         processedClassesThreshold = 9999 //change to change number of classes to run
         val timeLimit = 20 // increase if you want to debug something
+        val fuzzingRatio = 0.1 // sets fuzzing ratio to total test generation
 
         // Uncomment it for debug purposes:
         // you can specify method for test generation in format `classFqn.methodName`
@@ -283,12 +289,13 @@ fun main(args: Array<String>) {
             classesLists,
             jarsDir,
             "$timeLimit",
+            "$fuzzingRatio",
             outputDir,
             moduleTestDir
         )
     } else {
-        require(args.size == 6) {
-            "Wrong arguments: <classes dir> <classpath_dir> <time limit (s)> <output dir> <test dir> <junit jar path> expected, but got: ${args.toText()}"
+        require(args.size == 7) {
+            "Wrong arguments: <classes dir> <classpath_dir> <time limit (s)> <fuzzing ratio> <output dir> <test dir> <junit jar path> expected, but got: ${args.toText()}"
         }
         logger.info { "Command line: [${args.joinToString(" ")}]" }
 
@@ -315,8 +322,9 @@ fun runEstimator(
     val classesLists = File(args[0])
     val classpathDir = File(args[1])
     val timeLimit = args[2].toLong()
-    val outputDir = File(args[3])
-    // we don't use it: val moduleTestDir = File(args[4])
+    val fuzzingRatio = args[3].toDouble()
+    val outputDir = File(args[4])
+    // we don't use it: val moduleTestDir = File(args[5])
 
     val testCandidatesDir = File(outputDir, "test_candidates")
     val compiledTestDir = File(outputDir, "compiled")
@@ -411,7 +419,7 @@ fun runEstimator(
 
                     logger.info { "------------- [${project.name}] ---->--- [$classIndex:$classFqn] ---------------------" }
 
-                    tool.run(project, cut, timeLimit, methodNameFilter, globalStats, compiledTestDir, classFqn)
+                    tool.run(project, cut, timeLimit, fuzzingRatio, methodNameFilter, globalStats, compiledTestDir, classFqn)
                 }
             }
         }
@@ -423,10 +431,6 @@ fun runEstimator(
     logger.info { globalStats }
     ConcreteExecutor.defaultPool.close()
 
-//    For what?
-//    if (globalStats.statsForClasses.isNotEmpty())
-//        exitProcess(1)
-
     return globalStats
 }
 
diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Statistics.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Statistics.kt
index a980ab002c..74f33d3c88 100644
--- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Statistics.kt
+++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Statistics.kt
@@ -109,7 +109,7 @@ class GlobalStats {
             avgCoverage.format(PRECISION) + " %"
 }
 
-class StatsForClass {
+class StatsForClass(val project: String, val className: String) {
     val testedClassNames: MutableSet<String> = ConcurrentSkipListSet()
 
     var methodsCount: Int = -1
@@ -126,16 +126,16 @@ class StatsForClass {
     var fuzzedCoverage = CoverageInstructionsSet()
     var concolicCoverage = CoverageInstructionsSet()
 
-    /**
-     * Add class [className] to respect coverage from this class.
-     */
-    fun addTestedClass(className: String) {
-        testedClassNames.add(className)
-    }
-
     private fun CoverageInstructionsSet.prettyInfo(): String =
         getCoverageInfo(testedClassNames).run { "$covered/$total" }
 
+    fun getCoverageInfo(): CoverageStatistic =
+        coverage.getCoverageInfo(testedClassNames)
+    fun getFuzzedCoverageInfo(): CoverageStatistic =
+        fuzzedCoverage.getCoverageInfo(testedClassNames)
+    fun getConcolicCoverageInfo(): CoverageStatistic =
+        concolicCoverage.getCoverageInfo(testedClassNames)
+
     override fun toString(): String = "\n<StatsForClass> :" +
             "\n\tcanceled by timeout = $canceledByTimeout" +
             "\n\t#methods = $methodsCount, " +
diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringReport.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringReport.kt
new file mode 100644
index 0000000000..8e550616e5
--- /dev/null
+++ b/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringReport.kt
@@ -0,0 +1,118 @@
+package org.utbot.monitoring
+
+import kotlinx.serialization.Serializable
+import org.utbot.contest.GlobalStats
+import org.utbot.contest.StatsForClass
+
+@Serializable
+data class MonitoringReport(
+    val parameters: MonitoringParameters,
+    val summarised_metrics: SummarisedMetricsReport,
+    val targets: List<TargetReport>,
+) {
+    constructor(parameters: MonitoringParameters, targets: List<TargetReport>): this(
+        parameters,
+        SummarisedMetricsReport(targets),
+        targets
+    )
+
+    constructor(parameters: MonitoringParameters, stats: GlobalStats) : this(
+        parameters,
+        stats.statsForClasses.map {
+            TargetReport(
+                TargetDescription(it.project, it.className),
+                ClassMetricsReport(it)
+            )
+        }
+    )
+}
+
+@Serializable
+data class MonitoringParameters(
+    val fuzzing_ratio: Double,
+    val class_timeout_sec: Int,
+    val run_timeout_min: Int,
+)
+
+@Serializable
+data class TargetReport(
+    val target: TargetDescription,
+    val metrics: ClassMetricsReport
+)
+
+@Serializable
+data class TargetDescription(
+    val project: String,
+    val classname: String
+)
+
+private fun Int.cover(total: Int): Double =
+    if (total == 0) 0.0 else this.toDouble() / total
+
+@Serializable
+data class ClassMetricsReport(
+    val testcases_generated: Int,
+    val failed_to_compile: Boolean,
+    val canceled_by_timeout: Boolean,
+    val total_methods_in_class: Int,
+    val methods_with_at_least_one_testcase_generated: Int,
+    val methods_with_at_least_one_exception: Int,
+    val methods_without_any_tests_and_exceptions: Int,
+    val covered_bytecode_instructions_in_class: Int,
+    val covered_bytecode_instructions_in_class_by_fuzzing: Int,
+    val covered_bytecode_instructions_in_class_by_concolic: Int,
+    val total_bytecode_instructions_in_class: Int,
+    val bytecode_instructions_coverage_in_class: Double = covered_bytecode_instructions_in_class.cover(total_bytecode_instructions_in_class),
+    val bytecode_instructions_coverage_in_class_by_fuzzing: Double = covered_bytecode_instructions_in_class_by_fuzzing.cover(total_bytecode_instructions_in_class),
+    val bytecode_instructions_coverage_in_class_by_concolic: Double = covered_bytecode_instructions_in_class_by_concolic.cover(total_bytecode_instructions_in_class)
+) {
+    constructor(statsForClass: StatsForClass) : this(
+        testcases_generated = statsForClass.testcasesGenerated,
+        failed_to_compile = statsForClass.failedToCompile,
+        canceled_by_timeout = statsForClass.canceledByTimeout,
+        total_methods_in_class = statsForClass.methodsCount,
+        methods_with_at_least_one_testcase_generated = statsForClass.statsForMethods.count { it.testsGeneratedCount > 0 },
+        methods_with_at_least_one_exception = statsForClass.methodsWithAtLeastOneException,
+        methods_without_any_tests_and_exceptions = statsForClass.statsForMethods.count { it.isSuspicious },
+        covered_bytecode_instructions_in_class = statsForClass.getCoverageInfo().covered,
+        covered_bytecode_instructions_in_class_by_fuzzing = statsForClass.getFuzzedCoverageInfo().covered,
+        covered_bytecode_instructions_in_class_by_concolic = statsForClass.getConcolicCoverageInfo().covered,
+        total_bytecode_instructions_in_class = statsForClass.coverage.totalInstructions.toInt()
+    )
+}
+
+@Serializable
+data class SummarisedMetricsReport(
+    val total_classes: Int,
+    val testcases_generated: Int,
+    val classes_failed_to_compile: Int,
+    val classes_canceled_by_timeout: Int,
+    val total_methods: Int,
+    val methods_with_at_least_one_testcase_generated: Int,
+    val methods_with_at_least_one_exception: Int,
+    val methods_without_any_tests_and_exceptions: Int,
+    val covered_bytecode_instructions: Int,
+    val covered_bytecode_instructions_by_fuzzing: Int,
+    val covered_bytecode_instructions_by_concolic: Int,
+    val total_bytecode_instructions: Int,
+    val bytecode_instructions_coverage: Double = covered_bytecode_instructions.cover(total_bytecode_instructions),
+    val bytecode_instructions_coverage_by_fuzzing: Double = covered_bytecode_instructions_by_fuzzing.cover(total_bytecode_instructions),
+    val bytecode_instructions_coverage_by_concolic: Double = covered_bytecode_instructions_by_concolic.cover(total_bytecode_instructions),
+    val averaged_bytecode_instruction_coverage_by_classes: Double
+) {
+    constructor(targets: Collection<TargetReport>) : this(
+        total_classes = targets.size,
+        testcases_generated = targets.sumOf { it.metrics.testcases_generated },
+        classes_failed_to_compile = targets.count { it.metrics.failed_to_compile },
+        classes_canceled_by_timeout = targets.count { it.metrics.canceled_by_timeout },
+        total_methods = targets.sumOf { it.metrics.total_methods_in_class },
+        methods_with_at_least_one_testcase_generated = targets.sumOf { it.metrics.methods_with_at_least_one_testcase_generated },
+        methods_with_at_least_one_exception = targets.sumOf { it.metrics.methods_with_at_least_one_exception },
+        methods_without_any_tests_and_exceptions = targets.sumOf { it.metrics.methods_without_any_tests_and_exceptions },
+        covered_bytecode_instructions = targets.sumOf { it.metrics.covered_bytecode_instructions_in_class },
+        covered_bytecode_instructions_by_fuzzing = targets.sumOf { it.metrics.covered_bytecode_instructions_in_class_by_fuzzing },
+        covered_bytecode_instructions_by_concolic = targets.sumOf { it.metrics.covered_bytecode_instructions_in_class_by_concolic },
+        total_bytecode_instructions = targets.sumOf { it.metrics.total_bytecode_instructions_in_class },
+        averaged_bytecode_instruction_coverage_by_classes = targets.map { it.metrics.bytecode_instructions_coverage_in_class }.average()
+    )
+}
diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringSettings.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringSettings.kt
index 62dcff92b8..a9aa051241 100644
--- a/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringSettings.kt
+++ b/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringSettings.kt
@@ -22,19 +22,17 @@ object MonitoringSettings : AbstractSettings(
     val processedClassesThreshold by getIntProperty(9999)
 
     /**
-     * Number of running test generation.
+     * One running generation for all projects timeout in minutes.
      */
-    val runTries by getIntProperty(1)
+    val runTimeoutMinutes by getIntProperty(180)
 
     /**
-     * One running generation for all projects timeout in minutes.
+     * Target projects to generate tests.
      */
-    val runTimeoutMinutes by getIntProperty(180)
+    val projects by getListProperty(listOf("guava"))
 
     /**
-     * Target project to generate tests.
-     *
-     * TODO: change it into list and group GlobalStats by projects.
+     * Set up fuzzing timeout relatively total generation time.
      */
-    val project by getStringProperty("guava")
+    val fuzzingRatios by getListProperty(listOf(0.1), String::toDouble)
 }
\ No newline at end of file
diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/StatisticsMonitoring.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/StatisticsMonitoring.kt
index 0b4600571c..d36e90c2ae 100644
--- a/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/StatisticsMonitoring.kt
+++ b/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/StatisticsMonitoring.kt
@@ -2,6 +2,9 @@ package org.utbot.monitoring
 
 import java.io.File
 import java.util.concurrent.TimeUnit
+import kotlin.system.exitProcess
+import kotlinx.serialization.encodeToString
+import kotlinx.serialization.json.Json
 import mu.KotlinLogging
 import org.utbot.common.ThreadBasedExecutor
 import org.utbot.common.bracket
@@ -13,7 +16,6 @@ import org.utbot.contest.Tool
 import org.utbot.contest.runEstimator
 import org.utbot.framework.plugin.services.JdkInfoService
 import org.utbot.instrumentation.ConcreteExecutor
-import kotlin.system.exitProcess
 
 private val javaHome = System.getenv("JAVA_HOME")
 private val logger = KotlinLogging.logger {}
@@ -23,143 +25,60 @@ fun main(args: Array<String>) {
 
     val methodFilter: String?
     val processedClassesThreshold = MonitoringSettings.processedClassesThreshold
-    val tools: List<Tool> = listOf(Tool.UtBot)
     val timeLimit = MonitoringSettings.classTimeoutSeconds
 
-    val project = MonitoringSettings.project
+    val projects = MonitoringSettings.projects
     methodFilter = null
 
-    val outputFile = args.getOrNull(0)?.let { File(it) }
-    val runTries = MonitoringSettings.runTries
     val runTimeout = TimeUnit.MINUTES.toMillis(MonitoringSettings.runTimeoutMinutes.toLong())
 
-    val estimatorArgs: Array<String> = arrayOf(
-        Paths.classesLists,
-        Paths.jarsDir,
-        "$timeLimit",
-        Paths.outputDir,
-        Paths.moduleTestDir
-    )
-
-    val statistics = mutableListOf<GlobalStats>()
+    val measurementResults = mutableListOf<MonitoringReport>()
 
     JdkInfoService.jdkInfoProvider = ContestEstimatorJdkInfoProvider(javaHome)
     val executor = ThreadBasedExecutor()
 
-    repeat(runTries) { idx ->
-        logger.info().bracket("Run UTBot try number ${idx + 1}") {
+    MonitoringSettings.fuzzingRatios.forEach { fuzzingRatio ->
+        val estimatorArgs: Array<String> = arrayOf(
+            Paths.classesLists,
+            Paths.jarsDir,
+            "$timeLimit",
+            "$fuzzingRatio",
+            Paths.outputDir,
+            Paths.moduleTestDir
+        )
+
+        logger.info().bracket("Run UTBot generation [fuzzing ratio = $fuzzingRatio]") {
             val start = System.nanoTime()
 
             executor.invokeWithTimeout(runTimeout) {
                 runEstimator(
                     estimatorArgs, methodFilter,
-                    listOf(project), processedClassesThreshold, tools
+                    projects, processedClassesThreshold, listOf(Tool.UtBot)
                 )
-            }
-                ?.onSuccess {
-                    it as GlobalStats
-                    it.duration = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)
-                    statistics.add(it)
-                }
-                ?.onFailure { logger.error(it) { "Run failure!" } }
-                ?: run {
-                    logger.info { "Run timeout!" }
-                    ConcreteExecutor.defaultPool.forceTerminateProcesses()
-                }
+            }?.onSuccess {
+                it as GlobalStats
+                it.duration = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)
+                measurementResults.add(MonitoringReport(
+                    parameters = MonitoringParameters(
+                        fuzzingRatio, MonitoringSettings.classTimeoutSeconds, MonitoringSettings.runTimeoutMinutes
+                    ),
+                    stats = it
+                ))
+            }?.onFailure { logger.error(it) { "Run failure!" } }
+                ?: logger.info { "Run timeout!" }
 
         }
-    }
-
-    if (statistics.isEmpty())
-        exitProcess(1)
-
-    outputFile?.writeText(statistics.jsonString())
-}
-
-private fun StringBuilder.tab(tabs: Int) {
-    append("\t".repeat(tabs))
-}
-
-private fun StringBuilder.addValue(name: String, value: Any?, tabs: Int = 0, needComma: Boolean = true) {
-    tab(tabs)
-    append("\"$name\": $value")
-    if (needComma) append(',')
-    appendLine()
-}
-
-
-private fun objectString(vararg values: Pair<String, Any?>, baseTabs: Int = 0, needFirstTab: Boolean = false) =
-    buildString {
-        if (needFirstTab) tab(baseTabs)
-        appendLine("{")
 
-        val tabs = baseTabs + 1
-        values.forEachIndexed { index, (name, value) ->
-            addValue(name, value, tabs, index != values.lastIndex)
-        }
-
-        tab(baseTabs)
-        append("}")
+        ConcreteExecutor.defaultPool.forceTerminateProcesses()
+        System.gc()
     }
+    if (measurementResults.isEmpty())
+        exitProcess(1)
 
-private fun arrayString(vararg values: Any?, baseTabs: Int = 0, needFirstTab: Boolean = false) =
-    buildString {
-        if (needFirstTab) tab(baseTabs)
-        appendLine("[")
-
-        val tabs = baseTabs + 1
-        values.forEachIndexed { index, value ->
-            tab(tabs)
-            append(value)
-            if (index != values.lastIndex) append(",")
-            appendLine()
-        }
-
-        tab(baseTabs)
-        append("]")
+    val outputFile = args.getOrNull(0)?.let { File(it) }
+    outputFile?.let { file ->
+        val format = Json { prettyPrint = true }
+        val jsonString = format.encodeToString(measurementResults)
+        file.writeText(jsonString)
     }
-
-
-private fun GlobalStats.parametersObject(baseTabs: Int = 0, needFirstTab: Boolean = false) =
-    objectString(
-        "target" to "\"${MonitoringSettings.project}\"",
-        "class_timeout_sec" to MonitoringSettings.classTimeoutSeconds,
-        "run_timeout_min" to MonitoringSettings.runTimeoutMinutes,
-        baseTabs = baseTabs,
-        needFirstTab = needFirstTab
-    )
-
-
-private fun GlobalStats.metricsObject(baseTabs: Int = 0, needFirstTab: Boolean = false) =
-    objectString(
-        "duration_ms" to duration,
-        "classes_for_generation" to classesForGeneration,
-        "testcases_generated" to testCasesGenerated,
-        "classes_without_problems" to classesWithoutProblems,
-        "classes_canceled_by_timeout" to classesCanceledByTimeout,
-        "total_methods_for_generation" to totalMethodsForGeneration,
-        "methods_with_at_least_one_testcase_generated" to methodsWithAtLeastOneTestCaseGenerated,
-        "methods_with_exceptions" to methodsWithExceptions,
-        "suspicious_methods" to suspiciousMethods,
-        "test_classes_failed_to_compile" to testClassesFailedToCompile,
-        "covered_instructions" to coveredInstructions,
-        "covered_instructions_by_fuzzing" to coveredInstructionsByFuzzing,
-        "covered_instructions_by_concolic" to coveredInstructionsByConcolic,
-        "total_instructions" to totalInstructions,
-        "avg_coverage" to avgCoverage,
-        baseTabs = baseTabs,
-        needFirstTab = needFirstTab
-    )
-
-private fun GlobalStats.jsonString(baseTabs: Int = 0, needFirstTab: Boolean = false) =
-    objectString(
-        "parameters" to parametersObject(baseTabs + 1),
-        "metrics" to metricsObject(baseTabs + 1),
-        baseTabs = baseTabs,
-        needFirstTab = needFirstTab
-    )
-
-private fun Iterable<GlobalStats>.jsonString() =
-    arrayString(*map {
-        it.jsonString(baseTabs = 1)
-    }.toTypedArray())
+}

From 673910f234e67421d5a859af14b6f60d4cd6128f Mon Sep 17 00:00:00 2001
From: Rustam Sadykov <snob-1432e@mail.ru>
Date: Wed, 2 Nov 2022 07:37:36 +0300
Subject: [PATCH 02/18] small instructions counting fix

---
 .../src/main/kotlin/org/utbot/contest/Statistics.kt             | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Statistics.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Statistics.kt
index 74f33d3c88..f8f4250398 100644
--- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Statistics.kt
+++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Statistics.kt
@@ -207,7 +207,7 @@ private fun CoverageInstructionsSet?.getCoverageInfo(classNames: Set<String>): C
     CoverageStatistic(
         coveredInstructions.filter {
             instr -> classNames.contains(instr.className)
-        }.toSet().size,
+        }.map { it.id }.distinct().size,
         totalInstructions.toInt()
     )
 } ?: CoverageStatistic(covered = 0, total = 0)

From 05f375ee31516ed6113854e497cac357a2f7346f Mon Sep 17 00:00:00 2001
From: Rustam Sadykov <snob-1432e@mail.ru>
Date: Wed, 2 Nov 2022 08:01:22 +0300
Subject: [PATCH 03/18] add packages to export detection

---
 .../org/utbot/contest/ContestEstimator.kt     | 78 +++++++++++++------
 1 file changed, 56 insertions(+), 22 deletions(-)

diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt
index e3af1ccb40..22cf3aa7cd 100644
--- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt
+++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt
@@ -35,7 +35,6 @@ import org.utbot.framework.plugin.api.util.withUtContext
 import org.utbot.framework.plugin.services.JdkInfoService
 import org.utbot.instrumentation.ConcreteExecutor
 import org.utbot.predictors.MLPredictorFactoryImpl
-import kotlin.concurrent.thread
 import kotlin.math.min
 
 private val logger = KotlinLogging.logger {}
@@ -48,6 +47,57 @@ private val javaHome = System.getenv("JAVA_HOME")
 private val javacCmd = "$javaHome/bin/javac"
 private val javaCmd = "$javaHome/bin/java"
 
+private const val triesToCompile = 2
+
+private data class UnnamedPackageInfo(val pack: String, val module: String)
+
+private fun findAllNotExportedPackages(report: String): List<UnnamedPackageInfo> {
+    val regex = """package ([\d\w.]+) is declared in module ([\d\w.]+), which does not export it to the unnamed module""".toRegex()
+    return regex.findAll(report).map {
+        val pack = it.groupValues[1]
+        val module = it.groupValues[2]
+        UnnamedPackageInfo(pack, module)
+    }.toList().distinct()
+}
+
+private fun compileClass(testDir: String, classPath: String, testClass: String): Int {
+    val opens = mutableSetOf<UnnamedPackageInfo>()
+    var exitCode = 0
+    var errors = ""
+
+    repeat(triesToCompile) { tryNumber ->
+        logger.debug { "Compile try ${tryNumber + 1}" }
+        val cmd = arrayOf(
+            javacCmd,
+            *opens.flatMap {
+                listOf("--add-opens", "${it.module}/${it.pack}=ALL-UNNAMED")
+            }.toTypedArray(),
+            "-d", testDir,
+            "-cp", classPath,
+            "-nowarn",
+            "-XDignore.symbol.file",
+            testClass
+        )
+
+        logger.trace { cmd.toText() }
+        val process = Runtime.getRuntime().exec(cmd)
+
+        errors = process.errorStream.reader().buffered().readText()
+
+        exitCode = process.waitFor()
+        if (exitCode == 0) {
+            return 0
+        } else {
+            opens += findAllNotExportedPackages(errors)
+        }
+    }
+
+    if (errors.isNotEmpty())
+        logger.error { "Compilation errors: $errors" }
+
+    return exitCode
+}
+
 fun Array<String>.toText() = joinToString(separator = ",")
 
 @Suppress("unused")
@@ -116,28 +166,12 @@ enum class Tool {
                 val testClass = cut.generatedTestFile
                 classStats.testClassFile = testClass
 
-                val cmd = arrayOf(
-                    javacCmd,
-                    "-d", compiledTestDir.absolutePath,
-                    "-cp", project.compileClasspathString,
-                    "-nowarn",
-                    "-XDignore.symbol.file",
-                    testClass.absolutePath
-                )
-
                 logger.info().bracket("Compiling class ${testClass.absolutePath}") {
-
-                    logger.trace { cmd.toText() }
-                    val process = Runtime.getRuntime().exec(cmd)
-
-                    thread {
-                        val errors = process.errorStream.reader().buffered().readText()
-                        if (errors.isNotEmpty())
-                            logger.error { "Compilation errors: $errors" }
-                    }.join()
-
-
-                    val exitCode = process.waitFor()
+                    val exitCode = compileClass(
+                        compiledTestDir.absolutePath,
+                        project.compileClasspathString,
+                        testClass.absolutePath
+                    )
                     if (exitCode != 0) {
                         logger.error { "Failed to compile test class ${cut.testClassSimpleName}" }
                         classStats.failedToCompile = true

From ddf6012959797d5207adcfcb3e8e3705211d7031 Mon Sep 17 00:00:00 2001
From: Rustam Sadykov <snob-1432e@mail.ru>
Date: Wed, 2 Nov 2022 10:20:06 +0300
Subject: [PATCH 04/18] restruct statistics by projects

---
 .../org/utbot/contest/ContestEstimator.kt     | 32 +++++++++--------
 .../kotlin/org/utbot/contest/Statistics.kt    |  7 ++--
 .../org/utbot/monitoring/MonitoringReport.kt  | 35 ++++++++++---------
 .../utbot/monitoring/StatisticsMonitoring.kt  |  3 +-
 4 files changed, 43 insertions(+), 34 deletions(-)

diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt
index 22cf3aa7cd..0f6ffebcb9 100644
--- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt
+++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt
@@ -61,16 +61,14 @@ private fun findAllNotExportedPackages(report: String): List<UnnamedPackageInfo>
 }
 
 private fun compileClass(testDir: String, classPath: String, testClass: String): Int {
-    val opens = mutableSetOf<UnnamedPackageInfo>()
+    val exports = mutableSetOf<UnnamedPackageInfo>()
     var exitCode = 0
-    var errors = ""
 
     repeat(triesToCompile) { tryNumber ->
-        logger.debug { "Compile try ${tryNumber + 1}" }
         val cmd = arrayOf(
             javacCmd,
-            *opens.flatMap {
-                listOf("--add-opens", "${it.module}/${it.pack}=ALL-UNNAMED")
+            *exports.flatMap {
+                listOf("--add-exports", "${it.module}/${it.pack}=ALL-UNNAMED")
             }.toTypedArray(),
             "-d", testDir,
             "-cp", classPath,
@@ -78,23 +76,24 @@ private fun compileClass(testDir: String, classPath: String, testClass: String):
             "-XDignore.symbol.file",
             testClass
         )
+        logger.debug { "Compile try ${tryNumber + 1}" }
 
         logger.trace { cmd.toText() }
+
         val process = Runtime.getRuntime().exec(cmd)
 
-        errors = process.errorStream.reader().buffered().readText()
+        val errors = process.errorStream.reader().buffered().readText()
 
         exitCode = process.waitFor()
         if (exitCode == 0) {
             return 0
         } else {
-            opens += findAllNotExportedPackages(errors)
+            if (errors.isNotEmpty())
+                logger.error { "Compilation errors: $errors" }
+            exports += findAllNotExportedPackages(errors)
         }
     }
 
-    if (errors.isNotEmpty())
-        logger.error { "Compilation errors: $errors" }
-
     return exitCode
 }
 
@@ -135,7 +134,7 @@ enum class Tool {
             timeLimit: Long,
             fuzzingRatio: Double,
             methodNameFilter: String?,
-            globalStats: GlobalStats,
+            statsForProject: StatsForProject,
             compiledTestDir: File,
             classFqn: String
         ) {
@@ -160,7 +159,7 @@ enum class Tool {
                 return
             }
 
-            globalStats.statsForClasses.add(classStats)
+            statsForProject.statsForClasses.add(classStats)
 
             try {
                 val testClass = cut.generatedTestFile
@@ -206,7 +205,7 @@ enum class Tool {
             timeLimit: Long,
             fuzzingRatio: Double,
             methodNameFilter: String?,
-            globalStats: GlobalStats,
+            statsForProject: StatsForProject,
             compiledTestDir: File,
             classFqn: String
         ) {
@@ -277,7 +276,7 @@ enum class Tool {
         timeLimit: Long,
         fuzzingRatio: Double, // maybe create some specific settings
         methodNameFilter: String?,
-        globalStats: GlobalStats,
+        statsForProject: StatsForProject,
         compiledTestDir: File,
         classFqn: String
     )
@@ -432,6 +431,9 @@ fun runEstimator(
             outer@ for (project in projects) {
                 if (projectFilter != null && project.name !in projectFilter) continue
 
+                val statsForProject = StatsForProject(project.name)
+                globalStats.projectStats.add(statsForProject)
+
                 logger.info { "------------- project [${project.name}] ---- " }
 
                 // take all the classes from the corresponding jar if a list of the specified classes is empty
@@ -453,7 +455,7 @@ fun runEstimator(
 
                     logger.info { "------------- [${project.name}] ---->--- [$classIndex:$classFqn] ---------------------" }
 
-                    tool.run(project, cut, timeLimit, fuzzingRatio, methodNameFilter, globalStats, compiledTestDir, classFqn)
+                    tool.run(project, cut, timeLimit, fuzzingRatio, methodNameFilter, statsForProject, compiledTestDir, classFqn)
                 }
             }
         }
diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Statistics.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Statistics.kt
index f8f4250398..f3ac113226 100644
--- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Statistics.kt
+++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Statistics.kt
@@ -12,13 +12,16 @@ private fun Double.format(digits: Int) = "%.${digits}f".format(this)
 fun <T> Iterable<T>.printMultiline(printer: (T) -> Any?) = "\n" + joinToString("\n") { "${printer(it)}" } + "\n"
 
 class GlobalStats {
+    val projectStats = mutableListOf<StatsForProject>()
+    var duration: Long? = null
+}
+
+class StatsForProject(val project: String) {
 
     companion object {
         const val PRECISION: Int = 2
     }
 
-    var duration: Long? = null
-
     val statsForClasses = mutableListOf<StatsForClass>()
 
     val classesForGeneration: Int
diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringReport.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringReport.kt
index 8e550616e5..9ab736879a 100644
--- a/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringReport.kt
+++ b/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringReport.kt
@@ -7,21 +7,17 @@ import org.utbot.contest.StatsForClass
 @Serializable
 data class MonitoringReport(
     val parameters: MonitoringParameters,
-    val summarised_metrics: SummarisedMetricsReport,
     val targets: List<TargetReport>,
 ) {
-    constructor(parameters: MonitoringParameters, targets: List<TargetReport>): this(
-        parameters,
-        SummarisedMetricsReport(targets),
-        targets
-    )
 
     constructor(parameters: MonitoringParameters, stats: GlobalStats) : this(
         parameters,
-        stats.statsForClasses.map {
+        stats.projectStats.map { projectStats ->
             TargetReport(
-                TargetDescription(it.project, it.className),
-                ClassMetricsReport(it)
+                projectStats.project,
+                projectStats.statsForClasses.map {
+                    ClassReport(it.className, ClassMetricsReport(it))
+                }
             )
         }
     )
@@ -36,14 +32,21 @@ data class MonitoringParameters(
 
 @Serializable
 data class TargetReport(
-    val target: TargetDescription,
-    val metrics: ClassMetricsReport
-)
+    val target: String,
+    val summarised_metrics: SummarisedMetricsReport,
+    val metrics_by_class: List<ClassReport>
+) {
+    constructor(target: String, metrics_by_class: List<ClassReport>): this(
+        target,
+        SummarisedMetricsReport(metrics_by_class),
+        metrics_by_class
+    )
+}
 
 @Serializable
-data class TargetDescription(
-    val project: String,
-    val classname: String
+data class ClassReport(
+    val class_name: String,
+    val metrics: ClassMetricsReport
 )
 
 private fun Int.cover(total: Int): Double =
@@ -100,7 +103,7 @@ data class SummarisedMetricsReport(
     val bytecode_instructions_coverage_by_concolic: Double = covered_bytecode_instructions_by_concolic.cover(total_bytecode_instructions),
     val averaged_bytecode_instruction_coverage_by_classes: Double
 ) {
-    constructor(targets: Collection<TargetReport>) : this(
+    constructor(targets: Collection<ClassReport>) : this(
         total_classes = targets.size,
         testcases_generated = targets.sumOf { it.metrics.testcases_generated },
         classes_failed_to_compile = targets.count { it.metrics.failed_to_compile },
diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/StatisticsMonitoring.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/StatisticsMonitoring.kt
index d36e90c2ae..6246f089d5 100644
--- a/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/StatisticsMonitoring.kt
+++ b/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/StatisticsMonitoring.kt
@@ -21,6 +21,8 @@ private val javaHome = System.getenv("JAVA_HOME")
 private val logger = KotlinLogging.logger {}
 
 fun main(args: Array<String>) {
+    val outputFile = args.getOrNull(0)?.let { File(it) }
+
     logger.info { "Monitoring Settings:\n$MonitoringSettings" }
 
     val methodFilter: String?
@@ -75,7 +77,6 @@ fun main(args: Array<String>) {
     if (measurementResults.isEmpty())
         exitProcess(1)
 
-    val outputFile = args.getOrNull(0)?.let { File(it) }
     outputFile?.let { file ->
         val format = Json { prettyPrint = true }
         val jsonString = format.encodeToString(measurementResults)

From 42415cafeb4c98429068328ad62ffd70d6bdaff8 Mon Sep 17 00:00:00 2001
From: Rustam Sadykov <snob-1432e@mail.ru>
Date: Tue, 8 Nov 2022 15:10:35 +0300
Subject: [PATCH 05/18] update coroutines management

---
 .../src/main/kotlin/org/utbot/contest/Contest.kt           | 4 ++--
 .../kotlin/org/utbot/monitoring/StatisticsMonitoring.kt    | 7 +++++++
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt
index 4f208cd0e3..8e2646d1e0 100644
--- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt
+++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt
@@ -290,7 +290,7 @@ fun runGeneration(
                     UtSettings.fuzzingTimeoutInMillis = (budgetForMethod * fuzzingRatio).toLong()
 
                     //start controller that will activate symbolic execution
-                    GlobalScope.launch {
+                    GlobalScope.launch(currentContext) {
                         delay(budgetForSymbolicExecution)
 
                         if (methodJob.isActive) {
@@ -371,7 +371,7 @@ fun runGeneration(
         }
 
 
-        val cancellator = GlobalScope.launch {
+        val cancellator = GlobalScope.launch(currentContext) {
             delay(remainingBudget())
             if (engineJob.isActive) {
                 logger.warn { "Cancelling job because timeout $generationTimeout ms elapsed (real cancellation can take time)" }
diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/StatisticsMonitoring.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/StatisticsMonitoring.kt
index 6246f089d5..9eb711779e 100644
--- a/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/StatisticsMonitoring.kt
+++ b/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/StatisticsMonitoring.kt
@@ -3,6 +3,10 @@ package org.utbot.monitoring
 import java.io.File
 import java.util.concurrent.TimeUnit
 import kotlin.system.exitProcess
+import kotlinx.coroutines.GlobalScope
+import kotlinx.coroutines.cancel
+import kotlinx.coroutines.cancelChildren
+import kotlinx.coroutines.job
 import kotlinx.serialization.encodeToString
 import kotlinx.serialization.json.Json
 import mu.KotlinLogging
@@ -14,6 +18,7 @@ import org.utbot.contest.GlobalStats
 import org.utbot.contest.Paths
 import org.utbot.contest.Tool
 import org.utbot.contest.runEstimator
+import org.utbot.framework.plugin.api.util.utContext
 import org.utbot.framework.plugin.services.JdkInfoService
 import org.utbot.instrumentation.ConcreteExecutor
 
@@ -71,6 +76,8 @@ fun main(args: Array<String>) {
 
         }
 
+        utContext.cancelChildren()
+        utContext.cancel()
         ConcreteExecutor.defaultPool.forceTerminateProcesses()
         System.gc()
     }

From ca911d39870fc4ad7c6637849dd612e4d591c3ed Mon Sep 17 00:00:00 2001
From: Rustam Sadykov <snob-1432e@mail.ru>
Date: Tue, 8 Nov 2022 15:15:05 +0300
Subject: [PATCH 06/18] update monitoring scripts

---
 monitoring/build_aggregated_data.py | 129 --------------------------
 monitoring/insert_metadata.py       | 138 ++++++++++++++++++++++++++--
 monitoring/monitoring.properties    |   8 +-
 monitoring/monitoring_settings.py   |   7 +-
 monitoring/utils.py                 |  56 -----------
 5 files changed, 137 insertions(+), 201 deletions(-)
 delete mode 100644 monitoring/build_aggregated_data.py
 delete mode 100644 monitoring/utils.py

diff --git a/monitoring/build_aggregated_data.py b/monitoring/build_aggregated_data.py
deleted file mode 100644
index da2a873a99..0000000000
--- a/monitoring/build_aggregated_data.py
+++ /dev/null
@@ -1,129 +0,0 @@
-import argparse
-import json
-from os import listdir
-from os.path import isfile, join
-from time import time
-from typing import Iterator
-
-from monitoring_settings import JSON_VERSION
-from utils import *
-
-
-def get_file_seq(input_data_dir: str) -> Iterator[str]:
-    """
-    Get all files from specified directory
-    :param input_data_dir: path to directory with files
-    :return: sequence of filepaths
-    """
-    for filename in listdir(input_data_dir):
-        path = join(input_data_dir, filename)
-        if isfile(path):
-            yield path
-
-
-def check_stats(stats: dict, args: argparse.Namespace) -> bool:
-    """
-    Checks timestamp and version of given statistics
-    :param stats: dictionary with statistics and metadata
-    :param args: parsed program arguments
-    :return: is timestamp and version match
-    """
-    try:
-        timestamp = stats["metadata"]["timestamp"]
-        timestamp_match = args.timestamp_from <= timestamp <= args.timestamp_to
-        json_version_match = stats["version"] == JSON_VERSION
-        return timestamp_match and json_version_match
-    except:
-        return False
-
-
-def get_stats_seq(args: argparse.Namespace) -> Iterator[dict]:
-    """
-    Get statistics with metadata matched specified period
-    :param args: parsed program arguments
-    :return: sequence of statistics with metadata filtered by version and timestamp
-    """
-    for file in get_file_seq(args.input_data_dir):
-        with open(file, "r") as f:
-            stats = json.load(f)
-        if check_stats(stats, args):
-            yield stats
-
-
-def transform_target_stats(stats: dict) -> dict:
-    """
-    Transform metrics by computing total coverage
-    :param stats: metrics
-    :return: transformed metrics
-    """
-    common_prefix = "covered_instructions"
-    denum = stats["total_instructions"]
-
-    nums_keys = [(key, key.removeprefix(common_prefix)) for key in stats.keys() if key.startswith(common_prefix)]
-
-    for (key, by) in nums_keys:
-        num = stats[key]
-        stats["total_coverage" + by] = 100 * num / denum if denum != 0 else 0
-        del stats[key]
-
-    del stats["total_instructions"]
-
-    return stats
-
-
-def aggregate_stats(stats_seq: Iterator[dict]) -> List[dict]:
-    """
-    Aggregate list of metrics and parameters into list of transformed metrics and parameters grouped by targets
-    :param stats_seq: sequence of metrics and parameters
-    :return: list of metrics and parameters grouped by targets
-    """
-    result = get_default_metrics_dict()
-
-    for stats in stats_seq:
-        targets = stats["targets"]
-        timestamp = stats["metadata"]["timestamp"]
-        for target in targets:
-            full_name = f'{target["id"]}-{target["version"]}'
-            new_data = result[full_name]
-            for target_stats in target["metrics"]:
-                new_data["metrics"].append(transform_target_stats(target_stats))
-            for target_params in target["parameters"]:
-                target_params["timestamp"] = timestamp
-                new_data["parameters"].append(target_params)
-
-    return postprocess_targets(result)
-
-
-def get_args():
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        '--input_data_dir', required=True,
-        help='input directory with data', type=str
-    )
-    parser.add_argument(
-        '--output_file', required=True,
-        help='output file', type=str
-    )
-    parser.add_argument(
-        '--timestamp_from', help='timestamp started collection from',
-        type=int, default=0
-    )
-    parser.add_argument(
-        '--timestamp_to', help='timestamp finished collection to',
-        type=int, default=int(time())
-    )
-
-    args = parser.parse_args()
-    return args
-
-
-def main():
-    args = get_args()
-    stats_seq = get_stats_seq(args)
-    result = aggregate_stats(stats_seq)
-    with open(args.output_file, "w") as f:
-        json.dump(result, f, indent=4)
-
-
-if __name__ == '__main__':
-    main()
diff --git a/monitoring/insert_metadata.py b/monitoring/insert_metadata.py
index e202762cb2..c9188125ac 100644
--- a/monitoring/insert_metadata.py
+++ b/monitoring/insert_metadata.py
@@ -1,15 +1,16 @@
 import argparse
 import json
+import re
 import subprocess
+from collections import OrderedDict
 from datetime import datetime
 from os import environ
 from os.path import exists
 from platform import uname
 from time import time
-from typing import Optional
+from typing import Optional, List
 
 from monitoring_settings import JSON_VERSION
-from utils import *
 
 
 def load(json_file: str) -> Optional[any]:
@@ -91,17 +92,142 @@ def build_metadata(args: argparse.Namespace) -> dict:
     return metadata
 
 
+def build_target(target_name: str) -> dict:
+    return {
+        "target": target_name,
+        "summarised": [],
+        "by_class": OrderedDict()
+    }
+
+
+def transform_metrics(metrics: dict) -> dict:
+    """
+    Transform given metrics with calculation coverage
+    :param metrics: given metrics
+    :return: transformed metrics
+    """
+    result = OrderedDict()
+
+    instr_count_prefix = "covered_bytecode_instructions"
+    total_instr_count_prefix = "total_bytecode_instructions"
+
+    coverage_prefix = "total_bytecode_instruction_coverage"
+
+    total_count = 0
+    for metric in metrics:
+        if metric.startswith(total_instr_count_prefix):
+            total_count = metrics[metric]
+            break
+
+    for metric in metrics:
+        if metric.startswith(total_instr_count_prefix):
+            continue
+        if metric.startswith(instr_count_prefix):
+            coverage = metrics[metric] / total_count if total_count > 0 else 0.0
+            result[coverage_prefix + metric.removeprefix(instr_count_prefix)] = coverage
+        else:
+            result[metric] = metrics[metric]
+
+    return result
+
+
+def build_data(parameters: dict, metrics: dict) -> dict:
+    return {
+        "parameters": {
+            **parameters
+        },
+        "metrics": {
+            **transform_metrics(metrics)
+        }
+    }
+
+
+def build_by_class(class_name: str) -> dict:
+    return {
+        "class_name": class_name,
+        "data": []
+    }
+
+
+def update_from_class(by_class: dict, class_item: dict, parameters: dict):
+    """
+    Update class object using given class_item
+    :param by_class: dictionary with classname keys
+    :param class_item: class metrics of current run
+    :param parameters: parameters of current run
+    """
+    class_name = class_item["class_name"]
+    if class_name not in by_class:
+        by_class[class_name] = build_by_class(class_name)
+
+    metrics = class_item["metrics"]
+    by_class[class_name]["data"].append(
+        build_data(parameters, metrics)
+    )
+
+
+def update_from_target(targets: dict, target_item: dict, parameters: dict):
+    """
+    Update targets using given target_item
+    :param targets: dictionary with target keys
+    :param target_item: metrics of current run
+    :param parameters: parameters of current run
+    """
+    target_name = target_item["target"]
+    if target_name not in targets:
+        targets[target_name] = build_target(target_name)
+
+    summarised_metrics = target_item["summarised_metrics"]
+    targets[target_name]["summarised"].append(
+        build_data(parameters, summarised_metrics)
+    )
+
+    for class_item in target_item["metrics_by_class"]:
+        update_from_class(targets[target_name]["by_class"], class_item, parameters)
+
+
+def update_from_stats(targets: dict, stats: dict):
+    """
+    Updates targets using given statistics
+    :param targets: dictionary with target keys
+    :param stats: target object
+    """
+    parameters = stats["parameters"]
+    for target_item in stats["targets"]:
+        update_from_target(targets, target_item, parameters)
+
+
+def postprocess_by_class(by_class: dict) -> List[dict]:
+    """
+    Transform dictionary with classname keys into array with class objects
+    :param by_class: dictionary with classname keys
+    :return: array of class objects
+    """
+    return list(by_class.values())
+
+
+def postprocess_targets(targets: dict) -> List[dict]:
+    """
+    Transform dictionary with target keys into array with target objects
+    :param targets: dictionary with target keys
+    :return: array of targets
+    """
+    result = []
+    for target in targets.values():
+        target["by_class"] = postprocess_by_class(target["by_class"])
+        result.append(target)
+    return result
+
+
 def build_targets(stats_array: List[dict]) -> List[dict]:
     """
     Collect and group statistics by target
     :param stats_array: list of dictionaries with parameters and metrics
     :return: list of metrics and parameters grouped by target
     """
-    result = get_default_metrics_dict()
+    result = OrderedDict()
     for stats in stats_array:
-        target = stats['parameters']['target']
-        del stats['parameters']['target']
-        update_target(result[target], stats)
+        update_from_stats(result, stats)
 
     return postprocess_targets(result)
 
diff --git a/monitoring/monitoring.properties b/monitoring/monitoring.properties
index e3062857d0..0a1694eb88 100644
--- a/monitoring/monitoring.properties
+++ b/monitoring/monitoring.properties
@@ -1,4 +1,4 @@
-project=guava
-classTimeoutMillis=20
-runTries=1
-runTimeoutMinutes=20
\ No newline at end of file
+projects=guava
+classTimeoutSeconds=20
+runTimeoutMinutes=20
+fuzzingRatios=0.0;0.1;1.0
\ No newline at end of file
diff --git a/monitoring/monitoring_settings.py b/monitoring/monitoring_settings.py
index 301e7b7ae3..8b0fb83a0f 100644
--- a/monitoring/monitoring_settings.py
+++ b/monitoring/monitoring_settings.py
@@ -1,9 +1,4 @@
 """
 Json format version.
 """
-JSON_VERSION = 1
-
-"""
-Default version for projects without it.
-"""
-DEFAULT_PROJECT_VERSION = "0"
+JSON_VERSION = 2
diff --git a/monitoring/utils.py b/monitoring/utils.py
deleted file mode 100644
index d5fd1a5a2f..0000000000
--- a/monitoring/utils.py
+++ /dev/null
@@ -1,56 +0,0 @@
-import re
-from collections import defaultdict
-from typing import List
-
-from monitoring_settings import DEFAULT_PROJECT_VERSION
-
-
-def parse_name_and_version(full_name: str) -> tuple[str, str]:
-    """
-    Parse from string name and version of project
-    :param full_name: string with format <name>-<version>
-    :return: pair of name and version strings
-    """
-    regex = re.compile(r'([^.]+)-([\d.]+)')
-    result = regex.fullmatch(full_name)
-    if result is None:
-        return full_name, DEFAULT_PROJECT_VERSION
-    name = result.group(1)
-    version = result.group(2)
-    return name, version
-
-
-def postprocess_targets(targets: dict) -> List[dict]:
-    """
-    Transform dictionary with fullname target keys into array with target objects
-    :param targets: dictionary with fullname target keys
-    :return: array of targets
-    """
-    result = []
-    for target in targets:
-        (name, version) = parse_name_and_version(target)
-        result.append({
-            'id': name,
-            'version': version,
-            **targets[target]
-        })
-    return result
-
-
-def get_default_metrics_dict() -> dict:
-    return defaultdict(lambda: {
-        'parameters': [],
-        'metrics': []
-    })
-
-
-def update_target(target: dict, stats: dict) -> dict:
-    """
-    Update dictionary with target by new stats
-    :param target: dictionary with target metrics and parameters
-    :param stats: new metrics and parameters
-    :return: updated target dictionary
-    """
-    target['parameters'].append(stats['parameters'])
-    target['metrics'].append(stats['metrics'])
-    return target

From ad45fc9c9218a8dd3ac173a3f84c24670dc0a2d5 Mon Sep 17 00:00:00 2001
From: Rustam Sadykov <snob-1432e@mail.ru>
Date: Tue, 8 Nov 2022 15:15:35 +0300
Subject: [PATCH 07/18] update docs, add comments

---
 docs/NightStatisticsMonitoring.md | 377 +++++++++++-------------------
 1 file changed, 134 insertions(+), 243 deletions(-)

diff --git a/docs/NightStatisticsMonitoring.md b/docs/NightStatisticsMonitoring.md
index 2d5ae26cd3..e7866ec575 100644
--- a/docs/NightStatisticsMonitoring.md
+++ b/docs/NightStatisticsMonitoring.md
@@ -35,79 +35,53 @@ stats.json
 
 Output example (the result of three runs during one night):
 
-```json
+```json5
 [
   {
     "parameters": {
-      "target": "guava",
-      "class_timeout_sec": 20,
-      "run_timeout_min": 20
+      "fuzzing_ratio": 0.1, // how long does fuzzing takes
+      "class_timeout_sec": 20, // test generation timeout for one class
+      "run_timeout_min": 20 // total timeout for this run
     },
-    "metrics": {
-      "duration_ms": 604225,
-      "classes_for_generation": 20,
-      "testcases_generated": 1651,
-      "classes_without_problems": 12,
-      "classes_canceled_by_timeout": 2,
-      "total_methods_for_generation": 519,
-      "methods_with_at_least_one_testcase_generated": 365,
-      "methods_with_exceptions": 46,
-      "suspicious_methods": 85,
-      "test_classes_failed_to_compile": 0,
-      "covered_instructions": 5753,
-      "covered_instructions_by_fuzzing": 4375,
-      "covered_instructions_by_concolic": 4069,
-      "total_instructions": 10182,
-      "avg_coverage": 62.885408034613
-    }
-  },
-  {
-    "parameters": {
-      "target": "guava",
-      "class_timeout_sec": 20,
-      "run_timeout_min": 20
-    },
-    "metrics": {
-      "duration_ms": 633713,
-      "classes_for_generation": 20,
-      "testcases_generated": 1872,
-      "classes_without_problems": 12,
-      "classes_canceled_by_timeout": 2,
-      "total_methods_for_generation": 519,
-      "methods_with_at_least_one_testcase_generated": 413,
-      "methods_with_exceptions": 46,
-      "suspicious_methods": 38,
-      "test_classes_failed_to_compile": 0,
-      "covered_instructions": 6291,
-      "covered_instructions_by_fuzzing": 4470,
-      "covered_instructions_by_concolic": 5232,
-      "total_instructions": 11011,
-      "avg_coverage": 62.966064315865275
-    }
-  },
-  {
-    "parameters": {
-      "target": "guava",
-      "class_timeout_sec": 20,
-      "run_timeout_min": 20
-    },
-    "metrics": {
-      "duration_ms": 660421,
-      "classes_for_generation": 20,
-      "testcases_generated": 1770,
-      "classes_without_problems": 13,
-      "classes_canceled_by_timeout": 2,
-      "total_methods_for_generation": 519,
-      "methods_with_at_least_one_testcase_generated": 405,
-      "methods_with_exceptions": 44,
-      "suspicious_methods": 43,
-      "test_classes_failed_to_compile": 0,
-      "covered_instructions": 6266,
-      "covered_instructions_by_fuzzing": 4543,
-      "covered_instructions_by_concolic": 5041,
-      "total_instructions": 11011,
-      "avg_coverage": 61.59069193429194
-    }
+    "targets": [ // projects that have been processed
+      {
+        "target": "guava", // name of project
+        "summarised_metrics": { // summarised metrics for processed project 
+          "total_classes": 20, // classes count
+          "testcases_generated": 1042, // generated unit-tests count
+          "classes_failed_to_compile": 0, // classes that's tests are not compilable
+          "classes_canceled_by_timeout": 4, // classes that's generation was canceled because of timeout
+          "total_methods": 526, // methods count
+          "methods_with_at_least_one_testcase_generated": 345, // methods with at least one successfully generated test
+          "methods_with_at_least_one_exception": 32, // methods that's generation contains exceptions
+          "methods_without_any_tests_and_exceptions": 59, // suspicious methods without any tests and exceptions
+          "covered_bytecode_instructions": 4240, // amount of bytecode instructions that were covered by generated tests
+          "covered_bytecode_instructions_by_fuzzing": 2946, // amount of bytecode instructions that were covered by fuzzing's generated tests
+          "covered_bytecode_instructions_by_concolic": 3464, // amount of bytecode instructions that were covered by concolic's generated tests
+          "total_bytecode_instructions": 9531, // total amount of bytecode instructions in methods with at least one testcase generated 
+          "averaged_bytecode_instruction_coverage_by_classes": 0.5315060991492891 // mean bytecode coverage by class
+        },
+        "metrics_by_class": [ // metrics for all classes in this project
+          {
+            "class_name": "com.google.common.math.LongMath", // name of processed class
+            "metrics": { // metrics for specified class
+              "testcases_generated": 91, // amount of generated unit-tests 
+              "failed_to_compile": false, // compilation generated tests are failure 
+              "canceled_by_timeout": true, // generation is interrupted because of timeout
+              "total_methods_in_class": 31, // methods count in this class
+              "methods_with_at_least_one_testcase_generated": 26, // methods with at least one successfully generated test
+              "methods_with_at_least_one_exception": 0, // methods that's generation contains exceptions
+              "methods_without_any_tests_and_exceptions": 5, // suspicious methods without any tests and exceptions
+              "covered_bytecode_instructions_in_class": 585, // amount of bytecode instructions that were covered by generated tests
+              "covered_bytecode_instructions_in_class_by_fuzzing": 489, // amount of bytecode instructions that were covered by fuzzing's generated tests
+              "covered_bytecode_instructions_in_class_by_concolic": 376, // amount of bytecode instructions that were covered by concolic's generated tests
+              "total_bytecode_instructions_in_class": 1442 // total amount of bytecode instructions in methods with at least one testcase generated 
+            }
+          },
+          // ...
+        ]
+      }
+    ]
   }
 ]
 ```
@@ -120,7 +94,7 @@ The `insert_metadata.py` script is responsible for doing this. To run it you hav
 
 To get more information about input arguments call script with option `--help`.
 
-Output format: you get the JSON file, containing statistics and parameters grouped by target project and metadata.
+Output format: you get the JSON file, containing metadata, statistics and parameters grouped by target project and classes.
 
 Input example:
 ```
@@ -131,93 +105,78 @@ Input example:
 ```
 
 Output example (statistics followed by metadata):
-```json
+```json5
 {
-  "version": 1,
-  "targets": [
+  "version": 2, // version of json format
+  "targets": [ // projects and methods that have been processed
     {
-      "id": "guava",
-      "version": "0",
-      "parameters": [
+      "target": "guava", // name of project
+      "summarised": [ // list of summarised metrics with parameters on each run
         {
-          "class_timeout_sec": 20,
-          "run_timeout_min": 20
+          "parameters": {
+            "fuzzing_ratio": 0.1, // how long does fuzzing takes
+            "class_timeout_sec": 20, // test generation timeout for one class
+            "run_timeout_min": 20 // total timeout for this run
+          },
+          "metrics": {
+            "total_classes": 20, // classes count
+            "testcases_generated": 1042, // generated unit-tests count
+            "classes_failed_to_compile": 0, // classes that's tests are not compilable
+            "classes_canceled_by_timeout": 4, // classes that's generation was canceled because of timeout
+            "total_methods": 526, // methods count
+            "methods_with_at_least_one_testcase_generated": 345, // methods with at least one successfully generated test
+            "methods_with_at_least_one_exception": 32, // methods that's generation contains exceptions
+            "methods_without_any_tests_and_exceptions": 59, // suspicious methods without any tests and exceptions
+            "total_bytecode_instruction_coverage": 0.44486412758, // total bytecode coverage of generated tests
+            "total_bytecode_instruction_coverage_by_fuzzing": 0.30909663204, // total bytecode coverage of fuzzing's generated tests
+            "total_bytecode_instruction_coverage_by_concolic": 0.36344559857, // total bytecode coverage of concolic's generated tests
+            "averaged_bytecode_instruction_coverage_by_classes": 0.5315060991492891 // mean bytecode coverage by class
+          }
         },
-        {
-          "class_timeout_sec": 20,
-          "run_timeout_min": 20
-        },
-        {
-          "class_timeout_sec": 20,
-          "run_timeout_min": 20
-        }
+        // ...
       ],
-      "metrics": [
-        {
-          "duration_ms": 604225,
-          "classes_for_generation": 20,
-          "testcases_generated": 1651,
-          "classes_without_problems": 12,
-          "classes_canceled_by_timeout": 2,
-          "total_methods_for_generation": 519,
-          "methods_with_at_least_one_testcase_generated": 365,
-          "methods_with_exceptions": 46,
-          "suspicious_methods": 85,
-          "test_classes_failed_to_compile": 0,
-          "covered_instructions": 5753,
-          "covered_instructions_by_fuzzing": 4375,
-          "covered_instructions_by_concolic": 4069,
-          "total_instructions": 10182,
-          "avg_coverage": 62.885408034613
-        },
+      "by_class": [ // list of metrics and parameters for all classes in project and on each run
         {
-          "duration_ms": 633713,
-          "classes_for_generation": 20,
-          "testcases_generated": 1872,
-          "classes_without_problems": 12,
-          "classes_canceled_by_timeout": 2,
-          "total_methods_for_generation": 519,
-          "methods_with_at_least_one_testcase_generated": 413,
-          "methods_with_exceptions": 46,
-          "suspicious_methods": 38,
-          "test_classes_failed_to_compile": 0,
-          "covered_instructions": 6291,
-          "covered_instructions_by_fuzzing": 4470,
-          "covered_instructions_by_concolic": 5232,
-          "total_instructions": 11011,
-          "avg_coverage": 62.966064315865275
+          "class_name": "com.google.common.math.LongMath", // name of processed class
+          "data": [ // metrics and parameters on each run
+            {
+              "parameters": { // parameters on this run
+                "fuzzing_ratio": 0.1, // how long does fuzzing takes
+                "class_timeout_sec": 20, // test generation timeout for one class
+                "run_timeout_min": 20 // total timeout for this run
+              },
+              "metrics": { // metrics for specified class on this run
+                "testcases_generated": 91, // amount of generated unit-tests 
+                "failed_to_compile": false, // compilation generated tests are failure 
+                "canceled_by_timeout": true, // generation is interrupted because of timeout
+                "total_methods_in_class": 31, // methods count in this class
+                "methods_with_at_least_one_testcase_generated": 26, // methods with at least one successfully generated test
+                "methods_with_at_least_one_exception": 0, // methods that's generation contains exceptions
+                "methods_without_any_tests_and_exceptions": 5, // suspicious methods without any tests and exceptions
+                "total_bytecode_instruction_coverage_in_class": 0.40568654646, // bytecode coverage of generated tests
+                "total_bytecode_instruction_coverage_in_class_by_fuzzing": 0.33911234396, // bytecode coverage of fuzzing's generated tests
+                "total_bytecode_instruction_coverage_in_class_by_concolic": 0.26074895977 // bytecode coverage of concolic's generated tests
+              }
+            },
+            // ...
+          ]
         },
-        {
-          "duration_ms": 660421,
-          "classes_for_generation": 20,
-          "testcases_generated": 1770,
-          "classes_without_problems": 13,
-          "classes_canceled_by_timeout": 2,
-          "total_methods_for_generation": 519,
-          "methods_with_at_least_one_testcase_generated": 405,
-          "methods_with_exceptions": 44,
-          "suspicious_methods": 43,
-          "test_classes_failed_to_compile": 0,
-          "covered_instructions": 6266,
-          "covered_instructions_by_fuzzing": 4543,
-          "covered_instructions_by_concolic": 5041,
-          "total_instructions": 11011,
-          "avg_coverage": 61.59069193429194
-        }
+        // ...
       ]
-    }
+    },
+    // ...
   ],
-  "metadata": {
-    "source": {
+  "metadata": { // device's properties
+    "source": { // information about runner
       "type": "github-action",
       "id": "2917672580"
     },
-    "commit_hash": "66a1aeb6",
-    "branch": "main",
-    "build_number": "2022.8",
-    "timestamp": 1661330445,
-    "date": "2022-08-24T08:40:45",
-    "environment": {
+    "commit_hash": "66a1aeb6", // commit hash of used utbot build 
+    "branch": "main", // branch of used utbot build
+    "build_number": "2022.8", // build number of used utbot build
+    "timestamp": 1661330445, // run timestamp
+    "date": "2022-08-24T08:40:45", // human-readable run timestamp
+    "environment": { // device's environment
       "host": "fv-az183-700",
       "OS": "Linux version #20~20.04.1-Ubuntu SMP Fri Aug 5 12:16:53 UTC 2022",
       "java_version": "openjdk version \"11.0.16\" 2022-07-19 LTS\nOpenJDK Runtime Environment Zulu11.58+15-CA (build 11.0.16+8-LTS)\nOpenJDK 64-Bit Server VM Zulu11.58+15-CA (build 11.0.16+8-LTS, mixed mode)\n",
@@ -230,112 +189,44 @@ Output example (statistics followed by metadata):
 }
 ```
 
-### Aggregating
+### Datastorage structure
 
-The `build_aggregated_data.py` script gathers the results for several nights. The collected results for each of the nights are put together into one array. You can specify the period for aggregating. It is useful for visualising or finding statistical characteristics of UnitTestBot performance, e.g. the median or max/min values.
+We store the collected statistics in our repository. You can find two special branches: `monitoring-data` and `monitoring-aggregated-data`.
 
-To run aggregating you should provide the input.
+The `monitoring-data` branch is a storage for raw statistics data as well as metadata.
 
-To get more information about input arguments call script with option `--help`.
+The filename format: `<yyyy>-<mm>-<dd>-<hh>-<MM>-<ss>-<branch>-<short commit hash>.json`
 
-Output format: you get the JSON file, which contains arrays of grouped by target results for each of the nights during the specified period.
+### Grafana (in process)
 
-Input example:
+#### Usage
 
-```
---input_data_dir ./data --output_file aggregated_data.json 
---timestamp_from 0 --timestamp_to 1661330445
-```
+We can use [Grafana](https://monitoring.utbot.org) for more dynamic and detailed statistics visualisation. Grafana pulls data from our repository automatically.
+
+#### Metrics format
 
-Output example (You'll get an array of several outputs without metadata):
-```json
+```json5
 [
+  // summarised
   {
-    "id": "guava",
-    "version": "0",
-    "parameters": [
-      {
-        "class_timeout_sec": 20,
-        "run_timeout_min": 20,
-        "timestamp": 1661330445
-      },
-      {
-        "class_timeout_sec": 20,
-        "run_timeout_min": 20,
-        "timestamp": 1661330445
-      },
-      {
-        "class_timeout_sec": 20,
-        "run_timeout_min": 20,
-        "timestamp": 1661330445
-      }
-    ],
-    "metrics": [
-      {
-        "duration_ms": 604225,
-        "classes_for_generation": 20,
-        "testcases_generated": 1651,
-        "classes_without_problems": 12,
-        "classes_canceled_by_timeout": 2,
-        "total_methods_for_generation": 519,
-        "methods_with_at_least_one_testcase_generated": 365,
-        "methods_with_exceptions": 46,
-        "suspicious_methods": 85,
-        "test_classes_failed_to_compile": 0,
-        "avg_coverage": 62.885408034613,
-        "total_coverage": 56.50166961304262,
-        "total_coverage_by_fuzzing": 42.967982714594385,
-        "total_coverage_by_concolic": 39.96267923787075
-      },
-      {
-        "duration_ms": 633713,
-        "classes_for_generation": 20,
-        "testcases_generated": 1872,
-        "classes_without_problems": 12,
-        "classes_canceled_by_timeout": 2,
-        "total_methods_for_generation": 519,
-        "methods_with_at_least_one_testcase_generated": 413,
-        "methods_with_exceptions": 46,
-        "suspicious_methods": 38,
-        "test_classes_failed_to_compile": 0,
-        "avg_coverage": 62.966064315865275,
-        "total_coverage": 57.133775315593496,
-        "total_coverage_by_fuzzing": 40.595767868495145,
-        "total_coverage_by_concolic": 47.51612024339297
-      },
-      {
-        "duration_ms": 660421,
-        "classes_for_generation": 20,
-        "testcases_generated": 1770,
-        "classes_without_problems": 13,
-        "classes_canceled_by_timeout": 2,
-        "total_methods_for_generation": 519,
-        "methods_with_at_least_one_testcase_generated": 405,
-        "methods_with_exceptions": 44,
-        "suspicious_methods": 43,
-        "test_classes_failed_to_compile": 0,
-        "avg_coverage": 61.59069193429194,
-        "total_coverage": 56.90672963400236,
-        "total_coverage_by_fuzzing": 41.25874125874126,
-        "total_coverage_by_concolic": 45.78149123603669
-      }
-    ]
-  }
+    "metric": "testcases_generated",
+    "labels": {
+      "project": "guava",
+      "fuzzing_ratio": 0.1
+    },
+    "value": 1024
+  },
+  // ...
+  // by classes
+  {
+    "metric": "testcases_generated",
+    "labels": {
+      "project": "guava",
+      "class": "com.google.common.math.LongMath",
+      "fuzzing_ratio": 0.1
+    },
+    "value": 91
+  },
+  // ...
 ]
 ```
-
-### Datastorage structure
-
-We store the collected statistics in our repository. You can find two special branches: `monitoring-data` and `monitoring-aggregated-data`.
-
-The `monitoring-data` branch is a storage for raw statistics data as well as metadata.
-
-The filename format: `data-<branch>-<yyyy>-<mm>-<dd>-<timestamp>-<short commit hash>.json`
-
-The `monitoring-aggregated-data` branch is a storage for aggregated statistics. The aggregating period is set to one month by default.
-
-The filename format: `aggregated-data-<yyyy>-<mm>-<dd>.json`
-
-### Grafana (in process)
-
-We can use [Grafana](https://monitoring.utbot.org) for more dynamic and detailed statistics visualisation. Grafana pulls data from our repository automatically by means of GitHub API.

From cd98c8c3427e6a8238db8a08f9d34788418b4d97 Mon Sep 17 00:00:00 2001
From: Rustam Sadykov <snob-1432e@mail.ru>
Date: Tue, 8 Nov 2022 15:56:40 +0300
Subject: [PATCH 08/18] add metrics preparing for grafana

---
 monitoring/insert_metadata.py | 14 +-----
 monitoring/prepare_metrics.py | 91 +++++++++++++++++++++++++++++++++++
 monitoring/utils.py           | 15 ++++++
 3 files changed, 107 insertions(+), 13 deletions(-)
 create mode 100644 monitoring/prepare_metrics.py
 create mode 100644 monitoring/utils.py

diff --git a/monitoring/insert_metadata.py b/monitoring/insert_metadata.py
index c9188125ac..10be3c4b45 100644
--- a/monitoring/insert_metadata.py
+++ b/monitoring/insert_metadata.py
@@ -5,24 +5,12 @@
 from collections import OrderedDict
 from datetime import datetime
 from os import environ
-from os.path import exists
 from platform import uname
 from time import time
 from typing import Optional, List
 
 from monitoring_settings import JSON_VERSION
-
-
-def load(json_file: str) -> Optional[any]:
-    """
-    Try load object from json file
-    :param json_file: path to json file
-    :return: object from given json file or None
-    """
-    if exists(json_file):
-        with open(json_file, "r") as f:
-            return json.load(f)
-    return None
+from utils import load
 
 
 def try_get_output(args: str) -> Optional[str]:
diff --git a/monitoring/prepare_metrics.py b/monitoring/prepare_metrics.py
new file mode 100644
index 0000000000..0960811c98
--- /dev/null
+++ b/monitoring/prepare_metrics.py
@@ -0,0 +1,91 @@
+import argparse
+import json
+from typing import List
+
+from utils import load
+
+
+def build_metric_struct(name: str, value: any, labels: dict) -> dict:
+    return {
+        "metric": name,
+        "labels": labels,
+        "value": value
+    }
+
+
+def build_metrics_from_data(data: dict, labels: dict) -> List[dict]:
+    result = []
+    fuzzing_ratio = data["parameters"]["fuzzing_ratio"]
+    new_labels = {
+        **labels,
+        "fuzzing_ratio": fuzzing_ratio
+    }
+    metrics = data["metrics"]
+    for metric in metrics:
+        result.append(build_metric_struct(metric, metrics[metric], new_labels))
+    return result
+
+
+def build_metrics_from_data_array(metrics: List[dict], labels: dict) -> List[dict]:
+    result = []
+    for metric in metrics:
+        result.extend(build_metrics_from_data(metric, labels))
+    return result
+
+
+def build_metrics_from_target(target: dict) -> List[dict]:
+    result = []
+    project = target["target"]
+
+    result.extend(build_metrics_from_data_array(
+        target["summarised"],
+        {
+            "project": project
+        }
+    ))
+
+    for class_item in target["by_class"]:
+        class_name = class_item["class_name"]
+        result.extend(build_metrics_from_data_array(
+            class_item["data"],
+            {
+                "project": project,
+                "class": class_name
+            }
+        ))
+
+    return result
+
+
+def build_metrics_from_targets(targets: List[dict]) -> List[dict]:
+    metrics = []
+    for target in targets:
+        metrics.extend(build_metrics_from_target(target))
+    return metrics
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--stats_file', required=True,
+        help='files with statistics after insertion metadata', type=str
+    )
+    parser.add_argument(
+        '--output_file', required=True,
+        help='output file', type=str
+    )
+
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+    stats = load(args.stats_file)
+    metrics = build_metrics_from_targets(stats["targets"])
+    with open(args.output_file, "w") as f:
+        json.dump(metrics, f, indent=4)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/monitoring/utils.py b/monitoring/utils.py
new file mode 100644
index 0000000000..c897f46df6
--- /dev/null
+++ b/monitoring/utils.py
@@ -0,0 +1,15 @@
+import json
+from os.path import exists
+from typing import Optional
+
+
+def load(json_file: str) -> Optional[any]:
+    """
+    Try load object from json file
+    :param json_file: path to json file
+    :return: object from given json file or None
+    """
+    if exists(json_file):
+        with open(json_file, "r") as f:
+            return json.load(f)
+    return None

From 2a7c2d9c9a76247b2bd32f8e97edeecd779ac77f Mon Sep 17 00:00:00 2001
From: Rustam Sadykov <snob-1432e@mail.ru>
Date: Tue, 8 Nov 2022 16:14:00 +0300
Subject: [PATCH 09/18] update ci

---
 .github/workflows/collect-statistics.yml      | 97 +++++++------------
 .../workflows/night-statistics-monitoring.yml |  3 +-
 2 files changed, 37 insertions(+), 63 deletions(-)

diff --git a/.github/workflows/collect-statistics.yml b/.github/workflows/collect-statistics.yml
index 52696b4c9a..82dd04bba5 100644
--- a/.github/workflows/collect-statistics.yml
+++ b/.github/workflows/collect-statistics.yml
@@ -18,8 +18,13 @@ on:
         required: false
         default: manual-run
         type: string
-      aggregate:
-        description: 'Aggregate data'
+      push_results:
+        description: 'Push metrics into github'
+        required: false
+        default: false
+        type: boolean
+      send_to_grafana:
+        description: 'Send metrics to grafana'
         required: false
         default: false
         type: boolean
@@ -41,8 +46,13 @@ on:
         required: false
         default: manual-run
         type: string
-      aggregate:
-        description: 'Aggregate data'
+      push_results:
+        description: 'Push metrics into github and grafana'
+        required: false
+        default: false
+        type: boolean
+      send_to_grafana:
+        description: 'Send metrics to grafana'
         required: false
         default: false
         type: boolean
@@ -50,8 +60,6 @@ on:
 env:
   data_branch: monitoring-data
   data_path: monitoring/data
-  aggregated_data_branch: monitoring-aggregated-data
-  aggregated_data_path: monitoring/aggregated_data
   monitoring_properties: monitoring/monitoring.properties
   push_script: monitoring/push_with_rebase.sh
 
@@ -112,7 +120,7 @@ jobs:
       - name: Get current date
         id: date
         run: |
-          echo "::set-output name=date::$(date +'%Y-%m-%d')"
+          echo "::set-output name=date::$(date +'%Y-%m-%d-%H-%M-%S')"
           echo "::set-output name=timestamp::$(date +%s)"
           echo "::set-output name=last_month::$(date --date='last month' +%s)"
 
@@ -125,14 +133,18 @@ jobs:
           echo "::set-output name=build::$(date +'%Y.%-m')"
 
       - name: Insert metadata
+        id: insert
         shell: bash
         run: |
-          OUT_FILE="$data_path/data-$branch-$date-$timestamp-$short_commit-${{ matrix.value }}.json"
+          OUT_FILE="$data_path/$date-$branch-$short_commit-${{ matrix.value }}.json"
+          echo "::set-output name=output::$OUT_FILE"
+
           INPUTS=($(seq ${{ inputs.run_number }}))
           INPUTS=(${INPUTS[@]/#/stats-})
           INPUTS=(${INPUTS[@]/%/.json})
           INPUTS=${INPUTS[@]}
           echo $INPUTS
+
           python monitoring/insert_metadata.py \
             --stats_file $INPUTS \
             --output_file "$OUT_FILE" \
@@ -151,7 +163,14 @@ jobs:
           build: ${{ steps.metadata.outputs.build }}
           run_id: ${{ github.run_id }}-${{ matrix.value }}
 
+      - name: Upload statistics
+        uses: actions/upload-artifact@v3
+        with:
+          name: statistics-${{ matrix.value }}
+          path: ${{ steps.insert.outputs.output }}
+
       - name: Commit and push statistics
+        if: ${{ inputs.push_results }}
         run: |
           chmod +x $push_script
           ./$push_script
@@ -161,63 +180,17 @@ jobs:
           message: ${{ inputs.message_prefix }}-${{ steps.date.outputs.date }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Send metrics to grafana
+        if: ${{ inputs.send_to_grafana }}
+        run: |
+          python monitoring/prepare_metrics.py --stats_file $stats_file --output_file grafana_metrics.json
+          echo "TODO send metrics to grafana"
+        env:
+          stats_file: ${{ steps.insert.outputs.output }}
+
       - name: Upload logs
         if: ${{ always() }}
         uses: actions/upload-artifact@v3
         with:
           name: logs-${{ matrix.value }}
           path: logs/
-
-  aggregate:
-    needs: build_and_collect_statistics
-    if: ${{ inputs.aggregate }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-
-      - name: Checkout monitoring data
-        uses: actions/checkout@v3
-        with:
-          ref: ${{ env.data_branch }}
-          path: ${{ env.data_path }}
-
-      - name: Checkout aggregated monitoring data
-        uses: actions/checkout@v3
-        with:
-          ref: ${{ env.aggregated_data_branch }}
-          path: ${{ env.aggregated_data_path }}
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: '3.9'
-
-      - name: Get current date
-        id: date
-        run: |
-          echo "::set-output name=date::$(date +'%Y-%m-%d')"
-          echo "::set-output name=timestamp::$(date +%s)"
-          echo "::set-output name=last_month::$(date --date='last month' +%s)"
-
-      - name: Build aggregated data (last month)
-        run: |
-          OUT_FILE=$aggregated_data_path/aggregated-data-$date.json
-          python monitoring/build_aggregated_data.py \
-            --input_data_dir $data_path \
-            --output_file $OUT_FILE \
-            --timestamp_from $timestamp_from \
-            --timestamp_to $timestamp
-        env:
-          date: ${{ steps.date.outputs.date }}
-          timestamp: ${{ steps.date.outputs.timestamp }}
-          timestamp_from: ${{ steps.date.outputs.last_month }}
-
-      - name: Commit and push aggregated statistics
-        run: |
-          chmod +x $push_script
-          ./$push_script
-        env:
-          target_branch: ${{ env.aggregated_data_branch }}
-          target_directory: ${{ env.aggregated_data_path }}
-          message: ${{ inputs.message_prefix }}-${{ steps.date.outputs.date }}
-          github_token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/night-statistics-monitoring.yml b/.github/workflows/night-statistics-monitoring.yml
index c7cda2307b..99af44d6ac 100644
--- a/.github/workflows/night-statistics-monitoring.yml
+++ b/.github/workflows/night-statistics-monitoring.yml
@@ -12,4 +12,5 @@ jobs:
       runners: 3
       run_number: 1
       message_prefix: night-monitoring
-      aggregate: true
+      push_results: true
+      send_to_grafana: true

From 5580dc25029d7b1eef286cdf826c4a6ddef65c36 Mon Sep 17 00:00:00 2001
From: Rustam Sadykov <snob-1432e@mail.ru>
Date: Tue, 8 Nov 2022 16:14:14 +0300
Subject: [PATCH 10/18] update docs

---
 docs/NightStatisticsMonitoring.md | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/docs/NightStatisticsMonitoring.md b/docs/NightStatisticsMonitoring.md
index e7866ec575..a0a115a847 100644
--- a/docs/NightStatisticsMonitoring.md
+++ b/docs/NightStatisticsMonitoring.md
@@ -195,9 +195,9 @@ We store the collected statistics in our repository. You can find two special br
 
 The `monitoring-data` branch is a storage for raw statistics data as well as metadata.
 
-The filename format: `<yyyy>-<mm>-<dd>-<hh>-<MM>-<ss>-<branch>-<short commit hash>.json`
+The filename format: `<yyyy>-<mm>-<dd>-<hh>-<MM>-<ss>-<branch>-<short commit hash>-<runner number>.json`
 
-### Grafana (in process)
+### Grafana
 
 #### Usage
 
@@ -205,6 +205,15 @@ We can use [Grafana](https://monitoring.utbot.org) for more dynamic and detailed
 
 #### Metrics format
 
+Our goal after collecting statistics is uploading results into grafana. For this we should prepare data and send it to our server.
+
+The `prepare_metrics.py` script is responsible for doing this. To run it you have to specify the following arguments.
+
+To get more information about input arguments call script with option `--help`.
+
+Output format: you get the JSON file, containing array of metrics with some labels.
+
+Output example:
 ```json5
 [
   // summarised

From c096b41f259b997b35e8f771bb7e986518239cea Mon Sep 17 00:00:00 2001
From: Rustam Sadykov <snob-1432e@mail.ru>
Date: Tue, 8 Nov 2022 16:44:29 +0300
Subject: [PATCH 11/18] add runner label

---
 .github/workflows/collect-statistics.yml |  4 ++--
 monitoring/prepare_metrics.py            | 11 +++++++----
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/collect-statistics.yml b/.github/workflows/collect-statistics.yml
index 82dd04bba5..41639c0f4d 100644
--- a/.github/workflows/collect-statistics.yml
+++ b/.github/workflows/collect-statistics.yml
@@ -9,7 +9,7 @@ on:
         default: '1'
         type: string
       run_number:
-        description: 'Number of run tries per runner'
+        description: 'Number of run tries per runner (values greater than 1 are not supported with grafana)'
         required: false
         default: '1'
         type: string
@@ -37,7 +37,7 @@ on:
         default: '1'
         type: string
       run_number:
-        description: 'Number of run tries per runner'
+        description: 'Number of run tries per runner (values greater than 1 are not supported with grafana)'
         required: false
         default: '1'
         type: string
diff --git a/monitoring/prepare_metrics.py b/monitoring/prepare_metrics.py
index 0960811c98..c1f918b4e6 100644
--- a/monitoring/prepare_metrics.py
+++ b/monitoring/prepare_metrics.py
@@ -33,13 +33,14 @@ def build_metrics_from_data_array(metrics: List[dict], labels: dict) -> List[dic
     return result
 
 
-def build_metrics_from_target(target: dict) -> List[dict]:
+def build_metrics_from_target(target: dict, runner: str) -> List[dict]:
     result = []
     project = target["target"]
 
     result.extend(build_metrics_from_data_array(
         target["summarised"],
         {
+            "runner": runner,
             "project": project
         }
     ))
@@ -49,6 +50,7 @@ def build_metrics_from_target(target: dict) -> List[dict]:
         result.extend(build_metrics_from_data_array(
             class_item["data"],
             {
+                "runner": runner,
                 "project": project,
                 "class": class_name
             }
@@ -57,10 +59,10 @@ def build_metrics_from_target(target: dict) -> List[dict]:
     return result
 
 
-def build_metrics_from_targets(targets: List[dict]) -> List[dict]:
+def build_metrics_from_targets(targets: List[dict], runner: str) -> List[dict]:
     metrics = []
     for target in targets:
-        metrics.extend(build_metrics_from_target(target))
+        metrics.extend(build_metrics_from_target(target, runner))
     return metrics
 
 
@@ -82,7 +84,8 @@ def get_args():
 def main():
     args = get_args()
     stats = load(args.stats_file)
-    metrics = build_metrics_from_targets(stats["targets"])
+    runner = stats["metadata"]["environment"]["host"]
+    metrics = build_metrics_from_targets(stats["targets"], runner)
     with open(args.output_file, "w") as f:
         json.dump(metrics, f, indent=4)
 

From 1c900ccf23450943d21e010e65dd2737160c2055 Mon Sep 17 00:00:00 2001
From: Rustam Sadykov <snob-1432e@mail.ru>
Date: Tue, 8 Nov 2022 17:25:28 +0300
Subject: [PATCH 12/18] fix context management

---
 .../org/utbot/contest/ContestEstimator.kt     |  2 +-
 .../org/utbot/contest/ContextManager.kt       | 23 +++++++++++++++++++
 .../utbot/monitoring/StatisticsMonitoring.kt  |  4 ++--
 3 files changed, 26 insertions(+), 3 deletions(-)
 create mode 100644 utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContextManager.kt

diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt
index 0f6ffebcb9..574d2878d6 100644
--- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt
+++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt
@@ -139,7 +139,7 @@ enum class Tool {
             classFqn: String
         ) {
             val classStats: StatsForClass = try {
-                withUtContext(UtContext(project.classloader)) {
+                withUtContext(ContextManager.createNewContext(project.classloader)) {
                     runGeneration(
                         project.name,
                         cut,
diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContextManager.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContextManager.kt
new file mode 100644
index 0000000000..0f69975437
--- /dev/null
+++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContextManager.kt
@@ -0,0 +1,23 @@
+package org.utbot.contest
+
+import kotlinx.coroutines.cancel
+import kotlinx.coroutines.cancelChildren
+import org.utbot.framework.plugin.api.util.UtContext
+
+object ContextManager {
+    private val currentContexts = mutableListOf<UtContext>()
+
+    fun createNewContext(classLoader: ClassLoader) =
+        UtContext(classLoader).apply {
+            currentContexts.add(this)
+        }
+
+    fun cancelAll() {
+        currentContexts.forEach {
+            it.cancelChildren()
+            it.cancel()
+        }
+        currentContexts.clear()
+    }
+
+}
\ No newline at end of file
diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/StatisticsMonitoring.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/StatisticsMonitoring.kt
index 9eb711779e..4b68efc398 100644
--- a/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/StatisticsMonitoring.kt
+++ b/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/StatisticsMonitoring.kt
@@ -14,6 +14,7 @@ import org.utbot.common.ThreadBasedExecutor
 import org.utbot.common.bracket
 import org.utbot.common.info
 import org.utbot.contest.ContestEstimatorJdkInfoProvider
+import org.utbot.contest.ContextManager
 import org.utbot.contest.GlobalStats
 import org.utbot.contest.Paths
 import org.utbot.contest.Tool
@@ -76,8 +77,7 @@ fun main(args: Array<String>) {
 
         }
 
-        utContext.cancelChildren()
-        utContext.cancel()
+        ContextManager.cancelAll()
         ConcreteExecutor.defaultPool.forceTerminateProcesses()
         System.gc()
     }

From bf68fea5e8567f62301ce95454fd7771d2db96d3 Mon Sep 17 00:00:00 2001
From: Rustam Sadykov <snob-1432e@mail.ru>
Date: Tue, 8 Nov 2022 17:29:27 +0300
Subject: [PATCH 13/18] fix property name

---
 .github/workflows/collect-statistics.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/collect-statistics.yml b/.github/workflows/collect-statistics.yml
index 41639c0f4d..026b2d4cca 100644
--- a/.github/workflows/collect-statistics.yml
+++ b/.github/workflows/collect-statistics.yml
@@ -47,7 +47,7 @@ on:
         default: manual-run
         type: string
       push_results:
-        description: 'Push metrics into github and grafana'
+        description: 'Push metrics into github'
         required: false
         default: false
         type: boolean

From 39d3bdc7f338990c95da00ea819ed502125939af Mon Sep 17 00:00:00 2001
From: Rustam Sadykov <snob-1432e@mail.ru>
Date: Tue, 8 Nov 2022 18:22:34 +0300
Subject: [PATCH 14/18] fix action

---
 .github/workflows/collect-statistics.yml | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/collect-statistics.yml b/.github/workflows/collect-statistics.yml
index 026b2d4cca..f916f9f49c 100644
--- a/.github/workflows/collect-statistics.yml
+++ b/.github/workflows/collect-statistics.yml
@@ -73,7 +73,7 @@ jobs:
         id: set-matrix
         run: |
           arr=$(echo [$(seq -s , ${{ inputs.runners }})])
-          echo "::set-output name=matrix::$arr"
+          echo "matrix=$arr" >> $GITHUB_OUTPUT
           echo $arr
 
   build_and_collect_statistics:
@@ -120,24 +120,24 @@ jobs:
       - name: Get current date
         id: date
         run: |
-          echo "::set-output name=date::$(date +'%Y-%m-%d-%H-%M-%S')"
-          echo "::set-output name=timestamp::$(date +%s)"
-          echo "::set-output name=last_month::$(date --date='last month' +%s)"
+          echo "date=$(date +'%Y-%m-%d-%H-%M-%S')" >> $GITHUB_OUTPUT
+          echo "timestamp=$(date +%s)" >> $GITHUB_OUTPUT
+          echo "last_month=$(date --date='last month' +%s)" >> $GITHUB_OUTPUT
 
       - name: Get metadata
         id: metadata
         run: |
-          echo "::set-output name=commit::$(git rev-parse HEAD)"
-          echo "::set-output name=short_commit::$(git rev-parse --short HEAD)"
-          echo "::set-output name=branch::$(git name-rev --name-only HEAD)"
-          echo "::set-output name=build::$(date +'%Y.%-m')"
+          echo "commit=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
+          echo "short_commit=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+          echo "branch=$(git name-rev --name-only HEAD)" >> $GITHUB_OUTPUT
+          echo "build=$(date +'%Y.%-m')" >> $GITHUB_OUTPUT
 
       - name: Insert metadata
         id: insert
         shell: bash
         run: |
           OUT_FILE="$data_path/$date-$branch-$short_commit-${{ matrix.value }}.json"
-          echo "::set-output name=output::$OUT_FILE"
+          echo "output=$OUT_FILE" >> $GITHUB_OUTPUT
 
           INPUTS=($(seq ${{ inputs.run_number }}))
           INPUTS=(${INPUTS[@]/#/stats-})

From dfb6292494d64278e13f223f24f97204d4c46260 Mon Sep 17 00:00:00 2001
From: Rustam Sadykov <snob-1432e@mail.ru>
Date: Wed, 9 Nov 2022 16:27:28 +0300
Subject: [PATCH 15/18] fix according to review

---
 .../src/main/kotlin/org/utbot/contest/ContestEstimator.kt  | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt
index 574d2878d6..7c82e668f2 100644
--- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt
+++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt
@@ -29,7 +29,6 @@ import org.utbot.features.FeatureExtractorFactoryImpl
 import org.utbot.features.FeatureProcessorWithStatesRepetitionFactory
 import org.utbot.framework.PathSelectorType
 import org.utbot.framework.UtSettings
-import org.utbot.framework.plugin.api.util.UtContext
 import org.utbot.framework.plugin.api.util.id
 import org.utbot.framework.plugin.api.util.withUtContext
 import org.utbot.framework.plugin.services.JdkInfoService
@@ -47,7 +46,7 @@ private val javaHome = System.getenv("JAVA_HOME")
 private val javacCmd = "$javaHome/bin/javac"
 private val javaCmd = "$javaHome/bin/java"
 
-private const val triesToCompile = 2
+private const val compileAttempts = 2
 
 private data class UnnamedPackageInfo(val pack: String, val module: String)
 
@@ -64,7 +63,7 @@ private fun compileClass(testDir: String, classPath: String, testClass: String):
     val exports = mutableSetOf<UnnamedPackageInfo>()
     var exitCode = 0
 
-    repeat(triesToCompile) { tryNumber ->
+    repeat(compileAttempts) { attemptNumber ->
         val cmd = arrayOf(
             javacCmd,
             *exports.flatMap {
@@ -76,7 +75,7 @@ private fun compileClass(testDir: String, classPath: String, testClass: String):
             "-XDignore.symbol.file",
             testClass
         )
-        logger.debug { "Compile try ${tryNumber + 1}" }
+        logger.debug { "Compile attempt ${attemptNumber + 1}" }
 
         logger.trace { cmd.toText() }
 

From dfef0ea63860f1ff0a6613297309de5d74e3e619 Mon Sep 17 00:00:00 2001
From: Rustam Sadykov <snob-1432e@mail.ru>
Date: Wed, 9 Nov 2022 17:10:02 +0300
Subject: [PATCH 16/18] change metrics names

---
 monitoring/prepare_metrics.py | 43 ++++++++++++++++++++++++++++++++++-
 1 file changed, 42 insertions(+), 1 deletion(-)

diff --git a/monitoring/prepare_metrics.py b/monitoring/prepare_metrics.py
index c1f918b4e6..f09124f5fa 100644
--- a/monitoring/prepare_metrics.py
+++ b/monitoring/prepare_metrics.py
@@ -5,7 +5,47 @@
 from utils import load
 
 
+def remove_in_class(name: str) -> str:
+    in_class = "_in_class"
+    idx = name.find(in_class)
+    if idx == -1:
+        return name
+    return name[:idx] + name[idx:].removeprefix(in_class)
+
+
+def update_from_counter_name(key_word: str, name: str, labels: dict) -> str:
+    if name == f"total_{key_word}":
+        labels["type"] = "total"
+        return key_word
+    if name.startswith(key_word):
+        labels["type"] = name.removeprefix(f"{key_word}_")
+        return key_word
+    return name
+
+
+def update_from_coverage(name: str, labels: dict) -> str:
+    coverage_key = "bytecode_instruction_coverage"
+    idx = name.find(coverage_key)
+    if idx == -1:
+        return name
+    labels["type"] = name[:idx - 1]
+    source = name[idx:].removeprefix(f"{coverage_key}")
+    if len(source) > 0:
+        source = source.removeprefix("_by_")
+        if source == "classes":
+            labels["type"] = "averaged_by_classes"
+        else:
+            labels["source"] = source
+    return coverage_key
+
+
 def build_metric_struct(name: str, value: any, labels: dict) -> dict:
+    name = remove_in_class(name)
+    name = update_from_counter_name("classes", name, labels)
+    name = update_from_counter_name("methods", name, labels)
+    name = update_from_coverage(name, labels)
+
+    name = f"utbot_{name}"
     return {
         "metric": name,
         "labels": labels,
@@ -22,7 +62,7 @@ def build_metrics_from_data(data: dict, labels: dict) -> List[dict]:
     }
     metrics = data["metrics"]
     for metric in metrics:
-        result.append(build_metric_struct(metric, metrics[metric], new_labels))
+        result.append(build_metric_struct(metric, metrics[metric], new_labels.copy()))
     return result
 
 
@@ -86,6 +126,7 @@ def main():
     stats = load(args.stats_file)
     runner = stats["metadata"]["environment"]["host"]
     metrics = build_metrics_from_targets(stats["targets"], runner)
+    metrics.sort(key=lambda x: x["metric"])
     with open(args.output_file, "w") as f:
         json.dump(metrics, f, indent=4)
 

From 3636aaba278e2b7354b9defbb099811d9d21cb71 Mon Sep 17 00:00:00 2001
From: Rustam Sadykov <snob-1432e@mail.ru>
Date: Wed, 9 Nov 2022 17:19:15 +0300
Subject: [PATCH 17/18] fix metrics with bool values

---
 monitoring/prepare_metrics.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/monitoring/prepare_metrics.py b/monitoring/prepare_metrics.py
index f09124f5fa..a577f861d4 100644
--- a/monitoring/prepare_metrics.py
+++ b/monitoring/prepare_metrics.py
@@ -46,6 +46,10 @@ def build_metric_struct(name: str, value: any, labels: dict) -> dict:
     name = update_from_coverage(name, labels)
 
     name = f"utbot_{name}"
+
+    if type(value) == bool:
+        value = int(value)
+
     return {
         "metric": name,
         "labels": labels,

From 1f2d5d56398c53484ff234324b9015d521a88d1f Mon Sep 17 00:00:00 2001
From: Rustam Sadykov <snob-1432e@mail.ru>
Date: Wed, 9 Nov 2022 19:42:25 +0300
Subject: [PATCH 18/18] fix metrics names

---
 monitoring/prepare_metrics.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/monitoring/prepare_metrics.py b/monitoring/prepare_metrics.py
index a577f861d4..a5bffac2c4 100644
--- a/monitoring/prepare_metrics.py
+++ b/monitoring/prepare_metrics.py
@@ -45,10 +45,13 @@ def build_metric_struct(name: str, value: any, labels: dict) -> dict:
     name = update_from_counter_name("methods", name, labels)
     name = update_from_coverage(name, labels)
 
-    name = f"utbot_{name}"
-
     if type(value) == bool:
         value = int(value)
+        name = f"test_generation_{name}"
+    elif type(value) == int:
+        name = f"{name}_total"
+
+    name = f"utbot_{name}"
 
     return {
         "metric": name,