Add projects into monitoring (#1523)

Author: SBOne-Kenobi

.github/workflows/collect-statistics.yml

@@ -60,7 +60,7 @@ on:
 env:
   data_branch: monitoring-data
   data_path: monitoring/data
-  monitoring_properties: monitoring/monitoring.properties
+  monitoring_projects: monitoring/projects/
   push_script: monitoring/push_with_rebase.sh
   PUSHGATEWAY_HOSTNAME: monitoring.utbot.org
   PUSHGATEWAY_ADDITIONAL_PATH: /pushgateway-custom
@@ -70,11 +70,24 @@ jobs:
   setup_matrix:
     runs-on: ubuntu-latest
     outputs:
+      projects: ${{ steps.set-matrix.outputs.projects }}
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
       - name: Create matrix
         id: set-matrix
+        shell: bash
         run: |
+          read -r -a projects <<< $(ls --format=horizontal --indicator-style=none $monitoring_projects)
+          projects=(${projects[@]/#/\"})
+          projects=(${projects[@]/%/\"})
+          printf -v projects '%s,' "${projects[@]}"
+          projects=$(echo [${projects%,}])
+          echo "projects=$projects" >> $GITHUB_OUTPUT
+          echo $projects
+          
           arr=$(echo [$(seq -s , ${{ inputs.runners }})])
           echo "matrix=$arr" >> $GITHUB_OUTPUT
           echo $arr
@@ -83,7 +96,9 @@ jobs:
     needs: setup_matrix
     continue-on-error: true
     strategy:
+      max-parallel: 3
       matrix:
+        project: ${{ fromJson(needs.setup_matrix.outputs.projects) }}
         value: ${{ fromJson(needs.setup_matrix.outputs.matrix) }}
     runs-on: ubuntu-20.04
     container: unittestbot/java-env:java11-zulu-jdk-fx-gradle7.4.2-kotlinc1.7.0
@@ -114,7 +129,7 @@ jobs:
           for i in $(seq ${{ inputs.run_number }})
           do
             java -jar \
-              -Dutbot.monitoring.settings.path=$monitoring_properties \
+              -Dutbot.monitoring.settings.path=$monitoring_projects/${{ matrix.project }}/monitoring.properties \
               utbot-junit-contest/build/libs/monitoring.jar \
               stats-$i.json
             mv logs/utbot.log logs/utbot-$i.log
@@ -139,7 +154,7 @@ jobs:
         id: insert
         shell: bash
         run: |
-          OUT_FILE="$data_path/$date-$short_commit-${{ matrix.value }}.json"
+          OUT_FILE="$data_path/$date-$short_commit-${{ matrix.project }}-${{ matrix.value }}.json"
           echo "output=$OUT_FILE" >> $GITHUB_OUTPUT
 
           INPUTS=($(seq ${{ inputs.run_number }}))
@@ -169,7 +184,7 @@ jobs:
       - name: Upload statistics
         uses: actions/upload-artifact@v3
         with:
-          name: statistics-${{ matrix.value }}
+          name: statistics-${{ matrix.value }}-${{ matrix.project }}
           path: ${{ steps.insert.outputs.output }}
 
       - name: Commit and push statistics
@@ -188,7 +203,7 @@ jobs:
         run: |
           python monitoring/prepare_metrics.py --stats_file $stats_file --output_file grafana_metrics.json
           chmod +x scripts/project/json_to_prometheus.py
-          python3 scripts/project/json_to_prometheus.py grafana_metrics.json | curl -u "${{ secrets.PUSHGATEWAY_USER }}:${{ secrets.PUSHGATEWAY_PASSWORD }}" --data-binary @- "https://${PUSHGATEWAY_HOSTNAME}${PUSHGATEWAY_ADDITIONAL_PATH}/metrics/job/pushgateway-custom/instance/run-${{ matrix.value }}${PROM_ADDITIONAL_LABELS}"
+          python3 scripts/project/json_to_prometheus.py grafana_metrics.json | curl -u "${{ secrets.PUSHGATEWAY_USER }}:${{ secrets.PUSHGATEWAY_PASSWORD }}" --data-binary @- "https://${PUSHGATEWAY_HOSTNAME}${PUSHGATEWAY_ADDITIONAL_PATH}/metrics/job/pushgateway-custom/instance/run-${{ matrix.value }}-${{ matrix.project }}${PROM_ADDITIONAL_LABELS}"
           echo "Please visit Grafana to check metrics: https://monitoring.utbot.org/d/m6bagaD4z/utbot-nightly-statistic"
         env:
           stats_file: ${{ steps.insert.outputs.output }}
@@ -197,5 +212,13 @@ jobs:
         if: ${{ always() }}
         uses: actions/upload-artifact@v3
         with:
-          name: logs-${{ matrix.value }}
+          name: logs-${{ matrix.value }}-${{ matrix.project }}
           path: logs/
+
+      - name: Upload artifacts
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: generated-${{ matrix.value }}-${{ matrix.project }}
+          path: |
+            /tmp/UTBot/generated*/*

docs/NightStatisticsMonitoring.md

@@ -195,7 +195,7 @@ We store the collected statistics in our repository. You can find two special br
 
 The `monitoring-data` branch is a storage for raw statistics data as well as metadata.
 
-The filename format: `<yyyy>-<mm>-<dd>-<hh>-<MM>-<ss>-<branch>-<short commit hash>-<runner number>.json`
+The filename format: `<yyyy>-<mm>-<dd>-<hh>-<MM>-<ss>-<branch>-<short commit hash>-<project name>-<runner number>.json`
 
 ### Grafana
 

monitoring/MonitoringSettings.md

@@ -0,0 +1,24 @@
+# Monitoring Settings
+
+## Configuration files
+
+There are monitoring configuration files for each project in `project/<project name>/monitoring.properties`.
+
+### Monitoring configure
+The file `monitoring.properties` is passed as a java property `-Dutbot.monitoring.settings.path`. It configures `org.utbot.monitoring.MonitoringSettings` class.
+
+#### Properties description:
+- `project` is a name of project that will be run in monitoring.
+- `classTimeoutSeconds` is a unit-test generation timeout for one class.
+- `runTimeoutMinutes` is a timeout for one whole run of the project.
+- `fuzzingRatios` is a list of numbers that configure the ratio of fuzzing time to total generation time.
+
+## Which project can be run?
+
+### Prerequisites
+
+Firstly, you should read [this](../utbot-junit-contest/README.md) paper about available projects and how to extend them.
+
+### How to add projects to monitoring
+
+To add a project to monitoring you should create a folder with a project name and create a file `monitoring.properties` with needed configurations.

monitoring/projects/fescar/monitoring.properties

@@ -0,0 +1,4 @@
+project=fescar
+classTimeoutSeconds=20
+runTimeoutMinutes=20
+fuzzingRatios=0.0;0.05;1.0

monitoring/monitoring.properties renamed to monitoring/projects/guava/monitoring.properties

@@ -1,4 +1,4 @@
-projects=guava
+project=guava
 classTimeoutSeconds=20
 runTimeoutMinutes=20
-fuzzingRatios=0.0;0.1;1.0
+fuzzingRatios=0.0;0.05;1.0

monitoring/projects/pdfbox/monitoring.properties

@@ -0,0 +1,4 @@
+project=pdfbox
+classTimeoutSeconds=20
+runTimeoutMinutes=20
+fuzzingRatios=0.0;0.05;1.0

monitoring/projects/seata/monitoring.properties

@@ -0,0 +1,4 @@
+project=seata
+classTimeoutSeconds=20
+runTimeoutMinutes=20
+fuzzingRatios=0.0;0.05;1.0

monitoring/projects/spoon/monitoring.properties

@@ -0,0 +1,4 @@
+project=spoon
+classTimeoutSeconds=20
+runTimeoutMinutes=20
+fuzzingRatios=0.0;0.05;1.0

utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringReport.kt

@@ -116,6 +116,12 @@ data class SummarisedMetricsReport(
         covered_bytecode_instructions_by_fuzzing = targets.sumOf { it.metrics.covered_bytecode_instructions_in_class_by_fuzzing },
         covered_bytecode_instructions_by_concolic = targets.sumOf { it.metrics.covered_bytecode_instructions_in_class_by_concolic },
         total_bytecode_instructions = targets.sumOf { it.metrics.total_bytecode_instructions_in_class },
-        averaged_bytecode_instruction_coverage_by_classes = targets.map { it.metrics.bytecode_instructions_coverage_in_class }.average()
+        averaged_bytecode_instruction_coverage_by_classes = targets.map { it.metrics.bytecode_instructions_coverage_in_class }.average().fixNaN()
     )
 }
+
+private fun Double.fixNaN(): Double = if (isNaN() || isInfinite()) {
+    0.0
+} else {
+    this
+}

utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringSettings.kt

@@ -27,9 +27,9 @@ object MonitoringSettings : AbstractSettings(
     val runTimeoutMinutes by getIntProperty(180)
 
     /**
-     * Target projects to generate tests.
+     * The target project to generate tests.
      */
-    val projects by getListProperty(listOf("guava"))
+    val project by getStringProperty("guava")
 
     /**
      * Set up fuzzing timeout relatively total generation time.

utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/StatisticsMonitoring.kt

@@ -3,10 +3,6 @@ package org.utbot.monitoring
 import java.io.File
 import java.util.concurrent.TimeUnit
 import kotlin.system.exitProcess
-import kotlinx.coroutines.GlobalScope
-import kotlinx.coroutines.cancel
-import kotlinx.coroutines.cancelChildren
-import kotlinx.coroutines.job
 import kotlinx.serialization.encodeToString
 import kotlinx.serialization.json.Json
 import mu.KotlinLogging
@@ -19,7 +15,6 @@ import org.utbot.contest.GlobalStats
 import org.utbot.contest.Paths
 import org.utbot.contest.Tool
 import org.utbot.contest.runEstimator
-import org.utbot.framework.plugin.api.util.utContext
 import org.utbot.framework.plugin.services.JdkInfoService
 import org.utbot.instrumentation.ConcreteExecutor
 
@@ -35,7 +30,7 @@ fun main(args: Array<String>) {
     val processedClassesThreshold = MonitoringSettings.processedClassesThreshold
     val timeLimit = MonitoringSettings.classTimeoutSeconds
 
-    val projects = MonitoringSettings.projects
+    val projects = listOf(MonitoringSettings.project)
     methodFilter = null
 
     val runTimeout = TimeUnit.MINUTES.toMillis(MonitoringSettings.runTimeoutMinutes.toLong())

utbot-junit-contest/src/main/resources/log4j2.xml

@@ -24,7 +24,7 @@
 
 
         <Logger name="org.utbot.engine.UtBotSymbolicEngine" level="debug"/>
-        <Logger name="org.utbot.instrumentation" level="trace"/>
+        <Logger name="org.utbot.instrumentation" level="debug"/>
 
         <Logger name="org.utbot.engine.selectors.strategies.GraphViz" level="debug">
             <AppenderRef ref="Console"/>