diff --git a/.github/workflows/collect-statistics.yml b/.github/workflows/collect-statistics.yml index 6924cf9e20..df574fcc63 100644 --- a/.github/workflows/collect-statistics.yml +++ b/.github/workflows/collect-statistics.yml @@ -60,7 +60,7 @@ on: env: data_branch: monitoring-data data_path: monitoring/data - monitoring_properties: monitoring/monitoring.properties + monitoring_projects: monitoring/projects/ push_script: monitoring/push_with_rebase.sh PUSHGATEWAY_HOSTNAME: monitoring.utbot.org PUSHGATEWAY_ADDITIONAL_PATH: /pushgateway-custom @@ -70,11 +70,24 @@ jobs: setup_matrix: runs-on: ubuntu-latest outputs: + projects: ${{ steps.set-matrix.outputs.projects }} matrix: ${{ steps.set-matrix.outputs.matrix }} steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Create matrix id: set-matrix + shell: bash run: | + read -r -a projects <<< $(ls --format=horizontal --indicator-style=none $monitoring_projects) + projects=(${projects[@]/#/\"}) + projects=(${projects[@]/%/\"}) + printf -v projects '%s,' "${projects[@]}" + projects=$(echo [${projects%,}]) + echo "projects=$projects" >> $GITHUB_OUTPUT + echo $projects + arr=$(echo [$(seq -s , ${{ inputs.runners }})]) echo "matrix=$arr" >> $GITHUB_OUTPUT echo $arr @@ -83,7 +96,9 @@ jobs: needs: setup_matrix continue-on-error: true strategy: + max-parallel: 3 matrix: + project: ${{ fromJson(needs.setup_matrix.outputs.projects) }} value: ${{ fromJson(needs.setup_matrix.outputs.matrix) }} runs-on: ubuntu-20.04 container: unittestbot/java-env:java11-zulu-jdk-fx-gradle7.4.2-kotlinc1.7.0 @@ -114,7 +129,7 @@ jobs: for i in $(seq ${{ inputs.run_number }}) do java -jar \ - -Dutbot.monitoring.settings.path=$monitoring_properties \ + -Dutbot.monitoring.settings.path=$monitoring_projects/${{ matrix.project }}/monitoring.properties \ utbot-junit-contest/build/libs/monitoring.jar \ stats-$i.json mv logs/utbot.log logs/utbot-$i.log @@ -139,7 +154,7 @@ jobs: id: insert shell: bash run: | - OUT_FILE="$data_path/$date-$short_commit-${{ matrix.value }}.json" + OUT_FILE="$data_path/$date-$short_commit-${{ matrix.project }}-${{ matrix.value }}.json" echo "output=$OUT_FILE" >> $GITHUB_OUTPUT INPUTS=($(seq ${{ inputs.run_number }})) @@ -169,7 +184,7 @@ jobs: - name: Upload statistics uses: actions/upload-artifact@v3 with: - name: statistics-${{ matrix.value }} + name: statistics-${{ matrix.value }}-${{ matrix.project }} path: ${{ steps.insert.outputs.output }} - name: Commit and push statistics @@ -188,7 +203,7 @@ jobs: run: | python monitoring/prepare_metrics.py --stats_file $stats_file --output_file grafana_metrics.json chmod +x scripts/project/json_to_prometheus.py - python3 scripts/project/json_to_prometheus.py grafana_metrics.json | curl -u "${{ secrets.PUSHGATEWAY_USER }}:${{ secrets.PUSHGATEWAY_PASSWORD }}" --data-binary @- "https://${PUSHGATEWAY_HOSTNAME}${PUSHGATEWAY_ADDITIONAL_PATH}/metrics/job/pushgateway-custom/instance/run-${{ matrix.value }}${PROM_ADDITIONAL_LABELS}" + python3 scripts/project/json_to_prometheus.py grafana_metrics.json | curl -u "${{ secrets.PUSHGATEWAY_USER }}:${{ secrets.PUSHGATEWAY_PASSWORD }}" --data-binary @- "https://${PUSHGATEWAY_HOSTNAME}${PUSHGATEWAY_ADDITIONAL_PATH}/metrics/job/pushgateway-custom/instance/run-${{ matrix.value }}-${{ matrix.project }}${PROM_ADDITIONAL_LABELS}" echo "Please visit Grafana to check metrics: https://monitoring.utbot.org/d/m6bagaD4z/utbot-nightly-statistic" env: stats_file: ${{ steps.insert.outputs.output }} @@ -197,5 +212,13 @@ jobs: if: ${{ always() }} uses: actions/upload-artifact@v3 with: - name: logs-${{ matrix.value }} + name: logs-${{ matrix.value }}-${{ matrix.project }} path: logs/ + + - name: Upload artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: generated-${{ matrix.value }}-${{ matrix.project }} + path: | + /tmp/UTBot/generated*/* diff --git a/docs/NightStatisticsMonitoring.md b/docs/NightStatisticsMonitoring.md index a0a115a847..226356b97f 100644 --- a/docs/NightStatisticsMonitoring.md +++ b/docs/NightStatisticsMonitoring.md @@ -195,7 +195,7 @@ We store the collected statistics in our repository. You can find two special br The `monitoring-data` branch is a storage for raw statistics data as well as metadata. -The filename format: `--
------.json` +The filename format: `--
-------.json` ### Grafana diff --git a/monitoring/MonitoringSettings.md b/monitoring/MonitoringSettings.md new file mode 100644 index 0000000000..0a9c05e37e --- /dev/null +++ b/monitoring/MonitoringSettings.md @@ -0,0 +1,24 @@ +# Monitoring Settings + +## Configuration files + +There are monitoring configuration files for each project in `project//monitoring.properties`. + +### Monitoring configure +The file `monitoring.properties` is passed as a java property `-Dutbot.monitoring.settings.path`. It configures `org.utbot.monitoring.MonitoringSettings` class. + +#### Properties description: +- `project` is a name of project that will be run in monitoring. +- `classTimeoutSeconds` is a unit-test generation timeout for one class. +- `runTimeoutMinutes` is a timeout for one whole run of the project. +- `fuzzingRatios` is a list of numbers that configure the ratio of fuzzing time to total generation time. + +## Which project can be run? + +### Prerequisites + +Firstly, you should read [this](../utbot-junit-contest/README.md) paper about available projects and how to extend them. + +### How to add projects to monitoring + +To add a project to monitoring you should create a folder with a project name and create a file `monitoring.properties` with needed configurations. diff --git a/monitoring/projects/fescar/monitoring.properties b/monitoring/projects/fescar/monitoring.properties new file mode 100644 index 0000000000..cabd34e51e --- /dev/null +++ b/monitoring/projects/fescar/monitoring.properties @@ -0,0 +1,4 @@ +project=fescar +classTimeoutSeconds=20 +runTimeoutMinutes=20 +fuzzingRatios=0.0;0.05;1.0 \ No newline at end of file diff --git a/monitoring/monitoring.properties b/monitoring/projects/guava/monitoring.properties similarity index 52% rename from monitoring/monitoring.properties rename to monitoring/projects/guava/monitoring.properties index 0a1694eb88..0555a8c44a 100644 --- a/monitoring/monitoring.properties +++ b/monitoring/projects/guava/monitoring.properties @@ -1,4 +1,4 @@ -projects=guava +project=guava classTimeoutSeconds=20 runTimeoutMinutes=20 -fuzzingRatios=0.0;0.1;1.0 \ No newline at end of file +fuzzingRatios=0.0;0.05;1.0 \ No newline at end of file diff --git a/monitoring/projects/pdfbox/monitoring.properties b/monitoring/projects/pdfbox/monitoring.properties new file mode 100644 index 0000000000..3878e706fe --- /dev/null +++ b/monitoring/projects/pdfbox/monitoring.properties @@ -0,0 +1,4 @@ +project=pdfbox +classTimeoutSeconds=20 +runTimeoutMinutes=20 +fuzzingRatios=0.0;0.05;1.0 \ No newline at end of file diff --git a/monitoring/projects/seata/monitoring.properties b/monitoring/projects/seata/monitoring.properties new file mode 100644 index 0000000000..9cb7421c9a --- /dev/null +++ b/monitoring/projects/seata/monitoring.properties @@ -0,0 +1,4 @@ +project=seata +classTimeoutSeconds=20 +runTimeoutMinutes=20 +fuzzingRatios=0.0;0.05;1.0 \ No newline at end of file diff --git a/monitoring/projects/spoon/monitoring.properties b/monitoring/projects/spoon/monitoring.properties new file mode 100644 index 0000000000..41b56c94f0 --- /dev/null +++ b/monitoring/projects/spoon/monitoring.properties @@ -0,0 +1,4 @@ +project=spoon +classTimeoutSeconds=20 +runTimeoutMinutes=20 +fuzzingRatios=0.0;0.05;1.0 \ No newline at end of file diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringReport.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringReport.kt index 9ab736879a..9bbbc07e82 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringReport.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringReport.kt @@ -116,6 +116,12 @@ data class SummarisedMetricsReport( covered_bytecode_instructions_by_fuzzing = targets.sumOf { it.metrics.covered_bytecode_instructions_in_class_by_fuzzing }, covered_bytecode_instructions_by_concolic = targets.sumOf { it.metrics.covered_bytecode_instructions_in_class_by_concolic }, total_bytecode_instructions = targets.sumOf { it.metrics.total_bytecode_instructions_in_class }, - averaged_bytecode_instruction_coverage_by_classes = targets.map { it.metrics.bytecode_instructions_coverage_in_class }.average() + averaged_bytecode_instruction_coverage_by_classes = targets.map { it.metrics.bytecode_instructions_coverage_in_class }.average().fixNaN() ) } + +private fun Double.fixNaN(): Double = if (isNaN() || isInfinite()) { + 0.0 +} else { + this +} diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringSettings.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringSettings.kt index a9aa051241..e6ca9dca8a 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringSettings.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringSettings.kt @@ -27,9 +27,9 @@ object MonitoringSettings : AbstractSettings( val runTimeoutMinutes by getIntProperty(180) /** - * Target projects to generate tests. + * The target project to generate tests. */ - val projects by getListProperty(listOf("guava")) + val project by getStringProperty("guava") /** * Set up fuzzing timeout relatively total generation time. diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/StatisticsMonitoring.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/StatisticsMonitoring.kt index 4b68efc398..ab4c1ff718 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/StatisticsMonitoring.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/StatisticsMonitoring.kt @@ -3,10 +3,6 @@ package org.utbot.monitoring import java.io.File import java.util.concurrent.TimeUnit import kotlin.system.exitProcess -import kotlinx.coroutines.GlobalScope -import kotlinx.coroutines.cancel -import kotlinx.coroutines.cancelChildren -import kotlinx.coroutines.job import kotlinx.serialization.encodeToString import kotlinx.serialization.json.Json import mu.KotlinLogging @@ -19,7 +15,6 @@ import org.utbot.contest.GlobalStats import org.utbot.contest.Paths import org.utbot.contest.Tool import org.utbot.contest.runEstimator -import org.utbot.framework.plugin.api.util.utContext import org.utbot.framework.plugin.services.JdkInfoService import org.utbot.instrumentation.ConcreteExecutor @@ -35,7 +30,7 @@ fun main(args: Array) { val processedClassesThreshold = MonitoringSettings.processedClassesThreshold val timeLimit = MonitoringSettings.classTimeoutSeconds - val projects = MonitoringSettings.projects + val projects = listOf(MonitoringSettings.project) methodFilter = null val runTimeout = TimeUnit.MINUTES.toMillis(MonitoringSettings.runTimeoutMinutes.toLong()) diff --git a/utbot-junit-contest/src/main/resources/log4j2.xml b/utbot-junit-contest/src/main/resources/log4j2.xml index 8cb742f1ff..fd4a216229 100644 --- a/utbot-junit-contest/src/main/resources/log4j2.xml +++ b/utbot-junit-contest/src/main/resources/log4j2.xml @@ -24,7 +24,7 @@ - +