From be046b1dd53ff4e0cd7cf070812d047a0fddf56d Mon Sep 17 00:00:00 2001 From: Rustam Sadykov Date: Mon, 12 Dec 2022 09:59:11 +0300 Subject: [PATCH 1/7] increase timeouts --- monitoring/monitoring.properties | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/monitoring/monitoring.properties b/monitoring/monitoring.properties index 0a1694eb88..c353a312bd 100644 --- a/monitoring/monitoring.properties +++ b/monitoring/monitoring.properties @@ -1,4 +1,4 @@ projects=guava -classTimeoutSeconds=20 -runTimeoutMinutes=20 -fuzzingRatios=0.0;0.1;1.0 \ No newline at end of file +classTimeoutSeconds=30 +runTimeoutMinutes=30 +fuzzingRatios=0.0;0.05;1.0 \ No newline at end of file From 2268f804d29114754bae2b3fd03a55a4f07b26e9 Mon Sep 17 00:00:00 2001 From: Rustam Sadykov Date: Mon, 12 Dec 2022 12:56:48 +0300 Subject: [PATCH 2/7] add project detection --- .github/workflows/collect-statistics.yml | 19 ++++++++++++++----- .../guava}/monitoring.properties | 0 .../utbot/monitoring/MonitoringSettings.kt | 4 ++-- .../utbot/monitoring/StatisticsMonitoring.kt | 7 +------ 4 files changed, 17 insertions(+), 13 deletions(-) rename monitoring/{ => projects/guava}/monitoring.properties (100%) diff --git a/.github/workflows/collect-statistics.yml b/.github/workflows/collect-statistics.yml index 6924cf9e20..87095a5c7b 100644 --- a/.github/workflows/collect-statistics.yml +++ b/.github/workflows/collect-statistics.yml @@ -60,7 +60,7 @@ on: env: data_branch: monitoring-data data_path: monitoring/data - monitoring_properties: monitoring/monitoring.properties + monitoring_projects: monitoring/projects/ push_script: monitoring/push_with_rebase.sh PUSHGATEWAY_HOSTNAME: monitoring.utbot.org PUSHGATEWAY_ADDITIONAL_PATH: /pushgateway-custom @@ -70,11 +70,19 @@ jobs: setup_matrix: runs-on: ubuntu-latest outputs: + projects: ${{ steps.set-matrix.outputs.projects }} matrix: ${{ steps.set-matrix.outputs.matrix }} steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Create matrix id: set-matrix run: | + projects=$(echo [$(ls --format=commas --indicator-style=none $monitoring_projects)]) + echo "projects=$projects" >> $GITHUB_OUTPUT + echo $projects + arr=$(echo [$(seq -s , ${{ inputs.runners }})]) echo "matrix=$arr" >> $GITHUB_OUTPUT echo $arr @@ -85,6 +93,7 @@ jobs: strategy: matrix: value: ${{ fromJson(needs.setup_matrix.outputs.matrix) }} + project: ${{ fromJson(needs.setup_matrix.outputs.projects) }} runs-on: ubuntu-20.04 container: unittestbot/java-env:java11-zulu-jdk-fx-gradle7.4.2-kotlinc1.7.0 steps: @@ -114,7 +123,7 @@ jobs: for i in $(seq ${{ inputs.run_number }}) do java -jar \ - -Dutbot.monitoring.settings.path=$monitoring_properties \ + -Dutbot.monitoring.settings.path=$monitoring_projects/${{ matrix.project }}/monitoring.properties \ utbot-junit-contest/build/libs/monitoring.jar \ stats-$i.json mv logs/utbot.log logs/utbot-$i.log @@ -169,7 +178,7 @@ jobs: - name: Upload statistics uses: actions/upload-artifact@v3 with: - name: statistics-${{ matrix.value }} + name: statistics-${{ matrix.value }}-${{ matrix.project }} path: ${{ steps.insert.outputs.output }} - name: Commit and push statistics @@ -188,7 +197,7 @@ jobs: run: | python monitoring/prepare_metrics.py --stats_file $stats_file --output_file grafana_metrics.json chmod +x scripts/project/json_to_prometheus.py - python3 scripts/project/json_to_prometheus.py grafana_metrics.json | curl -u "${{ secrets.PUSHGATEWAY_USER }}:${{ secrets.PUSHGATEWAY_PASSWORD }}" --data-binary @- "https://${PUSHGATEWAY_HOSTNAME}${PUSHGATEWAY_ADDITIONAL_PATH}/metrics/job/pushgateway-custom/instance/run-${{ matrix.value }}${PROM_ADDITIONAL_LABELS}" + python3 scripts/project/json_to_prometheus.py grafana_metrics.json | curl -u "${{ secrets.PUSHGATEWAY_USER }}:${{ secrets.PUSHGATEWAY_PASSWORD }}" --data-binary @- "https://${PUSHGATEWAY_HOSTNAME}${PUSHGATEWAY_ADDITIONAL_PATH}/metrics/job/pushgateway-custom/instance/run-${{ matrix.value }}-${{ matrix.project }}${PROM_ADDITIONAL_LABELS}" echo "Please visit Grafana to check metrics: https://monitoring.utbot.org/d/m6bagaD4z/utbot-nightly-statistic" env: stats_file: ${{ steps.insert.outputs.output }} @@ -197,5 +206,5 @@ jobs: if: ${{ always() }} uses: actions/upload-artifact@v3 with: - name: logs-${{ matrix.value }} + name: logs-${{ matrix.value }}-${{ matrix.project }} path: logs/ diff --git a/monitoring/monitoring.properties b/monitoring/projects/guava/monitoring.properties similarity index 100% rename from monitoring/monitoring.properties rename to monitoring/projects/guava/monitoring.properties diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringSettings.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringSettings.kt index a9aa051241..e6ca9dca8a 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringSettings.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringSettings.kt @@ -27,9 +27,9 @@ object MonitoringSettings : AbstractSettings( val runTimeoutMinutes by getIntProperty(180) /** - * Target projects to generate tests. + * The target project to generate tests. */ - val projects by getListProperty(listOf("guava")) + val project by getStringProperty("guava") /** * Set up fuzzing timeout relatively total generation time. diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/StatisticsMonitoring.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/StatisticsMonitoring.kt index 4b68efc398..ab4c1ff718 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/StatisticsMonitoring.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/StatisticsMonitoring.kt @@ -3,10 +3,6 @@ package org.utbot.monitoring import java.io.File import java.util.concurrent.TimeUnit import kotlin.system.exitProcess -import kotlinx.coroutines.GlobalScope -import kotlinx.coroutines.cancel -import kotlinx.coroutines.cancelChildren -import kotlinx.coroutines.job import kotlinx.serialization.encodeToString import kotlinx.serialization.json.Json import mu.KotlinLogging @@ -19,7 +15,6 @@ import org.utbot.contest.GlobalStats import org.utbot.contest.Paths import org.utbot.contest.Tool import org.utbot.contest.runEstimator -import org.utbot.framework.plugin.api.util.utContext import org.utbot.framework.plugin.services.JdkInfoService import org.utbot.instrumentation.ConcreteExecutor @@ -35,7 +30,7 @@ fun main(args: Array) { val processedClassesThreshold = MonitoringSettings.processedClassesThreshold val timeLimit = MonitoringSettings.classTimeoutSeconds - val projects = MonitoringSettings.projects + val projects = listOf(MonitoringSettings.project) methodFilter = null val runTimeout = TimeUnit.MINUTES.toMillis(MonitoringSettings.runTimeoutMinutes.toLong()) From aa2a3d1cbd638267b49e586df89f0e36823ac5ba Mon Sep 17 00:00:00 2001 From: Rustam Sadykov Date: Mon, 12 Dec 2022 12:59:43 +0300 Subject: [PATCH 3/7] add new project --- monitoring/projects/spoon/monitoring.properties | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 monitoring/projects/spoon/monitoring.properties diff --git a/monitoring/projects/spoon/monitoring.properties b/monitoring/projects/spoon/monitoring.properties new file mode 100644 index 0000000000..977f764797 --- /dev/null +++ b/monitoring/projects/spoon/monitoring.properties @@ -0,0 +1,4 @@ +projects=spoon +classTimeoutSeconds=30 +runTimeoutMinutes=30 +fuzzingRatios=0.0;0.05;1.0 \ No newline at end of file From 41f7398d6f2056ca5164c6f4ab590fae53ad7cb6 Mon Sep 17 00:00:00 2001 From: Rustam Sadykov Date: Mon, 12 Dec 2022 13:29:50 +0300 Subject: [PATCH 4/7] fix workflow --- .github/workflows/collect-statistics.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/collect-statistics.yml b/.github/workflows/collect-statistics.yml index 87095a5c7b..9a151e5612 100644 --- a/.github/workflows/collect-statistics.yml +++ b/.github/workflows/collect-statistics.yml @@ -78,8 +78,13 @@ jobs: - name: Create matrix id: set-matrix + shell: bash run: | - projects=$(echo [$(ls --format=commas --indicator-style=none $monitoring_projects)]) + read -r -a projects <<< $(ls --format=horizontal --indicator-style=none $monitoring_projects) + projects=(${projects[@]/#/\"}) + projects=(${projects[@]/%/\"}) + printf -v projects '%s,' "${projects[@]}" + projects=$(echo [${projects%,}]) echo "projects=$projects" >> $GITHUB_OUTPUT echo $projects From 5de077be4d0f08c8b420d4c88dae02390c432607 Mon Sep 17 00:00:00 2001 From: Rustam Sadykov <54807972+SBOne-Kenobi@users.noreply.github.com> Date: Tue, 13 Dec 2022 11:01:00 +0300 Subject: [PATCH 5/7] Add max parallel restriction --- .github/workflows/collect-statistics.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/collect-statistics.yml b/.github/workflows/collect-statistics.yml index 9a151e5612..b3691e1cee 100644 --- a/.github/workflows/collect-statistics.yml +++ b/.github/workflows/collect-statistics.yml @@ -96,6 +96,7 @@ jobs: needs: setup_matrix continue-on-error: true strategy: + max-parallel: 3 matrix: value: ${{ fromJson(needs.setup_matrix.outputs.matrix) }} project: ${{ fromJson(needs.setup_matrix.outputs.projects) }} From 59e4dde9305314d2ef83913adced260a4e07bc47 Mon Sep 17 00:00:00 2001 From: Rustam Sadykov Date: Thu, 15 Dec 2022 09:50:53 +0300 Subject: [PATCH 6/7] add projects --- .github/workflows/collect-statistics.yml | 12 ++++++++++-- monitoring/projects/fescar/monitoring.properties | 4 ++++ monitoring/projects/guava/monitoring.properties | 6 +++--- monitoring/projects/pdfbox/monitoring.properties | 4 ++++ monitoring/projects/seata/monitoring.properties | 4 ++++ monitoring/projects/spoon/monitoring.properties | 6 +++--- .../kotlin/org/utbot/monitoring/MonitoringReport.kt | 8 +++++++- utbot-junit-contest/src/main/resources/log4j2.xml | 2 +- 8 files changed, 36 insertions(+), 10 deletions(-) create mode 100644 monitoring/projects/fescar/monitoring.properties create mode 100644 monitoring/projects/pdfbox/monitoring.properties create mode 100644 monitoring/projects/seata/monitoring.properties diff --git a/.github/workflows/collect-statistics.yml b/.github/workflows/collect-statistics.yml index b3691e1cee..df574fcc63 100644 --- a/.github/workflows/collect-statistics.yml +++ b/.github/workflows/collect-statistics.yml @@ -98,8 +98,8 @@ jobs: strategy: max-parallel: 3 matrix: - value: ${{ fromJson(needs.setup_matrix.outputs.matrix) }} project: ${{ fromJson(needs.setup_matrix.outputs.projects) }} + value: ${{ fromJson(needs.setup_matrix.outputs.matrix) }} runs-on: ubuntu-20.04 container: unittestbot/java-env:java11-zulu-jdk-fx-gradle7.4.2-kotlinc1.7.0 steps: @@ -154,7 +154,7 @@ jobs: id: insert shell: bash run: | - OUT_FILE="$data_path/$date-$short_commit-${{ matrix.value }}.json" + OUT_FILE="$data_path/$date-$short_commit-${{ matrix.project }}-${{ matrix.value }}.json" echo "output=$OUT_FILE" >> $GITHUB_OUTPUT INPUTS=($(seq ${{ inputs.run_number }})) @@ -214,3 +214,11 @@ jobs: with: name: logs-${{ matrix.value }}-${{ matrix.project }} path: logs/ + + - name: Upload artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: generated-${{ matrix.value }}-${{ matrix.project }} + path: | + /tmp/UTBot/generated*/* diff --git a/monitoring/projects/fescar/monitoring.properties b/monitoring/projects/fescar/monitoring.properties new file mode 100644 index 0000000000..cabd34e51e --- /dev/null +++ b/monitoring/projects/fescar/monitoring.properties @@ -0,0 +1,4 @@ +project=fescar +classTimeoutSeconds=20 +runTimeoutMinutes=20 +fuzzingRatios=0.0;0.05;1.0 \ No newline at end of file diff --git a/monitoring/projects/guava/monitoring.properties b/monitoring/projects/guava/monitoring.properties index c353a312bd..0555a8c44a 100644 --- a/monitoring/projects/guava/monitoring.properties +++ b/monitoring/projects/guava/monitoring.properties @@ -1,4 +1,4 @@ -projects=guava -classTimeoutSeconds=30 -runTimeoutMinutes=30 +project=guava +classTimeoutSeconds=20 +runTimeoutMinutes=20 fuzzingRatios=0.0;0.05;1.0 \ No newline at end of file diff --git a/monitoring/projects/pdfbox/monitoring.properties b/monitoring/projects/pdfbox/monitoring.properties new file mode 100644 index 0000000000..3878e706fe --- /dev/null +++ b/monitoring/projects/pdfbox/monitoring.properties @@ -0,0 +1,4 @@ +project=pdfbox +classTimeoutSeconds=20 +runTimeoutMinutes=20 +fuzzingRatios=0.0;0.05;1.0 \ No newline at end of file diff --git a/monitoring/projects/seata/monitoring.properties b/monitoring/projects/seata/monitoring.properties new file mode 100644 index 0000000000..9cb7421c9a --- /dev/null +++ b/monitoring/projects/seata/monitoring.properties @@ -0,0 +1,4 @@ +project=seata +classTimeoutSeconds=20 +runTimeoutMinutes=20 +fuzzingRatios=0.0;0.05;1.0 \ No newline at end of file diff --git a/monitoring/projects/spoon/monitoring.properties b/monitoring/projects/spoon/monitoring.properties index 977f764797..41b56c94f0 100644 --- a/monitoring/projects/spoon/monitoring.properties +++ b/monitoring/projects/spoon/monitoring.properties @@ -1,4 +1,4 @@ -projects=spoon -classTimeoutSeconds=30 -runTimeoutMinutes=30 +project=spoon +classTimeoutSeconds=20 +runTimeoutMinutes=20 fuzzingRatios=0.0;0.05;1.0 \ No newline at end of file diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringReport.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringReport.kt index 9ab736879a..9bbbc07e82 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringReport.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/monitoring/MonitoringReport.kt @@ -116,6 +116,12 @@ data class SummarisedMetricsReport( covered_bytecode_instructions_by_fuzzing = targets.sumOf { it.metrics.covered_bytecode_instructions_in_class_by_fuzzing }, covered_bytecode_instructions_by_concolic = targets.sumOf { it.metrics.covered_bytecode_instructions_in_class_by_concolic }, total_bytecode_instructions = targets.sumOf { it.metrics.total_bytecode_instructions_in_class }, - averaged_bytecode_instruction_coverage_by_classes = targets.map { it.metrics.bytecode_instructions_coverage_in_class }.average() + averaged_bytecode_instruction_coverage_by_classes = targets.map { it.metrics.bytecode_instructions_coverage_in_class }.average().fixNaN() ) } + +private fun Double.fixNaN(): Double = if (isNaN() || isInfinite()) { + 0.0 +} else { + this +} diff --git a/utbot-junit-contest/src/main/resources/log4j2.xml b/utbot-junit-contest/src/main/resources/log4j2.xml index 8cb742f1ff..fd4a216229 100644 --- a/utbot-junit-contest/src/main/resources/log4j2.xml +++ b/utbot-junit-contest/src/main/resources/log4j2.xml @@ -24,7 +24,7 @@ - + From 7b084160b2d7e9ca7d4ef5e75f10eea72fde57b5 Mon Sep 17 00:00:00 2001 From: Rustam Sadykov Date: Thu, 15 Dec 2022 10:04:40 +0300 Subject: [PATCH 7/7] add docs --- docs/NightStatisticsMonitoring.md | 2 +- monitoring/MonitoringSettings.md | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 monitoring/MonitoringSettings.md diff --git a/docs/NightStatisticsMonitoring.md b/docs/NightStatisticsMonitoring.md index a0a115a847..226356b97f 100644 --- a/docs/NightStatisticsMonitoring.md +++ b/docs/NightStatisticsMonitoring.md @@ -195,7 +195,7 @@ We store the collected statistics in our repository. You can find two special br The `monitoring-data` branch is a storage for raw statistics data as well as metadata. -The filename format: `--
------.json` +The filename format: `--
-------.json` ### Grafana diff --git a/monitoring/MonitoringSettings.md b/monitoring/MonitoringSettings.md new file mode 100644 index 0000000000..0a9c05e37e --- /dev/null +++ b/monitoring/MonitoringSettings.md @@ -0,0 +1,24 @@ +# Monitoring Settings + +## Configuration files + +There are monitoring configuration files for each project in `project//monitoring.properties`. + +### Monitoring configure +The file `monitoring.properties` is passed as a java property `-Dutbot.monitoring.settings.path`. It configures `org.utbot.monitoring.MonitoringSettings` class. + +#### Properties description: +- `project` is a name of project that will be run in monitoring. +- `classTimeoutSeconds` is a unit-test generation timeout for one class. +- `runTimeoutMinutes` is a timeout for one whole run of the project. +- `fuzzingRatios` is a list of numbers that configure the ratio of fuzzing time to total generation time. + +## Which project can be run? + +### Prerequisites + +Firstly, you should read [this](../utbot-junit-contest/README.md) paper about available projects and how to extend them. + +### How to add projects to monitoring + +To add a project to monitoring you should create a folder with a project name and create a file `monitoring.properties` with needed configurations.