From 1692824ed95de78d06c531b05e97e428dba369f8 Mon Sep 17 00:00:00 2001 From: Egor Vasilyev Date: Fri, 11 Nov 2022 17:17:44 +0300 Subject: [PATCH 1/8] send nightly builds statistics to monitoring #1335 --- .github/workflows/collect-statistics.yml | 6 +++- scripts/project/json_to_prometheus.py | 36 ++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 scripts/project/json_to_prometheus.py diff --git a/.github/workflows/collect-statistics.yml b/.github/workflows/collect-statistics.yml index f916f9f49c..72cea652f1 100644 --- a/.github/workflows/collect-statistics.yml +++ b/.github/workflows/collect-statistics.yml @@ -62,6 +62,9 @@ env: data_path: monitoring/data monitoring_properties: monitoring/monitoring.properties push_script: monitoring/push_with_rebase.sh + PUSHGATEWAY_HOSTNAME: monitoring.utbot.org + PUSHGATEWAY_ADDITIONAL_PATH: /pushgateway-custom + PROM_ADDITIONAL_LABELS: /service/github jobs: setup_matrix: @@ -184,7 +187,8 @@ jobs: if: ${{ inputs.send_to_grafana }} run: | python monitoring/prepare_metrics.py --stats_file $stats_file --output_file grafana_metrics.json - echo "TODO send metrics to grafana" + chmod +x scripts/project/json_to_prometheus.py + python3 scripts/project/json_to_prometheus.py grafana_metrics.json | curl -u "${{ secrets.PUSHGATEWAY_USER }}:${{ secrets.PUSHGATEWAY_PASSWORD }}" --data-binary @- https://${PUSHGATEWAY_HOSTNAME}${PUSHGATEWAY_ADDITIONAL_PATH}/metrics/job/pushgateway-custom/instance/nightly-statistics${PROM_ADDITIONAL_LABELS} env: stats_file: ${{ steps.insert.outputs.output }} diff --git a/scripts/project/json_to_prometheus.py b/scripts/project/json_to_prometheus.py new file mode 100644 index 0000000000..0a00270f9b --- /dev/null +++ b/scripts/project/json_to_prometheus.py @@ -0,0 +1,36 @@ +import sys +import json + +with open(sys.argv[1]) as metrics_raw: + metrics_json = json.load(metrics_raw) + +# metrics is a json list e.g.: +# [ +# { +# "metric": "total_classes", +# "labels": { +# "project": "guava", +# "fuzzing_ratio": 0.1 +# }, +# "value": 20 +# }, +# { +# "metric": "testcases_generated", +# "labels": { +# "project": "guava", +# "fuzzing_ratio": 0.1 +# }, +# "value": 1042 +# } +# ] +# +# the loop below iterates over each list item and constructs metrics set +metrics_set_str = "" +for metric in metrics_json: + labels_set_str = "" + comma = "" + for label, value in metric['labels'].items(): + labels_set_str = f'{labels_set_str}{comma}{label}=\"{value}\"' + comma = "," + metrics_set_str += f'{metric["metric"]}{{{labels_set_str}}} {metric["value"]}\n' +print(metrics_set_str) From d15c4d453ad1e5a23b3f686f764889b512cf901b Mon Sep 17 00:00:00 2001 From: Egor Vasilyev Date: Mon, 14 Nov 2022 11:41:12 +0300 Subject: [PATCH 2/8] grafana url added --- .github/workflows/collect-statistics.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/collect-statistics.yml b/.github/workflows/collect-statistics.yml index 72cea652f1..a07275b30f 100644 --- a/.github/workflows/collect-statistics.yml +++ b/.github/workflows/collect-statistics.yml @@ -189,6 +189,7 @@ jobs: python monitoring/prepare_metrics.py --stats_file $stats_file --output_file grafana_metrics.json chmod +x scripts/project/json_to_prometheus.py python3 scripts/project/json_to_prometheus.py grafana_metrics.json | curl -u "${{ secrets.PUSHGATEWAY_USER }}:${{ secrets.PUSHGATEWAY_PASSWORD }}" --data-binary @- https://${PUSHGATEWAY_HOSTNAME}${PUSHGATEWAY_ADDITIONAL_PATH}/metrics/job/pushgateway-custom/instance/nightly-statistics${PROM_ADDITIONAL_LABELS} + echo "Please visit Grafana to check metrics: https://monitoring.utbot.org/d/m6bagaD4z/utbot-nightly-statistic" env: stats_file: ${{ steps.insert.outputs.output }} From 3d2f2286ccce9ac78ade3eaa562330ecf91e9943 Mon Sep 17 00:00:00 2001 From: Egor Vasilyev Date: Mon, 14 Nov 2022 11:48:15 +0300 Subject: [PATCH 3/8] PUSHGATEWAY_HOSTNAME secret replaced with variable --- .github/workflows/run-chosen-tests-from-branch.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-chosen-tests-from-branch.yml b/.github/workflows/run-chosen-tests-from-branch.yml index e419ea0e0f..97264dac0d 100644 --- a/.github/workflows/run-chosen-tests-from-branch.yml +++ b/.github/workflows/run-chosen-tests-from-branch.yml @@ -30,6 +30,7 @@ on: env: # Environment variable setting gradle options. GRADLE_OPTS: "-XX:MaxHeapSize=2048m -Dorg.gradle.jvmargs='-XX:MaxHeapSize=2048m -XX:MaxPermSize=512m -Dorg.gradle.daemon=false' -Dorg.gradle.daemon=false" + PUSHGATEWAY_HOSTNAME: monitoring.utbot.org jobs: run-chosen-tests: @@ -46,7 +47,7 @@ jobs: run: | echo Find your Prometheus metrics using label {instance=\"${GITHUB_RUN_ID}-${HOSTNAME}\"} chmod +x ./scripts/project/monitoring.sh - ./scripts/project/monitoring.sh ${{ secrets.PUSHGATEWAY_HOSTNAME }} ${{ secrets.PUSHGATEWAY_USER }} ${{ secrets.PUSHGATEWAY_PASSWORD }} + ./scripts/project/monitoring.sh ${PUSHGATEWAY_HOSTNAME} ${{ secrets.PUSHGATEWAY_USER }} ${{ secrets.PUSHGATEWAY_PASSWORD }} - name: Run chosen tests run: | From 069d6900004eb06296311ccb30f6ea823594c2d4 Mon Sep 17 00:00:00 2001 From: Rustam Sadykov <54807972+SBOne-Kenobi@users.noreply.github.com> Date: Mon, 14 Nov 2022 13:57:11 +0300 Subject: [PATCH 4/8] Fix statistics file name --- .github/workflows/collect-statistics.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/collect-statistics.yml b/.github/workflows/collect-statistics.yml index a07275b30f..9410026771 100644 --- a/.github/workflows/collect-statistics.yml +++ b/.github/workflows/collect-statistics.yml @@ -139,7 +139,7 @@ jobs: id: insert shell: bash run: | - OUT_FILE="$data_path/$date-$branch-$short_commit-${{ matrix.value }}.json" + OUT_FILE="$data_path/$date-$short_commit-${{ matrix.value }}.json" echo "output=$OUT_FILE" >> $GITHUB_OUTPUT INPUTS=($(seq ${{ inputs.run_number }})) From 0a8a4afbf72a93cf9dc705deac56407a5efebeb1 Mon Sep 17 00:00:00 2001 From: Rustam Sadykov <54807972+SBOne-Kenobi@users.noreply.github.com> Date: Mon, 14 Nov 2022 16:19:55 +0300 Subject: [PATCH 5/8] Add run_id label --- monitoring/prepare_metrics.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/monitoring/prepare_metrics.py b/monitoring/prepare_metrics.py index a1847db039..59da56fca0 100644 --- a/monitoring/prepare_metrics.py +++ b/monitoring/prepare_metrics.py @@ -82,14 +82,14 @@ def build_metrics_from_data_array(metrics: List[dict], labels: dict) -> List[dic return result -def build_metrics_from_target(target: dict, runner: str) -> List[dict]: +def build_metrics_from_target(target: dict, run_id: str) -> List[dict]: result = [] project = target["target"] result.extend(build_metrics_from_data_array( target["summarised"], { - "runner": runner, + "run_id": run_id, "project": project, "class": "All" } @@ -100,7 +100,7 @@ def build_metrics_from_target(target: dict, runner: str) -> List[dict]: result.extend(build_metrics_from_data_array( class_item["data"], { - "runner": runner, + "run_id": run_id, "project": project, "class": class_name } @@ -109,10 +109,10 @@ def build_metrics_from_target(target: dict, runner: str) -> List[dict]: return result -def build_metrics_from_targets(targets: List[dict], runner: str) -> List[dict]: +def build_metrics_from_targets(targets: List[dict], run_id: str) -> List[dict]: metrics = [] for target in targets: - metrics.extend(build_metrics_from_target(target, runner)) + metrics.extend(build_metrics_from_target(target, run_id)) return metrics @@ -131,11 +131,16 @@ def get_args(): return args +def extract_run_id(text: str): + idx = text.find('-') + return "run" + text[idx:] + + def main(): args = get_args() stats = load(args.stats_file) - runner = stats["metadata"]["environment"]["host"] - metrics = build_metrics_from_targets(stats["targets"], runner) + run_id = extract_run_id(stats["metadata"]["source"]["id"]) + metrics = build_metrics_from_targets(stats["targets"], run_id) metrics.sort(key=lambda x: x["metric"]) with open(args.output_file, "w") as f: json.dump(metrics, f, indent=4) From 214540029ee4231740aace4f7e8128987ceb1e13 Mon Sep 17 00:00:00 2001 From: Egor Vasilyev Date: Thu, 17 Nov 2022 15:07:54 +0300 Subject: [PATCH 6/8] instance name changed --- .github/workflows/collect-statistics.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/collect-statistics.yml b/.github/workflows/collect-statistics.yml index 9410026771..b8795606c0 100644 --- a/.github/workflows/collect-statistics.yml +++ b/.github/workflows/collect-statistics.yml @@ -188,7 +188,7 @@ jobs: run: | python monitoring/prepare_metrics.py --stats_file $stats_file --output_file grafana_metrics.json chmod +x scripts/project/json_to_prometheus.py - python3 scripts/project/json_to_prometheus.py grafana_metrics.json | curl -u "${{ secrets.PUSHGATEWAY_USER }}:${{ secrets.PUSHGATEWAY_PASSWORD }}" --data-binary @- https://${PUSHGATEWAY_HOSTNAME}${PUSHGATEWAY_ADDITIONAL_PATH}/metrics/job/pushgateway-custom/instance/nightly-statistics${PROM_ADDITIONAL_LABELS} + python3 scripts/project/json_to_prometheus.py grafana_metrics.json | curl -u "${{ secrets.PUSHGATEWAY_USER }}:${{ secrets.PUSHGATEWAY_PASSWORD }}" --data-binary @- https://${PUSHGATEWAY_HOSTNAME}${PUSHGATEWAY_ADDITIONAL_PATH}/metrics/job/pushgateway-custom/instance/run-${{ matrix.value }}${PROM_ADDITIONAL_LABELS} echo "Please visit Grafana to check metrics: https://monitoring.utbot.org/d/m6bagaD4z/utbot-nightly-statistic" env: stats_file: ${{ steps.insert.outputs.output }} From 5d842f836ecd7cf7793cd3f4092e10bb980f4255 Mon Sep 17 00:00:00 2001 From: Egor Vasilyev Date: Mon, 21 Nov 2022 16:43:54 +0300 Subject: [PATCH 7/8] gzip added to push metrics --- .github/workflows/collect-statistics.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/collect-statistics.yml b/.github/workflows/collect-statistics.yml index b8795606c0..3746df6264 100644 --- a/.github/workflows/collect-statistics.yml +++ b/.github/workflows/collect-statistics.yml @@ -188,7 +188,7 @@ jobs: run: | python monitoring/prepare_metrics.py --stats_file $stats_file --output_file grafana_metrics.json chmod +x scripts/project/json_to_prometheus.py - python3 scripts/project/json_to_prometheus.py grafana_metrics.json | curl -u "${{ secrets.PUSHGATEWAY_USER }}:${{ secrets.PUSHGATEWAY_PASSWORD }}" --data-binary @- https://${PUSHGATEWAY_HOSTNAME}${PUSHGATEWAY_ADDITIONAL_PATH}/metrics/job/pushgateway-custom/instance/run-${{ matrix.value }}${PROM_ADDITIONAL_LABELS} + python3 scripts/project/json_to_prometheus.py grafana_metrics.json | gzip | curl -u "${{ secrets.PUSHGATEWAY_USER }}:${{ secrets.PUSHGATEWAY_PASSWORD }}" -H "Content-Encoding: gzip" --data-binary @- "https://${PUSHGATEWAY_HOSTNAME}${PUSHGATEWAY_ADDITIONAL_PATH}/metrics/job/pushgateway-custom/instance/run-${{ matrix.value }}${PROM_ADDITIONAL_LABELS}" echo "Please visit Grafana to check metrics: https://monitoring.utbot.org/d/m6bagaD4z/utbot-nightly-statistic" env: stats_file: ${{ steps.insert.outputs.output }} From 4248134f1ca46e1a56c03db4091e490a23c188f2 Mon Sep 17 00:00:00 2001 From: Egor Vasilyev Date: Tue, 22 Nov 2022 10:13:09 +0300 Subject: [PATCH 8/8] gzip pipe removed --- .github/workflows/collect-statistics.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/collect-statistics.yml b/.github/workflows/collect-statistics.yml index 3746df6264..6924cf9e20 100644 --- a/.github/workflows/collect-statistics.yml +++ b/.github/workflows/collect-statistics.yml @@ -188,7 +188,7 @@ jobs: run: | python monitoring/prepare_metrics.py --stats_file $stats_file --output_file grafana_metrics.json chmod +x scripts/project/json_to_prometheus.py - python3 scripts/project/json_to_prometheus.py grafana_metrics.json | gzip | curl -u "${{ secrets.PUSHGATEWAY_USER }}:${{ secrets.PUSHGATEWAY_PASSWORD }}" -H "Content-Encoding: gzip" --data-binary @- "https://${PUSHGATEWAY_HOSTNAME}${PUSHGATEWAY_ADDITIONAL_PATH}/metrics/job/pushgateway-custom/instance/run-${{ matrix.value }}${PROM_ADDITIONAL_LABELS}" + python3 scripts/project/json_to_prometheus.py grafana_metrics.json | curl -u "${{ secrets.PUSHGATEWAY_USER }}:${{ secrets.PUSHGATEWAY_PASSWORD }}" --data-binary @- "https://${PUSHGATEWAY_HOSTNAME}${PUSHGATEWAY_ADDITIONAL_PATH}/metrics/job/pushgateway-custom/instance/run-${{ matrix.value }}${PROM_ADDITIONAL_LABELS}" echo "Please visit Grafana to check metrics: https://monitoring.utbot.org/d/m6bagaD4z/utbot-nightly-statistic" env: stats_file: ${{ steps.insert.outputs.output }}