diff --git a/.github/workflows/collect-statistics.yml b/.github/workflows/collect-statistics.yml index f916f9f49c..6924cf9e20 100644 --- a/.github/workflows/collect-statistics.yml +++ b/.github/workflows/collect-statistics.yml @@ -62,6 +62,9 @@ env: data_path: monitoring/data monitoring_properties: monitoring/monitoring.properties push_script: monitoring/push_with_rebase.sh + PUSHGATEWAY_HOSTNAME: monitoring.utbot.org + PUSHGATEWAY_ADDITIONAL_PATH: /pushgateway-custom + PROM_ADDITIONAL_LABELS: /service/github jobs: setup_matrix: @@ -136,7 +139,7 @@ jobs: id: insert shell: bash run: | - OUT_FILE="$data_path/$date-$branch-$short_commit-${{ matrix.value }}.json" + OUT_FILE="$data_path/$date-$short_commit-${{ matrix.value }}.json" echo "output=$OUT_FILE" >> $GITHUB_OUTPUT INPUTS=($(seq ${{ inputs.run_number }})) @@ -184,7 +187,9 @@ jobs: if: ${{ inputs.send_to_grafana }} run: | python monitoring/prepare_metrics.py --stats_file $stats_file --output_file grafana_metrics.json - echo "TODO send metrics to grafana" + chmod +x scripts/project/json_to_prometheus.py + python3 scripts/project/json_to_prometheus.py grafana_metrics.json | curl -u "${{ secrets.PUSHGATEWAY_USER }}:${{ secrets.PUSHGATEWAY_PASSWORD }}" --data-binary @- "https://${PUSHGATEWAY_HOSTNAME}${PUSHGATEWAY_ADDITIONAL_PATH}/metrics/job/pushgateway-custom/instance/run-${{ matrix.value }}${PROM_ADDITIONAL_LABELS}" + echo "Please visit Grafana to check metrics: https://monitoring.utbot.org/d/m6bagaD4z/utbot-nightly-statistic" env: stats_file: ${{ steps.insert.outputs.output }} diff --git a/.github/workflows/run-chosen-tests-from-branch.yml b/.github/workflows/run-chosen-tests-from-branch.yml index e419ea0e0f..97264dac0d 100644 --- a/.github/workflows/run-chosen-tests-from-branch.yml +++ b/.github/workflows/run-chosen-tests-from-branch.yml @@ -30,6 +30,7 @@ on: env: # Environment variable setting gradle options. GRADLE_OPTS: "-XX:MaxHeapSize=2048m -Dorg.gradle.jvmargs='-XX:MaxHeapSize=2048m -XX:MaxPermSize=512m -Dorg.gradle.daemon=false' -Dorg.gradle.daemon=false" + PUSHGATEWAY_HOSTNAME: monitoring.utbot.org jobs: run-chosen-tests: @@ -46,7 +47,7 @@ jobs: run: | echo Find your Prometheus metrics using label {instance=\"${GITHUB_RUN_ID}-${HOSTNAME}\"} chmod +x ./scripts/project/monitoring.sh - ./scripts/project/monitoring.sh ${{ secrets.PUSHGATEWAY_HOSTNAME }} ${{ secrets.PUSHGATEWAY_USER }} ${{ secrets.PUSHGATEWAY_PASSWORD }} + ./scripts/project/monitoring.sh ${PUSHGATEWAY_HOSTNAME} ${{ secrets.PUSHGATEWAY_USER }} ${{ secrets.PUSHGATEWAY_PASSWORD }} - name: Run chosen tests run: | diff --git a/monitoring/prepare_metrics.py b/monitoring/prepare_metrics.py index a1847db039..59da56fca0 100644 --- a/monitoring/prepare_metrics.py +++ b/monitoring/prepare_metrics.py @@ -82,14 +82,14 @@ def build_metrics_from_data_array(metrics: List[dict], labels: dict) -> List[dic return result -def build_metrics_from_target(target: dict, runner: str) -> List[dict]: +def build_metrics_from_target(target: dict, run_id: str) -> List[dict]: result = [] project = target["target"] result.extend(build_metrics_from_data_array( target["summarised"], { - "runner": runner, + "run_id": run_id, "project": project, "class": "All" } @@ -100,7 +100,7 @@ def build_metrics_from_target(target: dict, runner: str) -> List[dict]: result.extend(build_metrics_from_data_array( class_item["data"], { - "runner": runner, + "run_id": run_id, "project": project, "class": class_name } @@ -109,10 +109,10 @@ def build_metrics_from_target(target: dict, runner: str) -> List[dict]: return result -def build_metrics_from_targets(targets: List[dict], runner: str) -> List[dict]: +def build_metrics_from_targets(targets: List[dict], run_id: str) -> List[dict]: metrics = [] for target in targets: - metrics.extend(build_metrics_from_target(target, runner)) + metrics.extend(build_metrics_from_target(target, run_id)) return metrics @@ -131,11 +131,16 @@ def get_args(): return args +def extract_run_id(text: str): + idx = text.find('-') + return "run" + text[idx:] + + def main(): args = get_args() stats = load(args.stats_file) - runner = stats["metadata"]["environment"]["host"] - metrics = build_metrics_from_targets(stats["targets"], runner) + run_id = extract_run_id(stats["metadata"]["source"]["id"]) + metrics = build_metrics_from_targets(stats["targets"], run_id) metrics.sort(key=lambda x: x["metric"]) with open(args.output_file, "w") as f: json.dump(metrics, f, indent=4) diff --git a/scripts/project/json_to_prometheus.py b/scripts/project/json_to_prometheus.py new file mode 100644 index 0000000000..0a00270f9b --- /dev/null +++ b/scripts/project/json_to_prometheus.py @@ -0,0 +1,36 @@ +import sys +import json + +with open(sys.argv[1]) as metrics_raw: + metrics_json = json.load(metrics_raw) + +# metrics is a json list e.g.: +# [ +# { +# "metric": "total_classes", +# "labels": { +# "project": "guava", +# "fuzzing_ratio": 0.1 +# }, +# "value": 20 +# }, +# { +# "metric": "testcases_generated", +# "labels": { +# "project": "guava", +# "fuzzing_ratio": 0.1 +# }, +# "value": 1042 +# } +# ] +# +# the loop below iterates over each list item and constructs metrics set +metrics_set_str = "" +for metric in metrics_json: + labels_set_str = "" + comma = "" + for label, value in metric['labels'].items(): + labels_set_str = f'{labels_set_str}{comma}{label}=\"{value}\"' + comma = "," + metrics_set_str += f'{metric["metric"]}{{{labels_set_str}}} {metric["value"]}\n' +print(metrics_set_str)