diff --git a/.github/workflows/collect-statistics.yml b/.github/workflows/collect-statistics.yml index 52696b4c9a..f916f9f49c 100644 --- a/.github/workflows/collect-statistics.yml +++ b/.github/workflows/collect-statistics.yml @@ -9,7 +9,7 @@ on: default: '1' type: string run_number: - description: 'Number of run tries per runner' + description: 'Number of run tries per runner (values greater than 1 are not supported with grafana)' required: false default: '1' type: string @@ -18,8 +18,13 @@ on: required: false default: manual-run type: string - aggregate: - description: 'Aggregate data' + push_results: + description: 'Push metrics into github' + required: false + default: false + type: boolean + send_to_grafana: + description: 'Send metrics to grafana' required: false default: false type: boolean @@ -32,7 +37,7 @@ on: default: '1' type: string run_number: - description: 'Number of run tries per runner' + description: 'Number of run tries per runner (values greater than 1 are not supported with grafana)' required: false default: '1' type: string @@ -41,8 +46,13 @@ on: required: false default: manual-run type: string - aggregate: - description: 'Aggregate data' + push_results: + description: 'Push metrics into github' + required: false + default: false + type: boolean + send_to_grafana: + description: 'Send metrics to grafana' required: false default: false type: boolean @@ -50,8 +60,6 @@ on: env: data_branch: monitoring-data data_path: monitoring/data - aggregated_data_branch: monitoring-aggregated-data - aggregated_data_path: monitoring/aggregated_data monitoring_properties: monitoring/monitoring.properties push_script: monitoring/push_with_rebase.sh @@ -65,7 +73,7 @@ jobs: id: set-matrix run: | arr=$(echo [$(seq -s , ${{ inputs.runners }})]) - echo "::set-output name=matrix::$arr" + echo "matrix=$arr" >> $GITHUB_OUTPUT echo $arr build_and_collect_statistics: @@ -112,27 +120,31 @@ jobs: - name: Get current date id: date run: | - echo "::set-output name=date::$(date +'%Y-%m-%d')" - echo "::set-output name=timestamp::$(date +%s)" - echo "::set-output name=last_month::$(date --date='last month' +%s)" + echo "date=$(date +'%Y-%m-%d-%H-%M-%S')" >> $GITHUB_OUTPUT + echo "timestamp=$(date +%s)" >> $GITHUB_OUTPUT + echo "last_month=$(date --date='last month' +%s)" >> $GITHUB_OUTPUT - name: Get metadata id: metadata run: | - echo "::set-output name=commit::$(git rev-parse HEAD)" - echo "::set-output name=short_commit::$(git rev-parse --short HEAD)" - echo "::set-output name=branch::$(git name-rev --name-only HEAD)" - echo "::set-output name=build::$(date +'%Y.%-m')" + echo "commit=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + echo "short_commit=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + echo "branch=$(git name-rev --name-only HEAD)" >> $GITHUB_OUTPUT + echo "build=$(date +'%Y.%-m')" >> $GITHUB_OUTPUT - name: Insert metadata + id: insert shell: bash run: | - OUT_FILE="$data_path/data-$branch-$date-$timestamp-$short_commit-${{ matrix.value }}.json" + OUT_FILE="$data_path/$date-$branch-$short_commit-${{ matrix.value }}.json" + echo "output=$OUT_FILE" >> $GITHUB_OUTPUT + INPUTS=($(seq ${{ inputs.run_number }})) INPUTS=(${INPUTS[@]/#/stats-}) INPUTS=(${INPUTS[@]/%/.json}) INPUTS=${INPUTS[@]} echo $INPUTS + python monitoring/insert_metadata.py \ --stats_file $INPUTS \ --output_file "$OUT_FILE" \ @@ -151,7 +163,14 @@ jobs: build: ${{ steps.metadata.outputs.build }} run_id: ${{ github.run_id }}-${{ matrix.value }} + - name: Upload statistics + uses: actions/upload-artifact@v3 + with: + name: statistics-${{ matrix.value }} + path: ${{ steps.insert.outputs.output }} + - name: Commit and push statistics + if: ${{ inputs.push_results }} run: | chmod +x $push_script ./$push_script @@ -161,63 +180,17 @@ jobs: message: ${{ inputs.message_prefix }}-${{ steps.date.outputs.date }} github_token: ${{ secrets.GITHUB_TOKEN }} + - name: Send metrics to grafana + if: ${{ inputs.send_to_grafana }} + run: | + python monitoring/prepare_metrics.py --stats_file $stats_file --output_file grafana_metrics.json + echo "TODO send metrics to grafana" + env: + stats_file: ${{ steps.insert.outputs.output }} + - name: Upload logs if: ${{ always() }} uses: actions/upload-artifact@v3 with: name: logs-${{ matrix.value }} path: logs/ - - aggregate: - needs: build_and_collect_statistics - if: ${{ inputs.aggregate }} - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Checkout monitoring data - uses: actions/checkout@v3 - with: - ref: ${{ env.data_branch }} - path: ${{ env.data_path }} - - - name: Checkout aggregated monitoring data - uses: actions/checkout@v3 - with: - ref: ${{ env.aggregated_data_branch }} - path: ${{ env.aggregated_data_path }} - - - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Get current date - id: date - run: | - echo "::set-output name=date::$(date +'%Y-%m-%d')" - echo "::set-output name=timestamp::$(date +%s)" - echo "::set-output name=last_month::$(date --date='last month' +%s)" - - - name: Build aggregated data (last month) - run: | - OUT_FILE=$aggregated_data_path/aggregated-data-$date.json - python monitoring/build_aggregated_data.py \ - --input_data_dir $data_path \ - --output_file $OUT_FILE \ - --timestamp_from $timestamp_from \ - --timestamp_to $timestamp - env: - date: ${{ steps.date.outputs.date }} - timestamp: ${{ steps.date.outputs.timestamp }} - timestamp_from: ${{ steps.date.outputs.last_month }} - - - name: Commit and push aggregated statistics - run: | - chmod +x $push_script - ./$push_script - env: - target_branch: ${{ env.aggregated_data_branch }} - target_directory: ${{ env.aggregated_data_path }} - message: ${{ inputs.message_prefix }}-${{ steps.date.outputs.date }} - github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/night-statistics-monitoring.yml b/.github/workflows/night-statistics-monitoring.yml index c7cda2307b..99af44d6ac 100644 --- a/.github/workflows/night-statistics-monitoring.yml +++ b/.github/workflows/night-statistics-monitoring.yml @@ -12,4 +12,5 @@ jobs: runners: 3 run_number: 1 message_prefix: night-monitoring - aggregate: true + push_results: true + send_to_grafana: true diff --git a/docs/NightStatisticsMonitoring.md b/docs/NightStatisticsMonitoring.md index 2d5ae26cd3..a0a115a847 100644 --- a/docs/NightStatisticsMonitoring.md +++ b/docs/NightStatisticsMonitoring.md @@ -35,79 +35,53 @@ stats.json Output example (the result of three runs during one night): -```json +```json5 [ { "parameters": { - "target": "guava", - "class_timeout_sec": 20, - "run_timeout_min": 20 + "fuzzing_ratio": 0.1, // how long does fuzzing takes + "class_timeout_sec": 20, // test generation timeout for one class + "run_timeout_min": 20 // total timeout for this run }, - "metrics": { - "duration_ms": 604225, - "classes_for_generation": 20, - "testcases_generated": 1651, - "classes_without_problems": 12, - "classes_canceled_by_timeout": 2, - "total_methods_for_generation": 519, - "methods_with_at_least_one_testcase_generated": 365, - "methods_with_exceptions": 46, - "suspicious_methods": 85, - "test_classes_failed_to_compile": 0, - "covered_instructions": 5753, - "covered_instructions_by_fuzzing": 4375, - "covered_instructions_by_concolic": 4069, - "total_instructions": 10182, - "avg_coverage": 62.885408034613 - } - }, - { - "parameters": { - "target": "guava", - "class_timeout_sec": 20, - "run_timeout_min": 20 - }, - "metrics": { - "duration_ms": 633713, - "classes_for_generation": 20, - "testcases_generated": 1872, - "classes_without_problems": 12, - "classes_canceled_by_timeout": 2, - "total_methods_for_generation": 519, - "methods_with_at_least_one_testcase_generated": 413, - "methods_with_exceptions": 46, - "suspicious_methods": 38, - "test_classes_failed_to_compile": 0, - "covered_instructions": 6291, - "covered_instructions_by_fuzzing": 4470, - "covered_instructions_by_concolic": 5232, - "total_instructions": 11011, - "avg_coverage": 62.966064315865275 - } - }, - { - "parameters": { - "target": "guava", - "class_timeout_sec": 20, - "run_timeout_min": 20 - }, - "metrics": { - "duration_ms": 660421, - "classes_for_generation": 20, - "testcases_generated": 1770, - "classes_without_problems": 13, - "classes_canceled_by_timeout": 2, - "total_methods_for_generation": 519, - "methods_with_at_least_one_testcase_generated": 405, - "methods_with_exceptions": 44, - "suspicious_methods": 43, - "test_classes_failed_to_compile": 0, - "covered_instructions": 6266, - "covered_instructions_by_fuzzing": 4543, - "covered_instructions_by_concolic": 5041, - "total_instructions": 11011, - "avg_coverage": 61.59069193429194 - } + "targets": [ // projects that have been processed + { + "target": "guava", // name of project + "summarised_metrics": { // summarised metrics for processed project + "total_classes": 20, // classes count + "testcases_generated": 1042, // generated unit-tests count + "classes_failed_to_compile": 0, // classes that's tests are not compilable + "classes_canceled_by_timeout": 4, // classes that's generation was canceled because of timeout + "total_methods": 526, // methods count + "methods_with_at_least_one_testcase_generated": 345, // methods with at least one successfully generated test + "methods_with_at_least_one_exception": 32, // methods that's generation contains exceptions + "methods_without_any_tests_and_exceptions": 59, // suspicious methods without any tests and exceptions + "covered_bytecode_instructions": 4240, // amount of bytecode instructions that were covered by generated tests + "covered_bytecode_instructions_by_fuzzing": 2946, // amount of bytecode instructions that were covered by fuzzing's generated tests + "covered_bytecode_instructions_by_concolic": 3464, // amount of bytecode instructions that were covered by concolic's generated tests + "total_bytecode_instructions": 9531, // total amount of bytecode instructions in methods with at least one testcase generated + "averaged_bytecode_instruction_coverage_by_classes": 0.5315060991492891 // mean bytecode coverage by class + }, + "metrics_by_class": [ // metrics for all classes in this project + { + "class_name": "com.google.common.math.LongMath", // name of processed class + "metrics": { // metrics for specified class + "testcases_generated": 91, // amount of generated unit-tests + "failed_to_compile": false, // compilation generated tests are failure + "canceled_by_timeout": true, // generation is interrupted because of timeout + "total_methods_in_class": 31, // methods count in this class + "methods_with_at_least_one_testcase_generated": 26, // methods with at least one successfully generated test + "methods_with_at_least_one_exception": 0, // methods that's generation contains exceptions + "methods_without_any_tests_and_exceptions": 5, // suspicious methods without any tests and exceptions + "covered_bytecode_instructions_in_class": 585, // amount of bytecode instructions that were covered by generated tests + "covered_bytecode_instructions_in_class_by_fuzzing": 489, // amount of bytecode instructions that were covered by fuzzing's generated tests + "covered_bytecode_instructions_in_class_by_concolic": 376, // amount of bytecode instructions that were covered by concolic's generated tests + "total_bytecode_instructions_in_class": 1442 // total amount of bytecode instructions in methods with at least one testcase generated + } + }, + // ... + ] + } + ] } ] ``` @@ -120,7 +94,7 @@ The `insert_metadata.py` script is responsible for doing this. To run it you hav To get more information about input arguments call script with option `--help`. -Output format: you get the JSON file, containing statistics and parameters grouped by target project and metadata. +Output format: you get the JSON file, containing metadata, statistics and parameters grouped by target project and classes. Input example: ``` @@ -131,93 +105,78 @@ Input example: ``` Output example (statistics followed by metadata): -```json +```json5 { - "version": 1, - "targets": [ + "version": 2, // version of json format + "targets": [ // projects and methods that have been processed { - "id": "guava", - "version": "0", - "parameters": [ + "target": "guava", // name of project + "summarised": [ // list of summarised metrics with parameters on each run { - "class_timeout_sec": 20, - "run_timeout_min": 20 + "parameters": { + "fuzzing_ratio": 0.1, // how long does fuzzing takes + "class_timeout_sec": 20, // test generation timeout for one class + "run_timeout_min": 20 // total timeout for this run + }, + "metrics": { + "total_classes": 20, // classes count + "testcases_generated": 1042, // generated unit-tests count + "classes_failed_to_compile": 0, // classes that's tests are not compilable + "classes_canceled_by_timeout": 4, // classes that's generation was canceled because of timeout + "total_methods": 526, // methods count + "methods_with_at_least_one_testcase_generated": 345, // methods with at least one successfully generated test + "methods_with_at_least_one_exception": 32, // methods that's generation contains exceptions + "methods_without_any_tests_and_exceptions": 59, // suspicious methods without any tests and exceptions + "total_bytecode_instruction_coverage": 0.44486412758, // total bytecode coverage of generated tests + "total_bytecode_instruction_coverage_by_fuzzing": 0.30909663204, // total bytecode coverage of fuzzing's generated tests + "total_bytecode_instruction_coverage_by_concolic": 0.36344559857, // total bytecode coverage of concolic's generated tests + "averaged_bytecode_instruction_coverage_by_classes": 0.5315060991492891 // mean bytecode coverage by class + } }, - { - "class_timeout_sec": 20, - "run_timeout_min": 20 - }, - { - "class_timeout_sec": 20, - "run_timeout_min": 20 - } + // ... ], - "metrics": [ - { - "duration_ms": 604225, - "classes_for_generation": 20, - "testcases_generated": 1651, - "classes_without_problems": 12, - "classes_canceled_by_timeout": 2, - "total_methods_for_generation": 519, - "methods_with_at_least_one_testcase_generated": 365, - "methods_with_exceptions": 46, - "suspicious_methods": 85, - "test_classes_failed_to_compile": 0, - "covered_instructions": 5753, - "covered_instructions_by_fuzzing": 4375, - "covered_instructions_by_concolic": 4069, - "total_instructions": 10182, - "avg_coverage": 62.885408034613 - }, + "by_class": [ // list of metrics and parameters for all classes in project and on each run { - "duration_ms": 633713, - "classes_for_generation": 20, - "testcases_generated": 1872, - "classes_without_problems": 12, - "classes_canceled_by_timeout": 2, - "total_methods_for_generation": 519, - "methods_with_at_least_one_testcase_generated": 413, - "methods_with_exceptions": 46, - "suspicious_methods": 38, - "test_classes_failed_to_compile": 0, - "covered_instructions": 6291, - "covered_instructions_by_fuzzing": 4470, - "covered_instructions_by_concolic": 5232, - "total_instructions": 11011, - "avg_coverage": 62.966064315865275 + "class_name": "com.google.common.math.LongMath", // name of processed class + "data": [ // metrics and parameters on each run + { + "parameters": { // parameters on this run + "fuzzing_ratio": 0.1, // how long does fuzzing takes + "class_timeout_sec": 20, // test generation timeout for one class + "run_timeout_min": 20 // total timeout for this run + }, + "metrics": { // metrics for specified class on this run + "testcases_generated": 91, // amount of generated unit-tests + "failed_to_compile": false, // compilation generated tests are failure + "canceled_by_timeout": true, // generation is interrupted because of timeout + "total_methods_in_class": 31, // methods count in this class + "methods_with_at_least_one_testcase_generated": 26, // methods with at least one successfully generated test + "methods_with_at_least_one_exception": 0, // methods that's generation contains exceptions + "methods_without_any_tests_and_exceptions": 5, // suspicious methods without any tests and exceptions + "total_bytecode_instruction_coverage_in_class": 0.40568654646, // bytecode coverage of generated tests + "total_bytecode_instruction_coverage_in_class_by_fuzzing": 0.33911234396, // bytecode coverage of fuzzing's generated tests + "total_bytecode_instruction_coverage_in_class_by_concolic": 0.26074895977 // bytecode coverage of concolic's generated tests + } + }, + // ... + ] }, - { - "duration_ms": 660421, - "classes_for_generation": 20, - "testcases_generated": 1770, - "classes_without_problems": 13, - "classes_canceled_by_timeout": 2, - "total_methods_for_generation": 519, - "methods_with_at_least_one_testcase_generated": 405, - "methods_with_exceptions": 44, - "suspicious_methods": 43, - "test_classes_failed_to_compile": 0, - "covered_instructions": 6266, - "covered_instructions_by_fuzzing": 4543, - "covered_instructions_by_concolic": 5041, - "total_instructions": 11011, - "avg_coverage": 61.59069193429194 - } + // ... ] - } + }, + // ... ], - "metadata": { - "source": { + "metadata": { // device's properties + "source": { // information about runner "type": "github-action", "id": "2917672580" }, - "commit_hash": "66a1aeb6", - "branch": "main", - "build_number": "2022.8", - "timestamp": 1661330445, - "date": "2022-08-24T08:40:45", - "environment": { + "commit_hash": "66a1aeb6", // commit hash of used utbot build + "branch": "main", // branch of used utbot build + "build_number": "2022.8", // build number of used utbot build + "timestamp": 1661330445, // run timestamp + "date": "2022-08-24T08:40:45", // human-readable run timestamp + "environment": { // device's environment "host": "fv-az183-700", "OS": "Linux version #20~20.04.1-Ubuntu SMP Fri Aug 5 12:16:53 UTC 2022", "java_version": "openjdk version \"11.0.16\" 2022-07-19 LTS\nOpenJDK Runtime Environment Zulu11.58+15-CA (build 11.0.16+8-LTS)\nOpenJDK 64-Bit Server VM Zulu11.58+15-CA (build 11.0.16+8-LTS, mixed mode)\n", @@ -230,112 +189,53 @@ Output example (statistics followed by metadata): } ``` -### Aggregating - -The `build_aggregated_data.py` script gathers the results for several nights. The collected results for each of the nights are put together into one array. You can specify the period for aggregating. It is useful for visualising or finding statistical characteristics of UnitTestBot performance, e.g. the median or max/min values. - -To run aggregating you should provide the input. - -To get more information about input arguments call script with option `--help`. +### Datastorage structure -Output format: you get the JSON file, which contains arrays of grouped by target results for each of the nights during the specified period. +We store the collected statistics in our repository. You can find two special branches: `monitoring-data` and `monitoring-aggregated-data`. -Input example: +The `monitoring-data` branch is a storage for raw statistics data as well as metadata. -``` ---input_data_dir ./data --output_file aggregated_data.json ---timestamp_from 0 --timestamp_to 1661330445 -``` +The filename format: `--
------.json` -Output example (You'll get an array of several outputs without metadata): -```json -[ - { - "id": "guava", - "version": "0", - "parameters": [ - { - "class_timeout_sec": 20, - "run_timeout_min": 20, - "timestamp": 1661330445 - }, - { - "class_timeout_sec": 20, - "run_timeout_min": 20, - "timestamp": 1661330445 - }, - { - "class_timeout_sec": 20, - "run_timeout_min": 20, - "timestamp": 1661330445 - } - ], - "metrics": [ - { - "duration_ms": 604225, - "classes_for_generation": 20, - "testcases_generated": 1651, - "classes_without_problems": 12, - "classes_canceled_by_timeout": 2, - "total_methods_for_generation": 519, - "methods_with_at_least_one_testcase_generated": 365, - "methods_with_exceptions": 46, - "suspicious_methods": 85, - "test_classes_failed_to_compile": 0, - "avg_coverage": 62.885408034613, - "total_coverage": 56.50166961304262, - "total_coverage_by_fuzzing": 42.967982714594385, - "total_coverage_by_concolic": 39.96267923787075 - }, - { - "duration_ms": 633713, - "classes_for_generation": 20, - "testcases_generated": 1872, - "classes_without_problems": 12, - "classes_canceled_by_timeout": 2, - "total_methods_for_generation": 519, - "methods_with_at_least_one_testcase_generated": 413, - "methods_with_exceptions": 46, - "suspicious_methods": 38, - "test_classes_failed_to_compile": 0, - "avg_coverage": 62.966064315865275, - "total_coverage": 57.133775315593496, - "total_coverage_by_fuzzing": 40.595767868495145, - "total_coverage_by_concolic": 47.51612024339297 - }, - { - "duration_ms": 660421, - "classes_for_generation": 20, - "testcases_generated": 1770, - "classes_without_problems": 13, - "classes_canceled_by_timeout": 2, - "total_methods_for_generation": 519, - "methods_with_at_least_one_testcase_generated": 405, - "methods_with_exceptions": 44, - "suspicious_methods": 43, - "test_classes_failed_to_compile": 0, - "avg_coverage": 61.59069193429194, - "total_coverage": 56.90672963400236, - "total_coverage_by_fuzzing": 41.25874125874126, - "total_coverage_by_concolic": 45.78149123603669 - } - ] - } -] -``` +### Grafana -### Datastorage structure +#### Usage -We store the collected statistics in our repository. You can find two special branches: `monitoring-data` and `monitoring-aggregated-data`. +We can use [Grafana](https://monitoring.utbot.org) for more dynamic and detailed statistics visualisation. Grafana pulls data from our repository automatically. -The `monitoring-data` branch is a storage for raw statistics data as well as metadata. +#### Metrics format -The filename format: `data----
--.json` +Our goal after collecting statistics is uploading results into grafana. For this we should prepare data and send it to our server. -The `monitoring-aggregated-data` branch is a storage for aggregated statistics. The aggregating period is set to one month by default. +The `prepare_metrics.py` script is responsible for doing this. To run it you have to specify the following arguments. -The filename format: `aggregated-data---
.json` +To get more information about input arguments call script with option `--help`. -### Grafana (in process) +Output format: you get the JSON file, containing array of metrics with some labels. -We can use [Grafana](https://monitoring.utbot.org) for more dynamic and detailed statistics visualisation. Grafana pulls data from our repository automatically by means of GitHub API. +Output example: +```json5 +[ + // summarised + { + "metric": "testcases_generated", + "labels": { + "project": "guava", + "fuzzing_ratio": 0.1 + }, + "value": 1024 + }, + // ... + // by classes + { + "metric": "testcases_generated", + "labels": { + "project": "guava", + "class": "com.google.common.math.LongMath", + "fuzzing_ratio": 0.1 + }, + "value": 91 + }, + // ... +] +``` diff --git a/monitoring/build_aggregated_data.py b/monitoring/build_aggregated_data.py deleted file mode 100644 index da2a873a99..0000000000 --- a/monitoring/build_aggregated_data.py +++ /dev/null @@ -1,129 +0,0 @@ -import argparse -import json -from os import listdir -from os.path import isfile, join -from time import time -from typing import Iterator - -from monitoring_settings import JSON_VERSION -from utils import * - - -def get_file_seq(input_data_dir: str) -> Iterator[str]: - """ - Get all files from specified directory - :param input_data_dir: path to directory with files - :return: sequence of filepaths - """ - for filename in listdir(input_data_dir): - path = join(input_data_dir, filename) - if isfile(path): - yield path - - -def check_stats(stats: dict, args: argparse.Namespace) -> bool: - """ - Checks timestamp and version of given statistics - :param stats: dictionary with statistics and metadata - :param args: parsed program arguments - :return: is timestamp and version match - """ - try: - timestamp = stats["metadata"]["timestamp"] - timestamp_match = args.timestamp_from <= timestamp <= args.timestamp_to - json_version_match = stats["version"] == JSON_VERSION - return timestamp_match and json_version_match - except: - return False - - -def get_stats_seq(args: argparse.Namespace) -> Iterator[dict]: - """ - Get statistics with metadata matched specified period - :param args: parsed program arguments - :return: sequence of statistics with metadata filtered by version and timestamp - """ - for file in get_file_seq(args.input_data_dir): - with open(file, "r") as f: - stats = json.load(f) - if check_stats(stats, args): - yield stats - - -def transform_target_stats(stats: dict) -> dict: - """ - Transform metrics by computing total coverage - :param stats: metrics - :return: transformed metrics - """ - common_prefix = "covered_instructions" - denum = stats["total_instructions"] - - nums_keys = [(key, key.removeprefix(common_prefix)) for key in stats.keys() if key.startswith(common_prefix)] - - for (key, by) in nums_keys: - num = stats[key] - stats["total_coverage" + by] = 100 * num / denum if denum != 0 else 0 - del stats[key] - - del stats["total_instructions"] - - return stats - - -def aggregate_stats(stats_seq: Iterator[dict]) -> List[dict]: - """ - Aggregate list of metrics and parameters into list of transformed metrics and parameters grouped by targets - :param stats_seq: sequence of metrics and parameters - :return: list of metrics and parameters grouped by targets - """ - result = get_default_metrics_dict() - - for stats in stats_seq: - targets = stats["targets"] - timestamp = stats["metadata"]["timestamp"] - for target in targets: - full_name = f'{target["id"]}-{target["version"]}' - new_data = result[full_name] - for target_stats in target["metrics"]: - new_data["metrics"].append(transform_target_stats(target_stats)) - for target_params in target["parameters"]: - target_params["timestamp"] = timestamp - new_data["parameters"].append(target_params) - - return postprocess_targets(result) - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument( - '--input_data_dir', required=True, - help='input directory with data', type=str - ) - parser.add_argument( - '--output_file', required=True, - help='output file', type=str - ) - parser.add_argument( - '--timestamp_from', help='timestamp started collection from', - type=int, default=0 - ) - parser.add_argument( - '--timestamp_to', help='timestamp finished collection to', - type=int, default=int(time()) - ) - - args = parser.parse_args() - return args - - -def main(): - args = get_args() - stats_seq = get_stats_seq(args) - result = aggregate_stats(stats_seq) - with open(args.output_file, "w") as f: - json.dump(result, f, indent=4) - - -if __name__ == '__main__': - main() diff --git a/monitoring/insert_metadata.py b/monitoring/insert_metadata.py index e202762cb2..10be3c4b45 100644 --- a/monitoring/insert_metadata.py +++ b/monitoring/insert_metadata.py @@ -1,27 +1,16 @@ import argparse import json +import re import subprocess +from collections import OrderedDict from datetime import datetime from os import environ -from os.path import exists from platform import uname from time import time -from typing import Optional +from typing import Optional, List from monitoring_settings import JSON_VERSION -from utils import * - - -def load(json_file: str) -> Optional[any]: - """ - Try load object from json file - :param json_file: path to json file - :return: object from given json file or None - """ - if exists(json_file): - with open(json_file, "r") as f: - return json.load(f) - return None +from utils import load def try_get_output(args: str) -> Optional[str]: @@ -91,17 +80,142 @@ def build_metadata(args: argparse.Namespace) -> dict: return metadata +def build_target(target_name: str) -> dict: + return { + "target": target_name, + "summarised": [], + "by_class": OrderedDict() + } + + +def transform_metrics(metrics: dict) -> dict: + """ + Transform given metrics with calculation coverage + :param metrics: given metrics + :return: transformed metrics + """ + result = OrderedDict() + + instr_count_prefix = "covered_bytecode_instructions" + total_instr_count_prefix = "total_bytecode_instructions" + + coverage_prefix = "total_bytecode_instruction_coverage" + + total_count = 0 + for metric in metrics: + if metric.startswith(total_instr_count_prefix): + total_count = metrics[metric] + break + + for metric in metrics: + if metric.startswith(total_instr_count_prefix): + continue + if metric.startswith(instr_count_prefix): + coverage = metrics[metric] / total_count if total_count > 0 else 0.0 + result[coverage_prefix + metric.removeprefix(instr_count_prefix)] = coverage + else: + result[metric] = metrics[metric] + + return result + + +def build_data(parameters: dict, metrics: dict) -> dict: + return { + "parameters": { + **parameters + }, + "metrics": { + **transform_metrics(metrics) + } + } + + +def build_by_class(class_name: str) -> dict: + return { + "class_name": class_name, + "data": [] + } + + +def update_from_class(by_class: dict, class_item: dict, parameters: dict): + """ + Update class object using given class_item + :param by_class: dictionary with classname keys + :param class_item: class metrics of current run + :param parameters: parameters of current run + """ + class_name = class_item["class_name"] + if class_name not in by_class: + by_class[class_name] = build_by_class(class_name) + + metrics = class_item["metrics"] + by_class[class_name]["data"].append( + build_data(parameters, metrics) + ) + + +def update_from_target(targets: dict, target_item: dict, parameters: dict): + """ + Update targets using given target_item + :param targets: dictionary with target keys + :param target_item: metrics of current run + :param parameters: parameters of current run + """ + target_name = target_item["target"] + if target_name not in targets: + targets[target_name] = build_target(target_name) + + summarised_metrics = target_item["summarised_metrics"] + targets[target_name]["summarised"].append( + build_data(parameters, summarised_metrics) + ) + + for class_item in target_item["metrics_by_class"]: + update_from_class(targets[target_name]["by_class"], class_item, parameters) + + +def update_from_stats(targets: dict, stats: dict): + """ + Updates targets using given statistics + :param targets: dictionary with target keys + :param stats: target object + """ + parameters = stats["parameters"] + for target_item in stats["targets"]: + update_from_target(targets, target_item, parameters) + + +def postprocess_by_class(by_class: dict) -> List[dict]: + """ + Transform dictionary with classname keys into array with class objects + :param by_class: dictionary with classname keys + :return: array of class objects + """ + return list(by_class.values()) + + +def postprocess_targets(targets: dict) -> List[dict]: + """ + Transform dictionary with target keys into array with target objects + :param targets: dictionary with target keys + :return: array of targets + """ + result = [] + for target in targets.values(): + target["by_class"] = postprocess_by_class(target["by_class"]) + result.append(target) + return result + + def build_targets(stats_array: List[dict]) -> List[dict]: """ Collect and group statistics by target :param stats_array: list of dictionaries with parameters and metrics :return: list of metrics and parameters grouped by target """ - result = get_default_metrics_dict() + result = OrderedDict() for stats in stats_array: - target = stats['parameters']['target'] - del stats['parameters']['target'] - update_target(result[target], stats) + update_from_stats(result, stats) return postprocess_targets(result) diff --git a/monitoring/monitoring.properties b/monitoring/monitoring.properties index e3062857d0..0a1694eb88 100644 --- a/monitoring/monitoring.properties +++ b/monitoring/monitoring.properties @@ -1,4 +1,4 @@ -project=guava -classTimeoutMillis=20 -runTries=1 -runTimeoutMinutes=20 \ No newline at end of file +projects=guava +classTimeoutSeconds=20 +runTimeoutMinutes=20 +fuzzingRatios=0.0;0.1;1.0 \ No newline at end of file diff --git a/monitoring/monitoring_settings.py b/monitoring/monitoring_settings.py index 301e7b7ae3..8b0fb83a0f 100644 --- a/monitoring/monitoring_settings.py +++ b/monitoring/monitoring_settings.py @@ -1,9 +1,4 @@ """ Json format version. """ -JSON_VERSION = 1 - -""" -Default version for projects without it. -""" -DEFAULT_PROJECT_VERSION = "0" +JSON_VERSION = 2 diff --git a/monitoring/prepare_metrics.py b/monitoring/prepare_metrics.py new file mode 100644 index 0000000000..a5bffac2c4 --- /dev/null +++ b/monitoring/prepare_metrics.py @@ -0,0 +1,142 @@ +import argparse +import json +from typing import List + +from utils import load + + +def remove_in_class(name: str) -> str: + in_class = "_in_class" + idx = name.find(in_class) + if idx == -1: + return name + return name[:idx] + name[idx:].removeprefix(in_class) + + +def update_from_counter_name(key_word: str, name: str, labels: dict) -> str: + if name == f"total_{key_word}": + labels["type"] = "total" + return key_word + if name.startswith(key_word): + labels["type"] = name.removeprefix(f"{key_word}_") + return key_word + return name + + +def update_from_coverage(name: str, labels: dict) -> str: + coverage_key = "bytecode_instruction_coverage" + idx = name.find(coverage_key) + if idx == -1: + return name + labels["type"] = name[:idx - 1] + source = name[idx:].removeprefix(f"{coverage_key}") + if len(source) > 0: + source = source.removeprefix("_by_") + if source == "classes": + labels["type"] = "averaged_by_classes" + else: + labels["source"] = source + return coverage_key + + +def build_metric_struct(name: str, value: any, labels: dict) -> dict: + name = remove_in_class(name) + name = update_from_counter_name("classes", name, labels) + name = update_from_counter_name("methods", name, labels) + name = update_from_coverage(name, labels) + + if type(value) == bool: + value = int(value) + name = f"test_generation_{name}" + elif type(value) == int: + name = f"{name}_total" + + name = f"utbot_{name}" + + return { + "metric": name, + "labels": labels, + "value": value + } + + +def build_metrics_from_data(data: dict, labels: dict) -> List[dict]: + result = [] + fuzzing_ratio = data["parameters"]["fuzzing_ratio"] + new_labels = { + **labels, + "fuzzing_ratio": fuzzing_ratio + } + metrics = data["metrics"] + for metric in metrics: + result.append(build_metric_struct(metric, metrics[metric], new_labels.copy())) + return result + + +def build_metrics_from_data_array(metrics: List[dict], labels: dict) -> List[dict]: + result = [] + for metric in metrics: + result.extend(build_metrics_from_data(metric, labels)) + return result + + +def build_metrics_from_target(target: dict, runner: str) -> List[dict]: + result = [] + project = target["target"] + + result.extend(build_metrics_from_data_array( + target["summarised"], + { + "runner": runner, + "project": project + } + )) + + for class_item in target["by_class"]: + class_name = class_item["class_name"] + result.extend(build_metrics_from_data_array( + class_item["data"], + { + "runner": runner, + "project": project, + "class": class_name + } + )) + + return result + + +def build_metrics_from_targets(targets: List[dict], runner: str) -> List[dict]: + metrics = [] + for target in targets: + metrics.extend(build_metrics_from_target(target, runner)) + return metrics + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + '--stats_file', required=True, + help='files with statistics after insertion metadata', type=str + ) + parser.add_argument( + '--output_file', required=True, + help='output file', type=str + ) + + args = parser.parse_args() + return args + + +def main(): + args = get_args() + stats = load(args.stats_file) + runner = stats["metadata"]["environment"]["host"] + metrics = build_metrics_from_targets(stats["targets"], runner) + metrics.sort(key=lambda x: x["metric"]) + with open(args.output_file, "w") as f: + json.dump(metrics, f, indent=4) + + +if __name__ == "__main__": + main() diff --git a/monitoring/utils.py b/monitoring/utils.py index d5fd1a5a2f..c897f46df6 100644 --- a/monitoring/utils.py +++ b/monitoring/utils.py @@ -1,56 +1,15 @@ -import re -from collections import defaultdict -from typing import List +import json +from os.path import exists +from typing import Optional -from monitoring_settings import DEFAULT_PROJECT_VERSION - -def parse_name_and_version(full_name: str) -> tuple[str, str]: - """ - Parse from string name and version of project - :param full_name: string with format - - :return: pair of name and version strings - """ - regex = re.compile(r'([^.]+)-([\d.]+)') - result = regex.fullmatch(full_name) - if result is None: - return full_name, DEFAULT_PROJECT_VERSION - name = result.group(1) - version = result.group(2) - return name, version - - -def postprocess_targets(targets: dict) -> List[dict]: - """ - Transform dictionary with fullname target keys into array with target objects - :param targets: dictionary with fullname target keys - :return: array of targets - """ - result = [] - for target in targets: - (name, version) = parse_name_and_version(target) - result.append({ - 'id': name, - 'version': version, - **targets[target] - }) - return result - - -def get_default_metrics_dict() -> dict: - return defaultdict(lambda: { - 'parameters': [], - 'metrics': [] - }) - - -def update_target(target: dict, stats: dict) -> dict: +def load(json_file: str) -> Optional[any]: """ - Update dictionary with target by new stats - :param target: dictionary with target metrics and parameters - :param stats: new metrics and parameters - :return: updated target dictionary + Try load object from json file + :param json_file: path to json file + :return: object from given json file or None """ - target['parameters'].append(stats['parameters']) - target['metrics'].append(stats['metrics']) - return target + if exists(json_file): + with open(json_file, "r") as f: + return json.load(f) + return None diff --git a/utbot-core/src/main/kotlin/org/utbot/common/AbstractSettings.kt b/utbot-core/src/main/kotlin/org/utbot/common/AbstractSettings.kt index f564d98a20..d3db9a002e 100644 --- a/utbot-core/src/main/kotlin/org/utbot/common/AbstractSettings.kt +++ b/utbot-core/src/main/kotlin/org/utbot/common/AbstractSettings.kt @@ -128,6 +128,9 @@ abstract class AbstractSettings( protected fun getStringProperty(defaultValue: String) = getProperty(defaultValue) { it } protected inline fun > getEnumProperty(defaultValue: T) = getProperty(defaultValue) { enumValueOf(it) } - + protected fun getListProperty(defaultValue: List) = + getProperty(defaultValue) { it.split(';') } + protected inline fun getListProperty(defaultValue: List, crossinline elementTransform: (String) -> T) = + getProperty(defaultValue) { it.split(';').map(elementTransform) } override fun toString(): String = container.toString() } \ No newline at end of file diff --git a/utbot-junit-contest/build.gradle b/utbot-junit-contest/build.gradle index 720494e9b8..dd40d660f7 100644 --- a/utbot-junit-contest/build.gradle +++ b/utbot-junit-contest/build.gradle @@ -1,3 +1,6 @@ +plugins { + id 'org.jetbrains.kotlin.plugin.serialization' version '1.7.20' +} apply plugin: 'jacoco' configurations { @@ -58,6 +61,7 @@ dependencies { implementation group: 'org.apache.commons', name: 'commons-exec', version: '1.2' implementation group: 'io.github.microutils', name: 'kotlin-logging', version: kotlinLoggingVersion implementation group: 'org.jsoup', name: 'jsoup', version: '1.6.2' + implementation 'org.jetbrains.kotlinx:kotlinx-serialization-json:1.4.1' // need for tests implementation group: 'org.mockito', name: 'mockito-core', version: '4.2.0' implementation group: 'org.mockito', name: 'mockito-inline', version: '4.2.0' diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt index 2bd9389f7f..8e2646d1e0 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt @@ -138,7 +138,15 @@ fun main(args: Array) { val timeBudgetSec = cmd[2].toLong() val cut = ClassUnderTest(classLoader.loadClass(classUnderTestName).id, outputDir, classfileDir.toFile()) - runGeneration(cut, timeBudgetSec, classpathString, runFromEstimator = false, methodNameFilter = null) + runGeneration( + project = "Contest", + cut, + timeBudgetSec, + fuzzingRatio = 0.1, + classpathString, + runFromEstimator = false, + methodNameFilter = null + ) println("${ContestMessage.READY}") } } @@ -162,8 +170,10 @@ fun setOptions() { @ObsoleteCoroutinesApi @SuppressWarnings fun runGeneration( + project: String, cut: ClassUnderTest, timeLimitSec: Long, + fuzzingRatio: Double, classpathString: String, runFromEstimator: Boolean, methodNameFilter: String? = null // For debug purposes you can specify method name @@ -205,7 +215,7 @@ fun runGeneration( logger.error("Seems like classloader for cut not valid (maybe it was backported to system): ${cut.classLoader}") } - val statsForClass = StatsForClass() + val statsForClass = StatsForClass(project, cut.fqn) val codeGenerator = CodeGenerator( cut.classId, @@ -276,9 +286,11 @@ fun runGeneration( val budgetForSymbolicExecution = max(0, budgetForMethod - budgetForLastSolverRequestAndConcreteExecutionRemainingStates) + UtSettings.utBotGenerationTimeoutInMillis = budgetForMethod + UtSettings.fuzzingTimeoutInMillis = (budgetForMethod * fuzzingRatio).toLong() //start controller that will activate symbolic execution - GlobalScope.launch { + GlobalScope.launch(currentContext) { delay(budgetForSymbolicExecution) if (methodJob.isActive) { @@ -359,7 +371,7 @@ fun runGeneration( } - val cancellator = GlobalScope.launch { + val cancellator = GlobalScope.launch(currentContext) { delay(remainingBudget()) if (engineJob.isActive) { logger.warn { "Cancelling job because timeout $generationTimeout ms elapsed (real cancellation can take time)" } diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt index 3cd74bb432..7c82e668f2 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt @@ -29,13 +29,11 @@ import org.utbot.features.FeatureExtractorFactoryImpl import org.utbot.features.FeatureProcessorWithStatesRepetitionFactory import org.utbot.framework.PathSelectorType import org.utbot.framework.UtSettings -import org.utbot.framework.plugin.api.util.UtContext import org.utbot.framework.plugin.api.util.id import org.utbot.framework.plugin.api.util.withUtContext import org.utbot.framework.plugin.services.JdkInfoService import org.utbot.instrumentation.ConcreteExecutor import org.utbot.predictors.MLPredictorFactoryImpl -import kotlin.concurrent.thread import kotlin.math.min private val logger = KotlinLogging.logger {} @@ -48,6 +46,56 @@ private val javaHome = System.getenv("JAVA_HOME") private val javacCmd = "$javaHome/bin/javac" private val javaCmd = "$javaHome/bin/java" +private const val compileAttempts = 2 + +private data class UnnamedPackageInfo(val pack: String, val module: String) + +private fun findAllNotExportedPackages(report: String): List { + val regex = """package ([\d\w.]+) is declared in module ([\d\w.]+), which does not export it to the unnamed module""".toRegex() + return regex.findAll(report).map { + val pack = it.groupValues[1] + val module = it.groupValues[2] + UnnamedPackageInfo(pack, module) + }.toList().distinct() +} + +private fun compileClass(testDir: String, classPath: String, testClass: String): Int { + val exports = mutableSetOf() + var exitCode = 0 + + repeat(compileAttempts) { attemptNumber -> + val cmd = arrayOf( + javacCmd, + *exports.flatMap { + listOf("--add-exports", "${it.module}/${it.pack}=ALL-UNNAMED") + }.toTypedArray(), + "-d", testDir, + "-cp", classPath, + "-nowarn", + "-XDignore.symbol.file", + testClass + ) + logger.debug { "Compile attempt ${attemptNumber + 1}" } + + logger.trace { cmd.toText() } + + val process = Runtime.getRuntime().exec(cmd) + + val errors = process.errorStream.reader().buffered().readText() + + exitCode = process.waitFor() + if (exitCode == 0) { + return 0 + } else { + if (errors.isNotEmpty()) + logger.error { "Compilation errors: $errors" } + exports += findAllNotExportedPackages(errors) + } + } + + return exitCode +} + fun Array.toText() = joinToString(separator = ",") @Suppress("unused") @@ -83,16 +131,19 @@ enum class Tool { project: ProjectToEstimate, cut: ClassUnderTest, timeLimit: Long, + fuzzingRatio: Double, methodNameFilter: String?, - globalStats: GlobalStats, + statsForProject: StatsForProject, compiledTestDir: File, classFqn: String ) { val classStats: StatsForClass = try { - withUtContext(UtContext(project.classloader)) { + withUtContext(ContextManager.createNewContext(project.classloader)) { runGeneration( + project.name, cut, timeLimit, + fuzzingRatio, project.sootClasspathString, runFromEstimator = true, methodNameFilter @@ -107,34 +158,18 @@ enum class Tool { return } - globalStats.statsForClasses.add(classStats) + statsForProject.statsForClasses.add(classStats) try { val testClass = cut.generatedTestFile classStats.testClassFile = testClass - val cmd = arrayOf( - javacCmd, - "-d", compiledTestDir.absolutePath, - "-cp", project.compileClasspathString, - "-nowarn", - "-XDignore.symbol.file", - testClass.absolutePath - ) - logger.info().bracket("Compiling class ${testClass.absolutePath}") { - - logger.trace { cmd.toText() } - val process = Runtime.getRuntime().exec(cmd) - - thread { - val errors = process.errorStream.reader().buffered().readText() - if (errors.isNotEmpty()) - logger.error { "Compilation errors: $errors" } - }.join() - - - val exitCode = process.waitFor() + val exitCode = compileClass( + compiledTestDir.absolutePath, + project.compileClasspathString, + testClass.absolutePath + ) if (exitCode != 0) { logger.error { "Failed to compile test class ${cut.testClassSimpleName}" } classStats.failedToCompile = true @@ -167,8 +202,9 @@ enum class Tool { project: ProjectToEstimate, cut: ClassUnderTest, timeLimit: Long, + fuzzingRatio: Double, methodNameFilter: String?, - globalStats: GlobalStats, + statsForProject: StatsForProject, compiledTestDir: File, classFqn: String ) { @@ -237,8 +273,9 @@ enum class Tool { project: ProjectToEstimate, cut: ClassUnderTest, timeLimit: Long, + fuzzingRatio: Double, // maybe create some specific settings methodNameFilter: String?, - globalStats: GlobalStats, + statsForProject: StatsForProject, compiledTestDir: File, classFqn: String ) @@ -257,6 +294,7 @@ fun main(args: Array) { if (args.isEmpty() && System.getProperty("os.name")?.run { contains("win", ignoreCase = true) } == true) { processedClassesThreshold = 9999 //change to change number of classes to run val timeLimit = 20 // increase if you want to debug something + val fuzzingRatio = 0.1 // sets fuzzing ratio to total test generation // Uncomment it for debug purposes: // you can specify method for test generation in format `classFqn.methodName` @@ -283,12 +321,13 @@ fun main(args: Array) { classesLists, jarsDir, "$timeLimit", + "$fuzzingRatio", outputDir, moduleTestDir ) } else { - require(args.size == 6) { - "Wrong arguments: