From 90b76e84deeb20e6a3c54d942b398f6c03d2e069 Mon Sep 17 00:00:00 2001 From: amandelpie Date: Mon, 29 Aug 2022 18:11:22 +0300 Subject: [PATCH] Refactored scripts location --- .../workflows/build-and-run-tests-from-branch.yml | 8 ++++---- .github/workflows/run-chosen-tests-from-branch.yml | 4 ++-- docs/jlearch/pipeline-training-usage.md | 6 +++--- docs/jlearch/scripts.md | 10 +++++----- docs/jlearch/setup.md | 14 +++++++------- .../codeforces_scrapper/codeforces_scrapper.py | 0 scripts/{ => ml}/prepare.sh | 0 scripts/{ => ml}/prog_list | 0 scripts/{ => ml}/quality_analysis.sh | 0 scripts/{ => ml}/requirements.txt | 0 scripts/{ => ml}/run_contest_estimator.sh | 0 scripts/{ => ml}/run_with_coverage.sh | 2 +- scripts/{ => ml}/selector_list | 0 scripts/{ => ml}/train.py | 0 scripts/{ => ml}/train_data.sh | 6 +++--- scripts/{ => ml}/train_iteratively.sh | 8 ++++---- scripts/{ => project}/monitoring.sh | 4 ++-- scripts/{ => project}/ps_parser.sh | 0 18 files changed, 31 insertions(+), 31 deletions(-) rename scripts/{ => ml}/codeforces_scrapper/codeforces_scrapper.py (100%) rename scripts/{ => ml}/prepare.sh (100%) rename scripts/{ => ml}/prog_list (100%) rename scripts/{ => ml}/quality_analysis.sh (100%) rename scripts/{ => ml}/requirements.txt (100%) rename scripts/{ => ml}/run_contest_estimator.sh (100%) rename scripts/{ => ml}/run_with_coverage.sh (84%) rename scripts/{ => ml}/selector_list (100%) rename scripts/{ => ml}/train.py (100%) rename scripts/{ => ml}/train_data.sh (61%) rename scripts/{ => ml}/train_iteratively.sh (63%) rename scripts/{ => project}/monitoring.sh (91%) rename scripts/{ => project}/ps_parser.sh (100%) diff --git a/.github/workflows/build-and-run-tests-from-branch.yml b/.github/workflows/build-and-run-tests-from-branch.yml index a6fb5a3bdc..46567228ee 100644 --- a/.github/workflows/build-and-run-tests-from-branch.yml +++ b/.github/workflows/build-and-run-tests-from-branch.yml @@ -73,8 +73,8 @@ jobs: - name: Run monitoring run: | echo Find your Prometheus metrics using label {instance=\"${GITHUB_RUN_ID}-${HOSTNAME}\"} - chmod +x ./scripts/monitoring.sh - ./scripts/monitoring.sh ${{ secrets.PUSHGATEWAY_HOSTNAME }} ${{ secrets.PUSHGATEWAY_USER }} ${{ secrets.PUSHGATEWAY_PASSWORD }} + chmod +x ./scripts/project/monitoring.sh + ./scripts/project/monitoring.sh ${{ secrets.PUSHGATEWAY_HOSTNAME }} ${{ secrets.PUSHGATEWAY_USER }} ${{ secrets.PUSHGATEWAY_PASSWORD }} - name: Run tests run: | gradle --no-daemon :utbot-framework:test ${{ matrix.project.TESTS_TO_RUN }} @@ -131,8 +131,8 @@ jobs: - name: Run monitoring run: | echo Find your Prometheus metrics using label {instance=\"${GITHUB_RUN_ID}-${HOSTNAME}\"} - chmod +x ./scripts/monitoring.sh - ./scripts/monitoring.sh ${{ secrets.PUSHGATEWAY_HOSTNAME }} ${{ secrets.PUSHGATEWAY_USER }} ${{ secrets.PUSHGATEWAY_PASSWORD }} + chmod +x ./scripts/project/monitoring.sh + ./scripts/project/monitoring.sh ${{ secrets.PUSHGATEWAY_HOSTNAME }} ${{ secrets.PUSHGATEWAY_USER }} ${{ secrets.PUSHGATEWAY_PASSWORD }} - name: Run tests run: | cd ${{ matrix.project }} diff --git a/.github/workflows/run-chosen-tests-from-branch.yml b/.github/workflows/run-chosen-tests-from-branch.yml index b6b60c3097..209fe5aa22 100644 --- a/.github/workflows/run-chosen-tests-from-branch.yml +++ b/.github/workflows/run-chosen-tests-from-branch.yml @@ -45,8 +45,8 @@ jobs: - name: Run monitoring run: | echo Find your Prometheus metrics using label {instance=\"${GITHUB_RUN_ID}-${HOSTNAME}\"} - chmod +x ./scripts/monitoring.sh - ./scripts/monitoring.sh ${{ secrets.PUSHGATEWAY_HOSTNAME }} ${{ secrets.PUSHGATEWAY_USER }} ${{ secrets.PUSHGATEWAY_PASSWORD }} + chmod +x ./scripts/project/monitoring.sh + ./scripts/project/monitoring.sh ${{ secrets.PUSHGATEWAY_HOSTNAME }} ${{ secrets.PUSHGATEWAY_USER }} ${{ secrets.PUSHGATEWAY_PASSWORD }} - name: Run chosen tests run: | diff --git a/docs/jlearch/pipeline-training-usage.md b/docs/jlearch/pipeline-training-usage.md index 32cbd0dddf..9c68d35f36 100644 --- a/docs/jlearch/pipeline-training-usage.md +++ b/docs/jlearch/pipeline-training-usage.md @@ -32,6 +32,6 @@ Briefly: To do this, you should: * Be sure that you use `Java 8` by `java` command and set `JAVA_HOME` to `Java 8`. * Put projects, on which you want to learn in `contest_input/projects` folder, then list classes, on which you want to learn in `contest_input/classes//list` (if it is empty, than we will take all classes from project jar). -* Run `pip install -r scripts/requirements.txt`. It is up to you to make it in virtual environment or not. -* List selectors in `scripts/selector_list` and projects in `scripts/prog_list` -* Run `./scripts/train_iteratively.sh ` +* Run `pip install -r scripts/ml/requirements.txt`. It is up to you to make it in virtual environment or not. +* List selectors in `scripts/ml/selector_list` and projects in `scripts/ml/prog_list` +* Run `./scripts/ml/train_iteratively.sh ` diff --git a/docs/jlearch/scripts.md b/docs/jlearch/scripts.md index 64b4294f47..4e23e43f9e 100644 --- a/docs/jlearch/scripts.md +++ b/docs/jlearch/scripts.md @@ -5,7 +5,7 @@ For each scenario: go to root of `UTBotJava` repository - it is `WORKDIR`. Before start of work run: ```bash -./scripts/prepare.sh +./scripts/ml/prepare.sh ``` It will copy contest resources in `contest_input` folder and build the project, because we use jars, so if you want to change something in code and re-run scripts, then you should run: @@ -16,11 +16,11 @@ It will copy contest resources in `contest_input` folder and build the project, ## To Train a few iterations of your models: By default features directory is `eval/features` - it should be created, to change it you should manually do it in source code of scripts. -List projects and selectors on what you want to train in `scripts/prog_list` and `scripts/selector_list`. You will be trained on all methods of all classes from `contest_input/classes//list`. +List projects and selectors on what you want to train in `scripts/ml/prog_list` and `scripts/selector_list`. You will be trained on all methods of all classes from `contest_input/classes//list`. Then just run: ```bash -./scripts/train_iteratively.sh +./scripts/ml/train_iteratively.sh ``` Python command is your command for python3, in the end of execution you will get iterations models in `` folder and features for each selector and project in `//` for `selector` from `selectors_list` and in `/jlearch//` for models. @@ -29,7 +29,7 @@ Check that `srcTestDir` with your project exist in `build.gradle` of `utbot-juni Then just run: ```bash -./scripts/run_with_coverage.sh +./scripts/ml/run_with_coverage.sh ``` In the end of execution you will get jacoco report in `eval/jacoco///` folder. @@ -37,7 +37,7 @@ In the end of execution you will get jacoco report in `eval/jacoco// +./scripts/ml/quality_analysis.sh ``` It will take coverage reports from relative report folders (at `eval/jacoco/project/alias`) and generate charts in `$outputDir//.html`. `outputDir` can be changed in `QualityAnalysisConfig`. Result file will contain information about 3 metrics: diff --git a/docs/jlearch/setup.md b/docs/jlearch/setup.md index d1bf89c364..d24be69503 100644 --- a/docs/jlearch/setup.md +++ b/docs/jlearch/setup.md @@ -1,7 +1,7 @@ # How to setup environment for experiments on Linux * Clone repository, go to root -* `chmod +x ./scripts/*` and `chmod +x gradlew`. +* `chmod +x ./scripts/ml/*` and `chmod +x gradlew`. * Set `Java 8` as default and set `JAVA_HOME` to this `Java`. For example * Go through [this](https://sdkman.io/install) until `Windows installation` @@ -17,19 +17,19 @@ * `python3 -m venv /path/to/new/virtual/environment` * `source /path/to/venv/bin/activate` * Check `which python3`, it should be somewhere in `path/to/env` folder. - * `pip install -r scripts/requirements.txt` -* `./scripts/prepare.sh` -* Change `scripts/prog_list` to run on smaller project or delete some classes from `contest_input/classes//list`. + * `pip install -r scripts/ml/requirements.txt` +* `./scripts/ml/prepare.sh` +* Change `scripts/ml/prog_list` to run on smaller project or delete some classes from `contest_input/classes//list`. # Default settings and how to change it -* You can reduce number of models in `models` variable in `scripts/train_iteratively.sh` +* You can reduce number of models in `models` variable in `scripts/ml/train_iteratively.sh` * You can change amount of required RAM in `run_contest_estimator.sh`: `16 gb` by default * You can change `batch_size` or `device` n `train.py`: `4096` and `gpu` by default * If you are completing setup on server, then you will need to uncomment tmp directory option in `run_contest_estimator.sh` # Continue setup -* `scripts/train_iteratively.sh 30 2 models ` +* `scripts/ml/train_iteratively.sh 30 2 models ` * In `models/` you should get models. * `mkdir eval/jacoco` -* `./scripts/run_with_coverage.sh 30 "NN_REWARD_GUIDED_SELECTOR path/to/model" some_alias`. `path/to/model` should be something like `models/nn32/0`, where `nn32` is a type of model and `0` is the iteration number +* `./scripts/ml/run_with_coverage.sh 30 "NN_REWARD_GUIDED_SELECTOR path/to/model" some_alias`. `path/to/model` should be something like `models/nn32/0`, where `nn32` is a type of model and `0` is the iteration number * You should get jacoco report in `eval/jacoco/guava-26.0/some_alias/` \ No newline at end of file diff --git a/scripts/codeforces_scrapper/codeforces_scrapper.py b/scripts/ml/codeforces_scrapper/codeforces_scrapper.py similarity index 100% rename from scripts/codeforces_scrapper/codeforces_scrapper.py rename to scripts/ml/codeforces_scrapper/codeforces_scrapper.py diff --git a/scripts/prepare.sh b/scripts/ml/prepare.sh similarity index 100% rename from scripts/prepare.sh rename to scripts/ml/prepare.sh diff --git a/scripts/prog_list b/scripts/ml/prog_list similarity index 100% rename from scripts/prog_list rename to scripts/ml/prog_list diff --git a/scripts/quality_analysis.sh b/scripts/ml/quality_analysis.sh similarity index 100% rename from scripts/quality_analysis.sh rename to scripts/ml/quality_analysis.sh diff --git a/scripts/requirements.txt b/scripts/ml/requirements.txt similarity index 100% rename from scripts/requirements.txt rename to scripts/ml/requirements.txt diff --git a/scripts/run_contest_estimator.sh b/scripts/ml/run_contest_estimator.sh similarity index 100% rename from scripts/run_contest_estimator.sh rename to scripts/ml/run_contest_estimator.sh diff --git a/scripts/run_with_coverage.sh b/scripts/ml/run_with_coverage.sh similarity index 84% rename from scripts/run_with_coverage.sh rename to scripts/ml/run_with_coverage.sh index 29f41ddf20..95ba77eb7b 100644 --- a/scripts/run_with_coverage.sh +++ b/scripts/ml/run_with_coverage.sh @@ -13,7 +13,7 @@ if [[ -n $COVERAGE_PROCESSING ]]; then fi WORKDIR="." -$WORKDIR/scripts/run_contest_estimator.sh $PROJECT $TIME_LIMIT "$PATH_SELECTOR" "" "$COVERAGE_PROCESSING" +$WORKDIR/scripts/ml/run_contest_estimator.sh $PROJECT $TIME_LIMIT "$PATH_SELECTOR" "" "$COVERAGE_PROCESSING" ./gradlew :utbot-junit-contest:test :utbot-junit-contest:jacocoTestReport diff --git a/scripts/selector_list b/scripts/ml/selector_list similarity index 100% rename from scripts/selector_list rename to scripts/ml/selector_list diff --git a/scripts/train.py b/scripts/ml/train.py similarity index 100% rename from scripts/train.py rename to scripts/ml/train.py diff --git a/scripts/train_data.sh b/scripts/ml/train_data.sh similarity index 61% rename from scripts/train_data.sh rename to scripts/ml/train_data.sh index a31e8609fe..25ef4b4709 100644 --- a/scripts/train_data.sh +++ b/scripts/ml/train_data.sh @@ -9,6 +9,6 @@ while read prog; do while read selector; do echo "Starting features collection from $prog with $selector" selector="${selector%%[[:cntrl:]]}" - $WORKDIR/scripts/run_contest_estimator.sh "$prog" "$TIME_LIMIT" "$selector" true - done <"$WORKDIR/scripts/selector_list" -done <"$WORKDIR/scripts/prog_list" + $WORKDIR/scripts/ml/run_contest_estimator.sh "$prog" "$TIME_LIMIT" "$selector" true + done <"$WORKDIR/scripts/ml/selector_list" +done <"$WORKDIR/scripts/ml/prog_list" diff --git a/scripts/train_iteratively.sh b/scripts/ml/train_iteratively.sh similarity index 63% rename from scripts/train_iteratively.sh rename to scripts/ml/train_iteratively.sh index 842fd771bd..a2169a9514 100644 --- a/scripts/train_iteratively.sh +++ b/scripts/ml/train_iteratively.sh @@ -11,7 +11,7 @@ WORKDIR="." echo "Start training data on heuristical based selectors" -$WORKDIR/scripts/train_data.sh $TIME_LIMIT +$WORKDIR/scripts/ml/train_data.sh $TIME_LIMIT echo "Start iterative learning of models" @@ -28,7 +28,7 @@ do echo "EXTRA_ARGS=$EXTRA_ARGS" fi - COMMAND="$PYTHON_COMMAND $WORKDIR/scripts/train.py --features_dir $WORKDIR/eval/features --output_dir $OUTPUT_DIR/$model/$i --prog_list $WORKDIR/scripts/prog_list --model $model $EXTRA_ARGS" + COMMAND="$PYTHON_COMMAND $WORKDIR/scripts/ml/train.py --features_dir $WORKDIR/eval/features --output_dir $OUTPUT_DIR/$model/$i --prog_list $WORKDIR/scripts/prog_list --model $model $EXTRA_ARGS" echo "TRAINING COMMAND=$COMMAND" $COMMAND done @@ -44,7 +44,7 @@ do PREDICTOR="LINEAR" fi - $WORKDIR/scripts/run_contest_estimator.sh $prog $TIME_LIMIT "NN_REWARD_GUIDED_SELECTOR $OUTPUT_DIR/$model/$i $PREDICTOR" "true eval/features/jlearch/$model$i/$prog" + $WORKDIR/scripts/ml/run_contest_estimator.sh $prog $TIME_LIMIT "NN_REWARD_GUIDED_SELECTOR $OUTPUT_DIR/$model/$i $PREDICTOR" "true eval/features/jlearch/$model$i/$prog" done - done <"$WORKDIR/scripts/prog_list" + done <"$WORKDIR/scripts/ml/prog_list" done diff --git a/scripts/monitoring.sh b/scripts/project/monitoring.sh similarity index 91% rename from scripts/monitoring.sh rename to scripts/project/monitoring.sh index ee82bd48f6..e706abce7a 100644 --- a/scripts/monitoring.sh +++ b/scripts/project/monitoring.sh @@ -53,9 +53,9 @@ if ! netstat -tulpn | grep -q ${PORT_NODE_EXPORTER} ; then fi # custom java processes memory metrics -chmod +x scripts/ps_parser.sh +chmod +x scripts/project/ps_parser.sh while true; do - ./scripts/ps_parser.sh | curl -u "${PUSHGATEWAY_USER}":"${PUSHGATEWAY_PASSWORD}" --data-binary @- "https://${PUSHGATEWAY_HOSTNAME}${PUSHGATEWAY_ADDITIONAL_PATH}/metrics/job/pushgateway/instance/${GITHUB_RUN_ID}-${HOSTNAME}${PROM_ADDITIONAL_LABELS}" 2>/dev/null + ./scripts/project/ps_parser.sh | curl -u "${PUSHGATEWAY_USER}":"${PUSHGATEWAY_PASSWORD}" --data-binary @- "https://${PUSHGATEWAY_HOSTNAME}${PUSHGATEWAY_ADDITIONAL_PATH}/metrics/job/pushgateway/instance/${GITHUB_RUN_ID}-${HOSTNAME}${PROM_ADDITIONAL_LABELS}" 2>/dev/null sleep ${SLEEP_TIME_SECONDS} done & diff --git a/scripts/ps_parser.sh b/scripts/project/ps_parser.sh similarity index 100% rename from scripts/ps_parser.sh rename to scripts/project/ps_parser.sh