From 2892b49459f2313928052c56ca6bf3c070bf0140 Mon Sep 17 00:00:00 2001 From: Egor Vasilyev Date: Thu, 22 Jun 2023 12:44:06 +0300 Subject: [PATCH 01/12] fix workflow image #2307 --- .github/workflows/collect-statistics.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/collect-statistics.yml b/.github/workflows/collect-statistics.yml index 499ea35e0b..6ecfb5795c 100644 --- a/.github/workflows/collect-statistics.yml +++ b/.github/workflows/collect-statistics.yml @@ -101,11 +101,10 @@ jobs: project: ${{ fromJson(needs.setup_matrix.outputs.projects) }} value: ${{ fromJson(needs.setup_matrix.outputs.matrix) }} runs-on: ubuntu-20.04 - container: unittestbot/java-env:java17-zulu-jdk-fx-gradle7.6.1-kotlinc1.8.0 + container: unittestbot/java-env:java17-zulu-jdk-gradle7.6.1-kotlinc1.8.0 steps: - name: Install git run: | - apt-get upgrade -y apt-get update -y apt-get install git -y git config --global --add safe.directory $(pwd) From 7e092e51e9fa34215fe928ea400c215317c4ae68 Mon Sep 17 00:00:00 2001 From: Egor Vasilyev Date: Thu, 22 Jun 2023 15:02:52 +0300 Subject: [PATCH 02/12] fix workflow name --- .github/workflows/collect-statistics.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/collect-statistics.yml b/.github/workflows/collect-statistics.yml index 6ecfb5795c..b810ff2fe4 100644 --- a/.github/workflows/collect-statistics.yml +++ b/.github/workflows/collect-statistics.yml @@ -1,4 +1,4 @@ -name: "UTBot Java: collect statistics" +name: "[M] UTBot Java: collect statistics" on: workflow_call: From 0be6e8964a62d0b525253436242dfae5dd2658f8 Mon Sep 17 00:00:00 2001 From: Egor Vasilyev Date: Thu, 22 Jun 2023 15:39:40 +0300 Subject: [PATCH 03/12] --add-opens options added --- .github/workflows/collect-statistics.yml | 49 ++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/.github/workflows/collect-statistics.yml b/.github/workflows/collect-statistics.yml index b810ff2fe4..bd73802bf4 100644 --- a/.github/workflows/collect-statistics.yml +++ b/.github/workflows/collect-statistics.yml @@ -128,6 +128,55 @@ jobs: for i in $(seq ${{ inputs.run_number }}) do java -jar \ + --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED \ + --add-opens java.base/java.lang.invoke=ALL-UNNAMED \ + --add-opens java.base/java.util.concurrent=ALL-UNNAMED \ + --add-opens java.base/java.util.concurrent.locks=ALL-UNNAMED \ + --add-opens java.base/java.text=ALL-UNNAMED \ + --add-opens java.base/java.io=ALL-UNNAMED \ + --add-opens java.base/java.nio=ALL-UNNAMED \ + --add-opens java.base/java.nio.file=ALL-UNNAMED \ + --add-opens java.base/java.net=ALL-UNNAMED \ + --add-opens java.base/sun.security.util=ALL-UNNAMED \ + --add-opens java.base/sun.reflect.generics.repository=ALL-UNNAMED \ + --add-opens java.base/sun.net.util=ALL-UNNAMED \ + --add-opens java.base/sun.net.fs=ALL-UNNAMED \ + --add-opens java.base/java.security=ALL-UNNAMED \ + --add-opens java.base/java.lang.ref=ALL-UNNAMED \ + --add-opens java.base/java.math=ALL-UNNAMED \ + --add-opens java.base/java.util.stream=ALL-UNNAMED \ + --add-opens java.base/java.util=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.misc=ALL-UNNAMED \ + --add-opens java.base/java.lang=ALL-UNNAMED \ + --add-opens java.base/java.lang.reflect=ALL-UNNAMED \ + --add-opens java.base/sun.security.provider=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.event=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.jimage=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.jimage.decompressor=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.jmod=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.jtrfs=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.loader=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.logger=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.math=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.misc=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.module=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.org.objectweb.asm.commons=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.org.objectweb.asm.signature=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.org.objectweb.asm.tree=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.org.objectweb.asm.tree.analysis=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.org.objectweb.asm.util=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.org.xml.sax=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.org.xml.sax.helpers=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.perf=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.platform=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.ref=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.reflect=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.util=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.util.jar=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.util.xml=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.util.xml.impl=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.vm=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.vm.annotation=ALL-UNNAMED \ -Dutbot.monitoring.settings.path=$monitoring_projects/${{ matrix.project }}/monitoring.properties \ utbot-junit-contest/build/libs/monitoring.jar \ stats-$i.json From db34df3fcc9a22a053415ba9d17034813ae9c88b Mon Sep 17 00:00:00 2001 From: Egor Vasilyev Date: Fri, 23 Jun 2023 15:34:41 +0300 Subject: [PATCH 04/12] -mx512 param added --- .github/workflows/collect-statistics.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/collect-statistics.yml b/.github/workflows/collect-statistics.yml index bd73802bf4..61632262f8 100644 --- a/.github/workflows/collect-statistics.yml +++ b/.github/workflows/collect-statistics.yml @@ -178,6 +178,7 @@ jobs: --add-opens java.base/jdk.internal.vm=ALL-UNNAMED \ --add-opens java.base/jdk.internal.vm.annotation=ALL-UNNAMED \ -Dutbot.monitoring.settings.path=$monitoring_projects/${{ matrix.project }}/monitoring.properties \ + -Xmx512M \ utbot-junit-contest/build/libs/monitoring.jar \ stats-$i.json mv logs/utbot.log logs/utbot-$i.log From 54ef2a38d5e1cfa6fd7ef23a8148901a9d992d95 Mon Sep 17 00:00:00 2001 From: Egor Vasilyev Date: Mon, 7 Aug 2023 09:36:29 +0300 Subject: [PATCH 05/12] add debug --- .github/workflows/collect-statistics.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/collect-statistics.yml b/.github/workflows/collect-statistics.yml index 61632262f8..fdf8297cb2 100644 --- a/.github/workflows/collect-statistics.yml +++ b/.github/workflows/collect-statistics.yml @@ -199,6 +199,13 @@ jobs: echo "branch=$(git name-rev --name-only HEAD)" >> $GITHUB_OUTPUT echo "build=$(date +'%Y.%-m')" >> $GITHUB_OUTPUT +# temporary debug + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + with: + sudo: false +# /temporary debug + - name: Insert metadata id: insert shell: bash From 9823fba3bb5ac7cbc69b62135ffc950a099a4e29 Mon Sep 17 00:00:00 2001 From: Egor Vasilyev Date: Mon, 7 Aug 2023 10:08:37 +0300 Subject: [PATCH 06/12] add fix to debug --- .github/workflows/collect-statistics.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/collect-statistics.yml b/.github/workflows/collect-statistics.yml index fdf8297cb2..841440943d 100644 --- a/.github/workflows/collect-statistics.yml +++ b/.github/workflows/collect-statistics.yml @@ -204,6 +204,7 @@ jobs: uses: mxschmitt/action-tmate@v3 with: sudo: false + limit-access-to-actor: false # /temporary debug - name: Insert metadata From 2eb2951dc92ed64b8ffd90e00fdb4e87e40ef146 Mon Sep 17 00:00:00 2001 From: Egor Vasilyev Date: Mon, 7 Aug 2023 10:09:20 +0300 Subject: [PATCH 07/12] change place of debug task --- .github/workflows/collect-statistics.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/collect-statistics.yml b/.github/workflows/collect-statistics.yml index 841440943d..debfc08be2 100644 --- a/.github/workflows/collect-statistics.yml +++ b/.github/workflows/collect-statistics.yml @@ -122,6 +122,14 @@ jobs: with: python-version: '3.9' + # temporary debug + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + with: + sudo: false + limit-access-to-actor: false + # /temporary debug + - name: Build and run monitoring UTBot Java run: | gradle :utbot-junit-contest:monitoringJar @@ -199,14 +207,6 @@ jobs: echo "branch=$(git name-rev --name-only HEAD)" >> $GITHUB_OUTPUT echo "build=$(date +'%Y.%-m')" >> $GITHUB_OUTPUT -# temporary debug - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - with: - sudo: false - limit-access-to-actor: false -# /temporary debug - - name: Insert metadata id: insert shell: bash From 837ec588a7a96d23b215a635dc37346639a3da81 Mon Sep 17 00:00:00 2001 From: Egor Vasilyev Date: Mon, 7 Aug 2023 10:51:15 +0300 Subject: [PATCH 08/12] add system monitoring; disable debug --- .github/workflows/collect-statistics.yml | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/.github/workflows/collect-statistics.yml b/.github/workflows/collect-statistics.yml index debfc08be2..29fd20a5a9 100644 --- a/.github/workflows/collect-statistics.yml +++ b/.github/workflows/collect-statistics.yml @@ -118,17 +118,26 @@ jobs: ref: ${{ env.data_branch }} path: ${{ env.data_path }} + - name: Run system monitoring + # secret uploaded using base64 encoding to have one-line output: + # cat file | base64 -w 0 + continue-on-error: true + run: | + chmod +x ./scripts/project/monitoring.sh + ./scripts/project/monitoring.sh "${PUSHGATEWAY_HOSTNAME}" "${{ secrets.PUSHGATEWAY_USER }}" "${{ secrets.PUSHGATEWAY_PASSWORD }}" + echo "Please visit Grafana to check metrics: https://${PUSHGATEWAY_HOSTNAME}/d/rYdddlPWk/node-exporter-full?orgId=1&from=now-1h&to=now&var-service=github&var-instance=${GITHUB_RUN_ID}-${HOSTNAME}&refresh=1m" + - uses: actions/setup-python@v4 with: python-version: '3.9' - # temporary debug - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - with: - sudo: false - limit-access-to-actor: false - # /temporary debug +# # temporary debug +# - name: Setup tmate session +# uses: mxschmitt/action-tmate@v3 +# with: +# sudo: false +# limit-access-to-actor: false +# # /temporary debug - name: Build and run monitoring UTBot Java run: | From a6a28a801d48b1274b845e6cf6432779455073ab Mon Sep 17 00:00:00 2001 From: Egor Vasilyev Date: Mon, 7 Aug 2023 11:39:08 +0300 Subject: [PATCH 09/12] add swap extending --- .github/workflows/collect-statistics.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/collect-statistics.yml b/.github/workflows/collect-statistics.yml index 29fd20a5a9..0e52309fcc 100644 --- a/.github/workflows/collect-statistics.yml +++ b/.github/workflows/collect-statistics.yml @@ -118,6 +118,22 @@ jobs: ref: ${{ env.data_branch }} path: ${{ env.data_path }} + - name: Expand system swap + shell: bash + run: | + docker run -ti \ + --privileged \ + --net=host \ + --pid=host \ + --ipc=host \ + --volume /:/host \ + busybox \ + chroot /host + swapoff /mnt/swapfile + dd if=/dev/zero of=/mnt/swapfile bs=1M count=1024 oflag=append conv=notrunc + mkswap /mnt/swapfile + swapon /mnt/swapfile + - name: Run system monitoring # secret uploaded using base64 encoding to have one-line output: # cat file | base64 -w 0 From 88286357bddad5ecd3c7a630c0c42833b85a4c90 Mon Sep 17 00:00:00 2001 From: Egor Vasilyev Date: Mon, 7 Aug 2023 11:44:41 +0300 Subject: [PATCH 10/12] return debug --- .github/workflows/collect-statistics.yml | 44 ++++++++++++------------ 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/.github/workflows/collect-statistics.yml b/.github/workflows/collect-statistics.yml index 0e52309fcc..240d8ab40f 100644 --- a/.github/workflows/collect-statistics.yml +++ b/.github/workflows/collect-statistics.yml @@ -118,21 +118,21 @@ jobs: ref: ${{ env.data_branch }} path: ${{ env.data_path }} - - name: Expand system swap - shell: bash - run: | - docker run -ti \ - --privileged \ - --net=host \ - --pid=host \ - --ipc=host \ - --volume /:/host \ - busybox \ - chroot /host - swapoff /mnt/swapfile - dd if=/dev/zero of=/mnt/swapfile bs=1M count=1024 oflag=append conv=notrunc - mkswap /mnt/swapfile - swapon /mnt/swapfile +# - name: Expand system swap +# shell: bash +# run: | +# docker run -ti \ +# --privileged \ +# --net=host \ +# --pid=host \ +# --ipc=host \ +# --volume /:/host \ +# busybox \ +# chroot /host +# swapoff /mnt/swapfile +# dd if=/dev/zero of=/mnt/swapfile bs=1M count=4096 oflag=append conv=notrunc +# mkswap /mnt/swapfile +# swapon /mnt/swapfile - name: Run system monitoring # secret uploaded using base64 encoding to have one-line output: @@ -147,13 +147,13 @@ jobs: with: python-version: '3.9' -# # temporary debug -# - name: Setup tmate session -# uses: mxschmitt/action-tmate@v3 -# with: -# sudo: false -# limit-access-to-actor: false -# # /temporary debug + # temporary debug + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + with: + sudo: false + limit-access-to-actor: false + # /temporary debug - name: Build and run monitoring UTBot Java run: | From 4297d5d38485c7843d7ee826a95775e343d58d58 Mon Sep 17 00:00:00 2001 From: Egor Vasilyev Date: Mon, 7 Aug 2023 15:00:39 +0300 Subject: [PATCH 11/12] Update collect-statistics.yml --- .github/workflows/collect-statistics.yml | 33 ++++++++++-------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/.github/workflows/collect-statistics.yml b/.github/workflows/collect-statistics.yml index 240d8ab40f..4f3ab21ec4 100644 --- a/.github/workflows/collect-statistics.yml +++ b/.github/workflows/collect-statistics.yml @@ -118,21 +118,14 @@ jobs: ref: ${{ env.data_branch }} path: ${{ env.data_path }} -# - name: Expand system swap -# shell: bash -# run: | -# docker run -ti \ -# --privileged \ -# --net=host \ -# --pid=host \ -# --ipc=host \ -# --volume /:/host \ -# busybox \ -# chroot /host -# swapoff /mnt/swapfile -# dd if=/dev/zero of=/mnt/swapfile bs=1M count=4096 oflag=append conv=notrunc -# mkswap /mnt/swapfile -# swapon /mnt/swapfile + - name: Expand system swap + shell: bash + run: | + docker run -d --rm --name busybox --privileged --net=host --pid=host --ipc=host --volume /:/host busybox sleep infinity + docker exec busybox /bin/sh -c 'chroot /host /bin/bash -c "swapoff /mnt/swapfile"' + docker exec busybox /bin/sh -c 'chroot /host /bin/bash -c "dd if=/dev/zero of=/mnt/swapfile bs=1M count=4096 oflag=append conv=notrunc"' + docker exec busybox /bin/sh -c 'chroot /host /bin/bash -c "mkswap /mnt/swapfile"' + docker exec busybox /bin/sh -c 'chroot /host /bin/bash -c "swapon /mnt/swapfile"' - name: Run system monitoring # secret uploaded using base64 encoding to have one-line output: @@ -148,11 +141,11 @@ jobs: python-version: '3.9' # temporary debug - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - with: - sudo: false - limit-access-to-actor: false +# - name: Setup tmate session +# uses: mxschmitt/action-tmate@v3 +# with: +# sudo: false +# limit-access-to-actor: false # /temporary debug - name: Build and run monitoring UTBot Java From f9e3bc4dc328f2edc9e291490db61d2dc10c1452 Mon Sep 17 00:00:00 2001 From: Egor Vasilyev Date: Tue, 8 Aug 2023 13:09:38 +0300 Subject: [PATCH 12/12] add comment; remove debug --- .github/workflows/collect-statistics.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/collect-statistics.yml b/.github/workflows/collect-statistics.yml index 4f3ab21ec4..c53f32f798 100644 --- a/.github/workflows/collect-statistics.yml +++ b/.github/workflows/collect-statistics.yml @@ -120,6 +120,7 @@ jobs: - name: Expand system swap shell: bash + # trying to configure swap on host from running container run: | docker run -d --rm --name busybox --privileged --net=host --pid=host --ipc=host --volume /:/host busybox sleep infinity docker exec busybox /bin/sh -c 'chroot /host /bin/bash -c "swapoff /mnt/swapfile"' @@ -140,14 +141,6 @@ jobs: with: python-version: '3.9' - # temporary debug -# - name: Setup tmate session -# uses: mxschmitt/action-tmate@v3 -# with: -# sudo: false -# limit-access-to-actor: false - # /temporary debug - - name: Build and run monitoring UTBot Java run: | gradle :utbot-junit-contest:monitoringJar