From 2c0212b40634a987ce81784c62a19e81de6f6091 Mon Sep 17 00:00:00 2001 From: Anton-Latukha Date: Sat, 11 Dec 2021 13:27:18 +0200 Subject: [PATCH 1/6] CI: workflows: disable stack where it is not used Hopefully it would remove it & its files from being cached in the action. & also not includde into environment. --- .github/workflows/build.yml | 1 + .github/workflows/caching.yml | 1 + .github/workflows/hackage.yml | 1 + .github/workflows/test.yml | 1 + 4 files changed, 4 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a51eab6058..1e30d37a60 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,6 +33,7 @@ jobs: with: ghc-version : ${{ matrix.ghc }} cabal-version: ${{ matrix.cabal }} + enable-stack: false # some alpines come with integer-simple instead of integer-gmp - name: Force integer-simple diff --git a/.github/workflows/caching.yml b/.github/workflows/caching.yml index 4ffe805af6..1fac16f6bd 100644 --- a/.github/workflows/caching.yml +++ b/.github/workflows/caching.yml @@ -69,6 +69,7 @@ jobs: with: ghc-version: ${{ matrix.ghc }} cabal-version: ${{ matrix.cabal }} + enable-stack: false - if: runner.os == 'Windows' name: (Windows) Platform config diff --git a/.github/workflows/hackage.yml b/.github/workflows/hackage.yml index f483c4c56f..968eb7cf0f 100644 --- a/.github/workflows/hackage.yml +++ b/.github/workflows/hackage.yml @@ -52,6 +52,7 @@ jobs: with: ghc-version : ${{ matrix.ghc }} cabal-version: ${{ matrix.cabal }} + enable-stack: false - name: Cache Cabal uses: actions/cache@v2 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a42991ef9d..0d16cb3fdc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -79,6 +79,7 @@ jobs: with: ghc-version : ${{ matrix.ghc }} cabal-version: ${{ matrix.cabal }} + enable-stack: false - if: runner.os == 'Windows' name: (Windows) Platform config From d0cd643ca87c6b92c77a0a313ab126af3dbd6c75 Mon Sep 17 00:00:00 2001 From: Anton-Latukha Date: Sat, 11 Dec 2021 13:31:15 +0200 Subject: [PATCH 2/6] CI: {bench,hackage} add Linux platform config For future caching configuraiton. --- .github/workflows/bench.yml | 4 ++++ .github/workflows/hackage.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 6507611c2d..8ec9093818 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -53,6 +53,10 @@ jobs: enable-stack: false - name: Cache Cabal + - name: Linux Platform config + run: | + echo "CABAL_PKGS_DIR=~/.cabal/packages" >> $GITHUB_ENV + uses: actions/cache@v2 with: path: | diff --git a/.github/workflows/hackage.yml b/.github/workflows/hackage.yml index 968eb7cf0f..7b3fb73ade 100644 --- a/.github/workflows/hackage.yml +++ b/.github/workflows/hackage.yml @@ -55,6 +55,10 @@ jobs: enable-stack: false - name: Cache Cabal + - name: Linux Platform config + run: | + echo "CABAL_PKGS_DIR=~/.cabal/packages" >> $GITHUB_ENV + uses: actions/cache@v2 env: cache-name: cache-cabal From ee64445300391af1c52685ad20b9dbb7c6c5a8a1 Mon Sep 17 00:00:00 2001 From: Anton-Latukha Date: Sat, 11 Dec 2021 13:33:15 +0200 Subject: [PATCH 3/6] CI: {caching,test,bench,hackage}: retrieving index timestamp For 1:1 according cache to it. --- .github/workflows/bench.yml | 8 ++++++++ .github/workflows/caching.yml | 8 ++++++++ .github/workflows/hackage.yml | 8 ++++++++ .github/workflows/test.yml | 8 ++++++++ 4 files changed, 32 insertions(+) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 8ec9093818..e1ec426533 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -57,6 +57,14 @@ jobs: run: | echo "CABAL_PKGS_DIR=~/.cabal/packages" >> $GITHUB_ENV + - name: Retrieving `cabal.project` Hackage timestamp + run: | + # Form: index-state: 2021-11-29T08:11:08Z + INDEX_STATE_ENTRY=$(grep index-state cabal.project) + # Form: 2021-11-29T08-11-08Z + INDEX_STATE1=$(echo "$INDEX_STATE_ENTRY" | cut -d' ' -f2 | tr ':' '-') + echo "INDEX_STATE=$INDEX_STATE1" >> $GITHUB_ENV + uses: actions/cache@v2 with: path: | diff --git a/.github/workflows/caching.yml b/.github/workflows/caching.yml index 1fac16f6bd..973ec132a6 100644 --- a/.github/workflows/caching.yml +++ b/.github/workflows/caching.yml @@ -94,6 +94,14 @@ jobs: echo " ghc-options: -O0" >> cabal.project - name: Cache Cabal + - name: Retrieving `cabal.project` Hackage timestamp + run: | + # Form: index-state: 2021-11-29T08:11:08Z + INDEX_STATE_ENTRY=$(grep index-state cabal.project) + # Form: 2021-11-29T08-11-08Z + INDEX_STATE1=$(echo "$INDEX_STATE_ENTRY" | cut -d' ' -f2 | tr ':' '-') + echo "INDEX_STATE=$INDEX_STATE1" >> $GITHUB_ENV + uses: actions/cache@v2 env: cache-name: cache-cabal diff --git a/.github/workflows/hackage.yml b/.github/workflows/hackage.yml index 7b3fb73ade..76a7b66258 100644 --- a/.github/workflows/hackage.yml +++ b/.github/workflows/hackage.yml @@ -59,6 +59,14 @@ jobs: run: | echo "CABAL_PKGS_DIR=~/.cabal/packages" >> $GITHUB_ENV + - name: Retrieving `cabal.project` Hackage timestamp + run: | + # Form: index-state: 2021-11-29T08:11:08Z + INDEX_STATE_ENTRY=$(grep index-state cabal.project) + # Form: 2021-11-29T08-11-08Z + INDEX_STATE1=$(echo "$INDEX_STATE_ENTRY" | cut -d' ' -f2 | tr ':' '-') + echo "INDEX_STATE=$INDEX_STATE1" >> $GITHUB_ENV + uses: actions/cache@v2 env: cache-name: cache-cabal diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0d16cb3fdc..78cd82e17c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -115,6 +115,14 @@ jobs: src/**/*.hs exe/*.hs - name: Cache Cabal + - name: Retrieving `cabal.project` Hackage timestamp + run: | + # Form: index-state: 2021-11-29T08:11:08Z + INDEX_STATE_ENTRY=$(grep index-state cabal.project) + # Form: 2021-11-29T08-11-08Z + INDEX_STATE1=$(echo "$INDEX_STATE_ENTRY" | cut -d' ' -f2 | tr ':' '-') + echo "INDEX_STATE=$INDEX_STATE1" >> $GITHUB_ENV + uses: actions/cache@v2 env: cache-name: cache-cabal From a4fce1f6056921b372166dade20c839502b9dc7f Mon Sep 17 00:00:00 2001 From: Anton-Latukha Date: Sat, 11 Dec 2021 13:36:48 +0200 Subject: [PATCH 4/6] CI: {caching,test,bench,hackage}: new caching process Sources are stored separately. Before this cache was storing per Platform/GHC sources & compiled results together. While deps sources does not depend on platform or GHC, they are platform & GHC agnostic. Because sources & compiled results were stored togather & sources took ~1/2-2/3 of that - cache stored multiple duplicates of the same source code. This config stores the source code separately. And at the same time tries to store & share all source code which may be needed. & all compiled deps that would be used in different workflows. --- .github/workflows/bench.yml | 26 ++++++++++++++++++-------- .github/workflows/caching.yml | 35 +++++++++++++++++++++++------------ .github/workflows/hackage.yml | 26 +++++++++++++++++--------- .github/workflows/test.yml | 28 +++++++++++++++++----------- 4 files changed, 75 insertions(+), 40 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index e1ec426533..e0e44098e2 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -47,12 +47,12 @@ jobs: - run: git fetch origin master # check the master branch for benchmarking - uses: haskell/actions/setup@v1 + id: HaskEnvSetup with: ghc-version : ${{ matrix.ghc }} cabal-version: ${{ matrix.cabal }} enable-stack: false - - name: Cache Cabal - name: Linux Platform config run: | echo "CABAL_PKGS_DIR=~/.cabal/packages" >> $GITHUB_ENV @@ -65,16 +65,26 @@ jobs: INDEX_STATE1=$(echo "$INDEX_STATE_ENTRY" | cut -d' ' -f2 | tr ':' '-') echo "INDEX_STATE=$INDEX_STATE1" >> $GITHUB_ENV + - name: Hackage sources cache uses: actions/cache@v2 + env: + cache-name: hackage-sources with: - path: | - ~/.cabal/packages - ~/.cabal/store - key: v2-${{ runner.os }}-${{ matrix.ghc }}-bench-${{ hashFiles('cabal.project') }} + path: ${{ env.CABAL_PKGS_DIR }} + key: ${{ env.cache-name }}-${{ env.INDEX_STATE }} + restore-keys: ${{ env.cache-name }}- + + - name: Compiled deps cache + uses: actions/cache@v2 + env: + cache-name: compiled-deps + with: + path: ${{ steps.HaskEnvSetup.outputs.cabal-store }} + key: ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}-${{ hashFiles('cabal.project') }} restore-keys: | - v2-${{ runner.os }}-${{ matrix.ghc }}-build-${{ hashFiles('cabal.project') }} - v2-${{ runner.os }}-${{ matrix.ghc }}-bench- - v2-${{ runner.os }}-${{ matrix.ghc }} + ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}- + ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}- + ${{ env.cache-name }}-${{ runner.os }}- - run: cabal update diff --git a/.github/workflows/caching.yml b/.github/workflows/caching.yml index 973ec132a6..ddb60a7f86 100644 --- a/.github/workflows/caching.yml +++ b/.github/workflows/caching.yml @@ -66,20 +66,19 @@ jobs: - uses: actions/checkout@v2 - uses: haskell/actions/setup@v1 + id: HaskEnvSetup with: - ghc-version: ${{ matrix.ghc }} + ghc-version : ${{ matrix.ghc }} cabal-version: ${{ matrix.cabal }} enable-stack: false - if: runner.os == 'Windows' name: (Windows) Platform config run: | - echo "CABAL_STORE_DIR=$SYSTEMDRIVE\\SR" >> $GITHUB_ENV echo "CABAL_PKGS_DIR=~\\AppData\\cabal\\packages" >> $GITHUB_ENV - if: ( runner.os == 'Linux' ) || ( runner.os == 'macOS' ) name: (Linux,macOS) Platform config run: | - echo "CABAL_STORE_DIR=~/.cabal/store" >> $GITHUB_ENV echo "CABAL_PKGS_DIR=~/.cabal/packages" >> $GITHUB_ENV # Needs to be before Cache Cabal so the cache can detect changes to the modified cabal.project file @@ -93,7 +92,6 @@ jobs: echo "package floskell" >> cabal.project echo " ghc-options: -O0" >> cabal.project - - name: Cache Cabal - name: Retrieving `cabal.project` Hackage timestamp run: | # Form: index-state: 2021-11-29T08:11:08Z @@ -102,18 +100,31 @@ jobs: INDEX_STATE1=$(echo "$INDEX_STATE_ENTRY" | cut -d' ' -f2 | tr ':' '-') echo "INDEX_STATE=$INDEX_STATE1" >> $GITHUB_ENV + # 2021-12-02: NOTE: Cabal Hackage source tree storage does not depend on OS or GHC really, + # but can depend on `base`. + # But this caching is happens only inside `master` for `master` purposes of compiling the deps + # so having a shared pool here that depends only on Hackage pin & does not depend on `base` is "good enough" + # & used such because it preserves 10% of a global cache storage pool. + - name: Hackage sources cache uses: actions/cache@v2 env: - cache-name: cache-cabal + cache-name: hackage-sources with: - path: | - ${{ env.CABAL_PKGS_DIR }} - ${{ env.CABAL_STORE_DIR }} - key: v2-${{ runner.os }}-${{ matrix.ghc }}-build-${{ hashFiles('cabal.project') }} + path: ${{ env.CABAL_PKGS_DIR }} + key: ${{ env.cache-name }}-${{ env.INDEX_STATE }} + restore-keys: ${{ env.cache-name }}- + + - name: Compiled deps cache + uses: actions/cache@v2 + env: + cache-name: compiled-deps + with: + path: ${{ steps.HaskEnvSetup.outputs.cabal-store }} + key: ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}-${{ hashFiles('cabal.project') }} restore-keys: | - v2-${{ runner.os }}-${{ matrix.ghc }}-bench-${{ hashFiles('cabal.project') }} - v2-${{ runner.os }}-${{ matrix.ghc }}-build- - v2-${{ runner.os }}-${{ matrix.ghc }} + ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}- + ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}- + ${{ env.cache-name }}-${{ runner.os }}- - run: cabal update diff --git a/.github/workflows/hackage.yml b/.github/workflows/hackage.yml index 76a7b66258..c5b8e1a890 100644 --- a/.github/workflows/hackage.yml +++ b/.github/workflows/hackage.yml @@ -49,12 +49,12 @@ jobs: - uses: actions/checkout@v2 - uses: haskell/actions/setup@v1 + id: HaskEnvSetup with: ghc-version : ${{ matrix.ghc }} cabal-version: ${{ matrix.cabal }} enable-stack: false - - name: Cache Cabal - name: Linux Platform config run: | echo "CABAL_PKGS_DIR=~/.cabal/packages" >> $GITHUB_ENV @@ -67,18 +67,26 @@ jobs: INDEX_STATE1=$(echo "$INDEX_STATE_ENTRY" | cut -d' ' -f2 | tr ':' '-') echo "INDEX_STATE=$INDEX_STATE1" >> $GITHUB_ENV + - name: Hackage sources cache uses: actions/cache@v2 env: - cache-name: cache-cabal + cache-name: hackage-sources with: - path: | - ~/.cabal/packages - ~/.cabal/store - key: v2-${{ runner.os }}-${{ matrix.ghc }}-build-${{ hashFiles('cabal.project') }} + path: ${{ env.CABAL_PKGS_DIR }} + key: ${{ env.cache-name }}-${{ env.INDEX_STATE }} + restore-keys: ${{ env.cache-name }}- + + - name: Compiled deps cache + uses: actions/cache@v2 + env: + cache-name: compiled-deps + with: + path: ${{ steps.HaskEnvSetup.outputs.cabal-store }} + key: ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}-${{ hashFiles('cabal.project') }} restore-keys: | - v2-${{ runner.os }}-${{ matrix.ghc }}-bench-${{ hashFiles('cabal.project') }} - v2-${{ runner.os }}-${{ matrix.ghc }}-build- - v2-${{ runner.os }}-${{ matrix.ghc }} + ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}- + ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}- + ${{ env.cache-name }}-${{ runner.os }}- - name: "Run cabal check" run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 78cd82e17c..173b3b44a6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -76,6 +76,7 @@ jobs: - uses: actions/checkout@v2 - uses: haskell/actions/setup@v1 + id: HaskEnvSetup with: ghc-version : ${{ matrix.ghc }} cabal-version: ${{ matrix.cabal }} @@ -84,12 +85,10 @@ jobs: - if: runner.os == 'Windows' name: (Windows) Platform config run: | - echo "CABAL_STORE_DIR=$SYSTEMDRIVE\\SR" >> $GITHUB_ENV echo "CABAL_PKGS_DIR=~\\AppData\\cabal\\packages" >> $GITHUB_ENV - if: ( runner.os == 'Linux' ) || ( runner.os == 'macOS' ) name: (Linux,macOS) Platform config run: | - echo "CABAL_STORE_DIR=~/.cabal/store" >> $GITHUB_ENV echo "CABAL_PKGS_DIR=~/.cabal/packages" >> $GITHUB_ENV # Needs to be before Cache Cabal so the cache can detect changes to the modified cabal.project file @@ -114,7 +113,6 @@ jobs: sed -i.bak -e 's/Paths_haskell_language_server/Paths_hls/g' \ src/**/*.hs exe/*.hs - - name: Cache Cabal - name: Retrieving `cabal.project` Hackage timestamp run: | # Form: index-state: 2021-11-29T08:11:08Z @@ -123,18 +121,26 @@ jobs: INDEX_STATE1=$(echo "$INDEX_STATE_ENTRY" | cut -d' ' -f2 | tr ':' '-') echo "INDEX_STATE=$INDEX_STATE1" >> $GITHUB_ENV + - name: Hackage sources cache uses: actions/cache@v2 env: - cache-name: cache-cabal + cache-name: hackage-sources with: - path: | - ${{ env.CABAL_PKGS_DIR }} - ${{ env.CABAL_STORE_DIR }} - key: v2-${{ runner.os }}-${{ matrix.ghc }}-build-${{ hashFiles('cabal.project') }} + path: ${{ env.CABAL_PKGS_DIR }} + key: ${{ env.cache-name }}-${{ env.INDEX_STATE }} + restore-keys: ${{ env.cache-name }}- + + - name: Compiled deps cache + uses: actions/cache@v2 + env: + cache-name: compiled-deps + with: + path: ${{ steps.HaskEnvSetup.outputs.cabal-store }} + key: ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}-${{ hashFiles('cabal.project') }} restore-keys: | - v2-${{ runner.os }}-${{ matrix.ghc }}-bench-${{ hashFiles('cabal.project') }} - v2-${{ runner.os }}-${{ matrix.ghc }}-build- - v2-${{ runner.os }}-${{ matrix.ghc }} + ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}- + ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}- + ${{ env.cache-name }}-${{ runner.os }}- - run: cabal update From 999c8727ca2607279e8a26ada2eb6636bc2f4283 Mon Sep 17 00:00:00 2001 From: Anton-Latukha Date: Sat, 11 Dec 2021 13:46:34 +0200 Subject: [PATCH 5/6] CI: caching: separate source retrieval step & separate build step --- .github/workflows/caching.yml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/caching.yml b/.github/workflows/caching.yml index ddb60a7f86..0e960354da 100644 --- a/.github/workflows/caching.yml +++ b/.github/workflows/caching.yml @@ -31,6 +31,9 @@ on: branches: - master +env: + cabalBuild: "v2-build all --enable-tests --enable-benchmarks" + jobs: pre_job: @@ -128,6 +131,16 @@ jobs: - run: cabal update + - name: Download all sources + run: | + cabal $cabalBuild --only-download + # repeating builds to workaround segfaults in windows and ghc-8.8.4 - - name: Build - run: cabal build --only-dependencies || cabal build --only-dependencies || cabal build --only-dependencies + # This build agenda in not to have successful code, + # but to cache what can be cached, so step is fault tolerant & would always succseed. + # 2021-12-11: NOTE: Building all targets, since + # current Cabal does not allow `all --enable-tests --enable-benchmarks --only-dependencies` + - name: Build all targets; try 3 times + continue-on-error: true + run: | + cabal $cabalBuild || cabal $cabalBuild || cabal $cabalBuild From 6483d96acb5d347b5386bfa8c22cf48e76d12339 Mon Sep 17 00:00:00 2001 From: Anton-Latukha Date: Sat, 11 Dec 2021 16:22:46 +0200 Subject: [PATCH 6/6] CI: {caching,bench}: add executable patching for cache --- .github/workflows/bench.yml | 9 +++++++++ .github/workflows/caching.yml | 12 ++++++++++++ .github/workflows/test.yml | 4 +--- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index e0e44098e2..dd73fc23c5 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -57,6 +57,15 @@ jobs: run: | echo "CABAL_PKGS_DIR=~/.cabal/packages" >> $GITHUB_ENV + # All workflows which distinquishes cache on `cabal.project` needs this. + - name: Workaround shorten binary names + run: | + sed -i.bak -e 's/haskell-language-server/hls/g' \ + -e 's/haskell_language_server/hls/g' \ + haskell-language-server.cabal cabal.project + sed -i.bak -e 's/Paths_haskell_language_server/Paths_hls/g' \ + src/**/*.hs exe/*.hs + - name: Retrieving `cabal.project` Hackage timestamp run: | # Form: index-state: 2021-11-29T08:11:08Z diff --git a/.github/workflows/caching.yml b/.github/workflows/caching.yml index 0e960354da..b1a57c1906 100644 --- a/.github/workflows/caching.yml +++ b/.github/workflows/caching.yml @@ -95,6 +95,18 @@ jobs: echo "package floskell" >> cabal.project echo " ghc-options: -O0" >> cabal.project + # Shorten binary names as a workaround for filepath length limits in Windows, + # but since tests are hardcoded on this workaround - + # all platforms (in 2021-12-07) need it. + # All workflows which distinquishes cache on `cabal.project` needs this. + - name: Workaround shorten binary names + run: | + sed -i.bak -e 's/haskell-language-server/hls/g' \ + -e 's/haskell_language_server/hls/g' \ + haskell-language-server.cabal cabal.project + sed -i.bak -e 's/Paths_haskell_language_server/Paths_hls/g' \ + src/**/*.hs exe/*.hs + - name: Retrieving `cabal.project` Hackage timestamp run: | # Form: index-state: 2021-11-29T08:11:08Z diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 173b3b44a6..43afbc7c63 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -102,9 +102,7 @@ jobs: echo "package floskell" >> cabal.project echo " ghc-options: -O0" >> cabal.project - # Shorten binary names as a workaround for filepath length limits in Windows, - # but since tests are hardcoded on this workaround - - # all platforms (in 2021-12-07) need it. + # All workflows which distinquishes cache on `cabal.project` needs this. - name: Workaround shorten binary names run: | sed -i.bak -e 's/haskell-language-server/hls/g' \