From 08bda1b0de4cc014b5c05551233f84e64e733e7a Mon Sep 17 00:00:00 2001 From: jneira Date: Mon, 3 Jan 2022 21:29:02 +0100 Subject: [PATCH 1/2] Fix ci update hackage index --- .github/workflows/bench.yml | 21 ++++++++++++--------- .github/workflows/caching.yml | 31 +++++++++++++++++-------------- .github/workflows/flags.yml | 21 ++++++++++++--------- .github/workflows/hackage.yml | 24 +++++++++++++++++++----- .github/workflows/test.yml | 21 ++++++++++++--------- 5 files changed, 72 insertions(+), 46 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 90d0d05ada..bebf529fae 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -85,7 +85,19 @@ jobs: INDEX_STATE1=$(echo "$INDEX_STATE_ENTRY" | cut -d' ' -f2 | tr ':' '-') echo "INDEX_STATE=$INDEX_STATE1" >> $GITHUB_ENV + # We have to restore package sources before `cabal update` + # cause it overwrites the hackage index with the cached one + - name: Hackage sources cache + uses: actions/cache@v2 + env: + cache-name: hackage-sources + with: + path: ${{ env.CABAL_PKGS_DIR }} + key: ${{ env.cache-name }}-${{ env.INDEX_STATE }} + restore-keys: ${{ env.cache-name }}- + # To ensure we get the lastest hackage index and not relying on haskell action logic + # It has to be done before `cabal freeze` to make it aware of the new index - run: cabal update - name: Form the package list ('cabal.project.freeze') @@ -100,15 +112,6 @@ jobs: echo 'WARNING: Could not produce the `freeze`.' echo ::set-output name=value::${{ hashFiles('cabal.project.freeze') }} - - name: Hackage sources cache - uses: actions/cache@v2 - env: - cache-name: hackage-sources - with: - path: ${{ env.CABAL_PKGS_DIR }} - key: ${{ env.cache-name }}-${{ env.INDEX_STATE }} - restore-keys: ${{ env.cache-name }}- - - name: Compiled deps cache id: compiled-deps uses: actions/cache@v2 diff --git a/.github/workflows/caching.yml b/.github/workflows/caching.yml index 789ec0ff46..0df746f83c 100644 --- a/.github/workflows/caching.yml +++ b/.github/workflows/caching.yml @@ -142,7 +142,24 @@ jobs: INDEX_STATE1=$(echo "$INDEX_STATE_ENTRY" | cut -d' ' -f2 | tr ':' '-') echo "INDEX_STATE=$INDEX_STATE1" >> $GITHUB_ENV + # 2021-12-02: NOTE: Cabal Hackage source tree storage does not depend on OS or GHC really, + # but can depend on `base`. + # But this caching is happens only inside `master` for `master` purposes of compiling the deps + # so having a shared pool here that depends only on Hackage pin & does not depend on `base` is "good enough" + # & used such because it preserves 10% of a global cache storage pool. + # We have to restore package sources before `cabal update` + # cause it overwrites the hackage index with the cached one + - name: Hackage sources cache + uses: actions/cache@v2 + env: + cache-name: hackage-sources + with: + path: ${{ env.CABAL_PKGS_DIR }} + key: ${{ env.cache-name }}-${{ env.INDEX_STATE }} + restore-keys: ${{ env.cache-name }}- + # To ensure we get the lastest hackage index and not relying on haskell action logic + # It has to be done before `cabal freeze` to make it aware of the new index - run: cabal update - name: Form the package list ('cabal.project.freeze') @@ -157,20 +174,6 @@ jobs: echo 'WARNING: Could not produce the `freeze`.' echo ::set-output name=value::${{ hashFiles('cabal.project.freeze') }} - # 2021-12-02: NOTE: Cabal Hackage source tree storage does not depend on OS or GHC really, - # but can depend on `base`. - # But this caching is happens only inside `master` for `master` purposes of compiling the deps - # so having a shared pool here that depends only on Hackage pin & does not depend on `base` is "good enough" - # & used such because it preserves 10% of a global cache storage pool. - - name: Hackage sources cache - uses: actions/cache@v2 - env: - cache-name: hackage-sources - with: - path: ${{ env.CABAL_PKGS_DIR }} - key: ${{ env.cache-name }}-${{ env.INDEX_STATE }} - restore-keys: ${{ env.cache-name }}- - - name: Compiled deps cache id: compiled-deps uses: actions/cache@v2 diff --git a/.github/workflows/flags.yml b/.github/workflows/flags.yml index 608d9ed544..4a5338e6f9 100644 --- a/.github/workflows/flags.yml +++ b/.github/workflows/flags.yml @@ -85,7 +85,19 @@ jobs: INDEX_STATE1=$(echo "$INDEX_STATE_ENTRY" | cut -d' ' -f2 | tr ':' '-') echo "INDEX_STATE=$INDEX_STATE1" >> $GITHUB_ENV + # We have to restore package sources before `cabal update` + # cause it overwrites the hackage index with the cached one + - name: Hackage sources cache + uses: actions/cache@v2 + env: + cache-name: hackage-sources + with: + path: ${{ env.CABAL_PKGS_DIR }} + key: ${{ env.cache-name }}-${{ env.INDEX_STATE }} + restore-keys: ${{ env.cache-name }}- + # To ensure we get the lastest hackage index and not relying on haskell action logic + # It has to be done before `cabal freeze` to make it aware of the new index - run: cabal update - name: Form the package list ('cabal.project.freeze') @@ -102,15 +114,6 @@ jobs: # Removing freeze file as it breaks builds with alternative flags rm -rf cabal.project.freeze - - name: Hackage sources cache - uses: actions/cache@v2 - env: - cache-name: hackage-sources - with: - path: ${{ env.CABAL_PKGS_DIR }} - key: ${{ env.cache-name }}-${{ env.INDEX_STATE }} - restore-keys: ${{ env.cache-name }}- - - name: Compiled deps cache id: compiled-deps uses: actions/cache@v2 diff --git a/.github/workflows/hackage.yml b/.github/workflows/hackage.yml index 7fa7714afd..f3c6bc7baf 100644 --- a/.github/workflows/hackage.yml +++ b/.github/workflows/hackage.yml @@ -78,16 +78,33 @@ jobs: cache-name: hackage-sources with: path: ${{ env.CABAL_PKGS_DIR }} - key: ${{ env.cache-name }}-${{ env.INDEX_STATE }} + key: ${{ env.cache-name }}-${{ env.INDEX_STATE }} restore-keys: ${{ env.cache-name }}- + # To ensure we get the lastest hackage index and not relying on haskell action logic + # It has to be done before `cabal freeze` to make it aware of the new index + - name: "Ensure we will use hackage head" + run: cabal update + + - name: Compute the cache key + id: compute-cache-key + run: | + cabal v2-freeze && \ + echo "" && \ + echo 'Output:' && \ + echo "" && \ + cat 'cabal.project.freeze' && \ + echo '' || \ + echo 'WARNING: Could not produce the `freeze`.' + echo ::set-output name=value::${{ hashFiles('cabal.project.freeze') }} + - name: Compiled deps cache uses: actions/cache@v2 env: cache-name: compiled-deps with: path: ${{ steps.HaskEnvSetup.outputs.cabal-store }} - key: ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}-${{ hashFiles('cabal.project') }} + key: ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}-${{ steps.compute-cache-key.outputs.value }} restore-keys: | ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}- ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}- @@ -116,9 +133,6 @@ jobs: - name: "Unpack package source in an isolated location" run: cabal unpack ${{ steps.generate-dist-tarball.outputs.path }} --destdir=./incoming - - name: "Ensure we will use hackage head" - run: cabal update - - name: "Try to get the current hackage version" id: get-hackage-version run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7ed443d518..0ec683f03c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -146,7 +146,19 @@ jobs: INDEX_STATE1=$(echo "$INDEX_STATE_ENTRY" | cut -d' ' -f2 | tr ':' '-') echo "INDEX_STATE=$INDEX_STATE1" >> $GITHUB_ENV + # We have to restore package sources before `cabal update` + # cause it overwrites the hackage index with the cached one + - name: Hackage sources cache + uses: actions/cache@v2 + env: + cache-name: hackage-sources + with: + path: ${{ env.CABAL_PKGS_DIR }} + key: ${{ env.cache-name }}-${{ env.INDEX_STATE }} + restore-keys: ${{ env.cache-name }}- + # To ensure we get the lastest hackage index and not relying on haskell action logic + # It has to be done before `cabal freeze` to make it aware of the new index - run: cabal update - name: Compute the cache key @@ -161,15 +173,6 @@ jobs: echo 'WARNING: Could not produce the `freeze`.' echo ::set-output name=value::${{ hashFiles('cabal.project.freeze') }} - - name: Hackage sources cache - uses: actions/cache@v2 - env: - cache-name: hackage-sources - with: - path: ${{ env.CABAL_PKGS_DIR }} - key: ${{ env.cache-name }}-${{ env.INDEX_STATE }} - restore-keys: ${{ env.cache-name }}- - - name: Compiled deps cache id: compiled-deps uses: actions/cache@v2 From 873c0fa47423f8f10b8b161b71f65b2aa4f50446 Mon Sep 17 00:00:00 2001 From: jneira Date: Mon, 3 Jan 2022 23:10:04 +0100 Subject: [PATCH 2/2] Make effective freeze file hash --- .github/workflows/bench.yml | 4 +--- .github/workflows/caching.yml | 4 +--- .github/workflows/flags.yml | 12 ++++++------ .github/workflows/hackage.yml | 7 +++---- .github/workflows/test.yml | 6 ++---- 5 files changed, 13 insertions(+), 20 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index bebf529fae..37cb975a9b 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -101,7 +101,6 @@ jobs: - run: cabal update - name: Form the package list ('cabal.project.freeze') - id: compute-cache-key run: | cabal v2-freeze && \ echo "" && \ @@ -110,7 +109,6 @@ jobs: cat 'cabal.project.freeze' && \ echo '' || \ echo 'WARNING: Could not produce the `freeze`.' - echo ::set-output name=value::${{ hashFiles('cabal.project.freeze') }} - name: Compiled deps cache id: compiled-deps @@ -119,7 +117,7 @@ jobs: cache-name: compiled-deps with: path: ${{ steps.HaskEnvSetup.outputs.cabal-store }} - key: ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}-${{ steps.compute-cache-key.outputs.value }} + key: ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}-${{ hashFiles('cabal.project.freeze') }} restore-keys: | ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}- ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}- diff --git a/.github/workflows/caching.yml b/.github/workflows/caching.yml index 0df746f83c..6e9a7306c7 100644 --- a/.github/workflows/caching.yml +++ b/.github/workflows/caching.yml @@ -163,7 +163,6 @@ jobs: - run: cabal update - name: Form the package list ('cabal.project.freeze') - id: compute-cache-key run: | cabal v2-freeze && \ echo "" && \ @@ -172,7 +171,6 @@ jobs: cat 'cabal.project.freeze' && \ echo '' || \ echo 'WARNING: Could not produce the `freeze`.' - echo ::set-output name=value::${{ hashFiles('cabal.project.freeze') }} - name: Compiled deps cache id: compiled-deps @@ -181,7 +179,7 @@ jobs: cache-name: compiled-deps with: path: ${{ steps.HaskEnvSetup.outputs.cabal-store }} - key: ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}-${{ steps.compute-cache-key.outputs.value }} + key: ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}-${{ hashFiles('cabal.project.freeze') }} restore-keys: | ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}- ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}- diff --git a/.github/workflows/flags.yml b/.github/workflows/flags.yml index 4a5338e6f9..ac95aeac80 100644 --- a/.github/workflows/flags.yml +++ b/.github/workflows/flags.yml @@ -101,18 +101,14 @@ jobs: - run: cabal update - name: Form the package list ('cabal.project.freeze') - id: compute-cache-key run: | cabal v2-freeze && \ echo "" && \ echo 'Output:' && \ echo "" && \ cat 'cabal.project.freeze' && \ - echo "" || \ + echo '' || \ echo 'WARNING: Could not produce the `freeze`.' - echo ::set-output name=value::${{ hashFiles('cabal.project.freeze') }} - # Removing freeze file as it breaks builds with alternative flags - rm -rf cabal.project.freeze - name: Compiled deps cache id: compiled-deps @@ -121,12 +117,16 @@ jobs: cache-name: compiled-deps with: path: ${{ steps.HaskEnvSetup.outputs.cabal-store }} - key: ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}-${{ steps.compute-cache-key.outputs.value }} + key: ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}-${{ hashFiles('cabal.project.freeze') }} restore-keys: | ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}- ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}- ${{ env.cache-name }}-${{ runner.os }}- + # Removing freeze file cause it breaks builds with alternative flags + - name: Remove freeze file + run: rm -f cabal.project.freeze + - name: Build `hls-graph` with flags run: cabal v2-build hls-graph --flags="pedantic embed-files stm-stats" diff --git a/.github/workflows/hackage.yml b/.github/workflows/hackage.yml index f3c6bc7baf..0c16b40cfd 100644 --- a/.github/workflows/hackage.yml +++ b/.github/workflows/hackage.yml @@ -86,8 +86,7 @@ jobs: - name: "Ensure we will use hackage head" run: cabal update - - name: Compute the cache key - id: compute-cache-key + - name: Form the package list ('cabal.project.freeze') run: | cabal v2-freeze && \ echo "" && \ @@ -96,15 +95,15 @@ jobs: cat 'cabal.project.freeze' && \ echo '' || \ echo 'WARNING: Could not produce the `freeze`.' - echo ::set-output name=value::${{ hashFiles('cabal.project.freeze') }} - name: Compiled deps cache + id: compiled-deps uses: actions/cache@v2 env: cache-name: compiled-deps with: path: ${{ steps.HaskEnvSetup.outputs.cabal-store }} - key: ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}-${{ steps.compute-cache-key.outputs.value }} + key: ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}-${{ hashFiles('cabal.project.freeze') }} restore-keys: | ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}- ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}- diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0ec683f03c..f83653a0be 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -161,8 +161,7 @@ jobs: # It has to be done before `cabal freeze` to make it aware of the new index - run: cabal update - - name: Compute the cache key - id: compute-cache-key + - name: Form the package list ('cabal.project.freeze') run: | cabal v2-freeze && \ echo "" && \ @@ -171,7 +170,6 @@ jobs: cat 'cabal.project.freeze' && \ echo '' || \ echo 'WARNING: Could not produce the `freeze`.' - echo ::set-output name=value::${{ hashFiles('cabal.project.freeze') }} - name: Compiled deps cache id: compiled-deps @@ -180,7 +178,7 @@ jobs: cache-name: compiled-deps with: path: ${{ steps.HaskEnvSetup.outputs.cabal-store }} - key: ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}-${{ steps.compute-cache-key.outputs.value }} + key: ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}-${{ hashFiles('cabal.project.freeze') }} restore-keys: | ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}- ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-