From 07b069f6c3511d01fa7ec100f17db9a9cf0917c3 Mon Sep 17 00:00:00 2001 From: Fendor Date: Sun, 7 May 2023 19:10:40 +0200 Subject: [PATCH 1/5] CI: don't rerun failed testsuites --- .github/workflows/test.yml | 60 +++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 247795d811..dd7a93540a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -118,14 +118,14 @@ jobs: - if: matrix.test name: Test hls-plugin-api - run: cabal test hls-plugin-api --test-options="$TEST_OPTS" || cabal test hls-plugin-api --test-options="$TEST_OPTS" + run: cabal test hls-plugin-api --test-options="$TEST_OPTS" - if: matrix.test name: Test func-test suite env: HLS_TEST_EXE: hls HLS_WRAPPER_TEST_EXE: hls-wrapper - run: cabal test func-test --test-options="$TEST_OPTS" || cabal test func-test --test-options="$TEST_OPTS" + run: cabal test func-test --test-options="$TEST_OPTS" - if: matrix.test name: Test wrapper-test suite @@ -136,116 +136,116 @@ jobs: - if: matrix.test name: Test hls-refactor-plugin - run: cabal test hls-refactor-plugin --test-options="$TEST_OPTS" || cabal test hls-refactor-plugin --test-options="$TEST_OPTS" + run: cabal test hls-refactor-plugin --test-options="$TEST_OPTS" - if: matrix.test && matrix.ghc != '9.6.1' name: Test hls-floskell-plugin - run: cabal test hls-floskell-plugin --test-options="$TEST_OPTS" || cabal test hls-floskell-plugin --test-options="$TEST_OPTS" + run: cabal test hls-floskell-plugin --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-class-plugin - run: cabal test hls-class-plugin --test-options="$TEST_OPTS" || cabal test hls-class-plugin --test-options="$TEST_OPTS" + run: cabal test hls-class-plugin --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-pragmas-plugin - run: cabal test hls-pragmas-plugin --test-options="$TEST_OPTS" || cabal test hls-pragmas-plugin --test-options="$TEST_OPTS" + run: cabal test hls-pragmas-plugin --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-eval-plugin - run: cabal test hls-eval-plugin --test-options="$TEST_OPTS" || cabal test hls-eval-plugin --test-options="$TEST_OPTS" + run: cabal test hls-eval-plugin --test-options="$TEST_OPTS" - if: matrix.test && matrix.ghc != '9.2.7' && matrix.ghc != '9.4.4' && matrix.ghc != '9.6.1' name: Test hls-haddock-comments-plugin - run: cabal test hls-haddock-comments-plugin --test-options="$TEST_OPTS" || cabal test hls-haddock-comments-plugin --test-options="$TEST_OPTS" + run: cabal test hls-haddock-comments-plugin --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-splice-plugin - run: cabal test hls-splice-plugin --test-options="$TEST_OPTS" || cabal test hls-splice-plugin --test-options="$TEST_OPTS" + run: cabal test hls-splice-plugin --test-options="$TEST_OPTS" - if: matrix.test && matrix.ghc != '9.6.1' name: Test hls-stylish-haskell-plugin - run: cabal test hls-stylish-haskell-plugin --test-options="$TEST_OPTS" || cabal test hls-stylish-haskell-plugin --test-options="$TEST_OPTS" + run: cabal test hls-stylish-haskell-plugin --test-options="$TEST_OPTS" - if: matrix.test && matrix.ghc != '9.6.1' name: Test hls-ormolu-plugin - run: cabal test hls-ormolu-plugin --test-options="$TEST_OPTS" || cabal test hls-ormolu-plugin --test-options="$TEST_OPTS" + run: cabal test hls-ormolu-plugin --test-options="$TEST_OPTS" - if: matrix.test && matrix.ghc != '9.6.1' name: Test hls-fourmolu-plugin - run: cabal test hls-fourmolu-plugin --test-options="$TEST_OPTS" || cabal test hls-fourmolu-plugin --test-options="$TEST_OPTS" + run: cabal test hls-fourmolu-plugin --test-options="$TEST_OPTS" - if: matrix.test && matrix.ghc != '9.2.7' && matrix.ghc != '9.4.4' && matrix.ghc != '9.6.1' name: Test hls-tactics-plugin test suite - run: cabal test hls-tactics-plugin --test-options="$TEST_OPTS" || cabal test hls-tactics-plugin --test-options="$TEST_OPTS" + run: cabal test hls-tactics-plugin --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-refine-imports-plugin test suite - run: cabal test hls-refine-imports-plugin --test-options="$TEST_OPTS" || cabal test hls-refine-imports-plugin --test-options="$TEST_OPTS" + run: cabal test hls-refine-imports-plugin --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-explicit-imports-plugin test suite - run: cabal test hls-explicit-imports-plugin --test-options="$TEST_OPTS" || cabal test hls-explicit-imports-plugin --test-options="$TEST_OPTS" + run: cabal test hls-explicit-imports-plugin --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-call-hierarchy-plugin test suite - run: cabal test hls-call-hierarchy-plugin --test-options="$TEST_OPTS" || cabal test hls-call-hierarchy-plugin --test-options="$TEST_OPTS" + run: cabal test hls-call-hierarchy-plugin --test-options="$TEST_OPTS" - if: matrix.test && matrix.os != 'windows-latest' name: Test hls-rename-plugin test suite - run: cabal test hls-rename-plugin --test-options="$TEST_OPTS" || cabal test hls-rename-plugin --test-options="$TEST_OPTS" + run: cabal test hls-rename-plugin --test-options="$TEST_OPTS" - if: matrix.test && matrix.ghc != '9.6.1' name: Test hls-hlint-plugin test suite - run: cabal test hls-hlint-plugin --test-options="$TEST_OPTS" || cabal test hls-hlint-plugin --test-options="$TEST_OPTS" + run: cabal test hls-hlint-plugin --test-options="$TEST_OPTS" - if: matrix.test && matrix.ghc != '9.0.2' && matrix.ghc != '9.2.7' && matrix.ghc != '9.4.4' && matrix.ghc != '9.6.1' name: Test hls-stan-plugin test suite - run: cabal test hls-stan-plugin --test-options="$TEST_OPTS" || cabal test hls-stan-plugin --test-options="$TEST_OPTS" + run: cabal test hls-stan-plugin --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-module-name-plugin test suite - run: cabal test hls-module-name-plugin --test-options="$TEST_OPTS" || cabal test hls-module-name-plugin --test-options="$TEST_OPTS" + run: cabal test hls-module-name-plugin --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-alternate-number-format-plugin test suite - run: cabal test hls-alternate-number-format-plugin --test-options="$TEST_OPTS" || cabal test hls-alternate-number-format-plugin --test-options="$TEST_OPTS" + run: cabal test hls-alternate-number-format-plugin --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-qualify-imported-names-plugin test suite - run: cabal test hls-qualify-imported-names-plugin --test-options="$TEST_OPTS" || cabal test hls-qualify-imported-names-plugin --test-options="$TEST_OPTS" + run: cabal test hls-qualify-imported-names-plugin --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-code-range-plugin test suite - run: cabal test hls-code-range-plugin --test-options="$TEST_OPTS" || cabal test hls-code-range-plugin --test-options="$TEST_OPTS" + run: cabal test hls-code-range-plugin --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-change-type-signature test suite - run: cabal test hls-change-type-signature-plugin --test-options="$TEST_OPTS" || cabal test hls-change-type-signature-plugin --test-options="$TEST_OPTS" + run: cabal test hls-change-type-signature-plugin --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-gadt-plugin test suit - run: cabal test hls-gadt-plugin --test-options="$TEST_OPTS" || cabal test hls-gadt-plugin --test-options="$TEST_OPTS" + run: cabal test hls-gadt-plugin --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-explicit-fixity-plugin test suite - run: cabal test hls-explicit-fixity-plugin --test-options="$TEST_OPTS" || cabal test hls-explicit-fixity-plugin --test-options="$TEST_OPTS" + run: cabal test hls-explicit-fixity-plugin --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-explicit-record-fields-plugin test suite - run: cabal test hls-explicit-record-fields-plugin --test-options="$TEST_OPTS" || cabal test hls-explicit-record-fields-plugin --test-options="$TEST_OPTS" + run: cabal test hls-explicit-record-fields-plugin --test-options="$TEST_OPTS" ## version needs to be limited since the tests depend on cabal-fmt which only builds using specific ghc versions - if: matrix.test && matrix.ghc == '8.10.7' name: Test hls-cabal-fmt-plugin test suite - run: cabal test hls-cabal-fmt-plugin --flag=isolateTests --test-options="$TEST_OPTS" || cabal test hls-cabal-fmt-plugin --flag=isolateTests --test-options="$TEST_OPTS" + run: cabal test hls-cabal-fmt-plugin --flag=isolateTests --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-cabal-plugin test suite - run: cabal test hls-cabal-plugin --test-options="$TEST_OPTS" || cabal test hls-cabal-plugin --test-options="$TEST_OPTS" + run: cabal test hls-cabal-plugin --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-retrie-plugin test suite - run: cabal test hls-retrie-plugin --test-options="$TEST_OPTS" || cabal test hls-retrie-plugin --test-options="$TEST_OPTS" + run: cabal test hls-retrie-plugin --test-options="$TEST_OPTS" test_post_job: if: always() From b00dc1ebaae869b6cae7a4d48ea3291981a583c0 Mon Sep 17 00:00:00 2001 From: Fendor Date: Sun, 7 May 2023 19:15:05 +0200 Subject: [PATCH 2/5] Introduce test-utils for marking tests as flaky --- ghcide/src/Development/IDE/GHC/Compat.hs | 2 +- ghcide/test/exe/Main.hs | 38 ++++++++++++++++++++---- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/ghcide/src/Development/IDE/GHC/Compat.hs b/ghcide/src/Development/IDE/GHC/Compat.hs index 0f9069b006..8f28d8ca72 100644 --- a/ghcide/src/Development/IDE/GHC/Compat.hs +++ b/ghcide/src/Development/IDE/GHC/Compat.hs @@ -616,7 +616,7 @@ data GhcVersion | GHC92 | GHC94 | GHC96 - deriving (Eq, Ord, Show) + deriving (Eq, Ord, Show, Enum, Bounded) ghcVersionStr :: String ghcVersionStr = VERSION_ghc diff --git a/ghcide/test/exe/Main.hs b/ghcide/test/exe/Main.hs index c690c0b9bd..38f65fffc5 100644 --- a/ghcide/test/exe/Main.hs +++ b/ghcide/test/exe/Main.hs @@ -138,8 +138,7 @@ import Development.IDE.Types.Logger (Logger (Logger), WithPriority (WithPriority, priority), cfilter, cmapWithPrio, - makeDefaultStderrRecorder, - toCologActionWithPrio) + makeDefaultStderrRecorder) import qualified FuzzySearch import GHC.Stack (emptyCallStack) import qualified HieDbRetry @@ -151,12 +150,11 @@ import Language.LSP.Types.Lens (didChangeWatchedFiles import qualified Language.LSP.Types.Lens as L import qualified Progress import System.Time.Extra -import qualified Test.QuickCheck.Monadic as MonadicQuickCheck -import Test.QuickCheck.Monadic (forAllM, monadicIO) import Test.Tasty import Test.Tasty.ExpectedFailure import Test.Tasty.HUnit import Test.Tasty.Ingredients.Rerun +import Test.Tasty.Runners as Runners import Test.Tasty.QuickCheck import Text.Printf (printf) import Text.Regex.TDFA ((=~)) @@ -2369,7 +2367,7 @@ knownBrokenForGhcVersions ghcVers = knownBrokenFor (BrokenForGHC ghcVers) data BrokenOS = Linux | MacOS | Windows deriving (Show) -data IssueSolution = Broken | Ignore deriving (Show) +data IssueSolution = Broken | Ignore | Flaky deriving (Show) data BrokenTarget = BrokenSpecific BrokenOS [GhcVersion] @@ -2388,6 +2386,14 @@ ignoreFor = knownIssueFor Ignore knownBrokenFor :: BrokenTarget -> String -> TestTree -> TestTree knownBrokenFor = knownIssueFor Broken +-- | Mark test as flaky for a specific target +knownFlakyFor :: BrokenTarget -> String -> TestTree -> TestTree +knownFlakyFor = knownIssueFor Flaky + +-- | Mark test as flaky for all GHC versions +knownFlaky :: String -> TestTree -> TestTree +knownFlaky = knownIssueFor Flaky (BrokenForGHC [minBound .. maxBound]) + -- | Deal with `IssueSolution` for specific OS and GHC. knownIssueFor :: IssueSolution -> BrokenTarget -> String -> TestTree -> TestTree knownIssueFor solution = go . \case @@ -2405,8 +2411,30 @@ knownIssueFor solution = go . \case go True = case solution of Broken -> expectFailBecause Ignore -> ignoreTestBecause + Flaky -> knownFlakyBecause go False = \_ -> id +-- | The test can no longer fail and is marked as flaky. +-- Flaky tests are tests that sometimes fail but sometimes also succeed. +-- Ideally, such test cases should not exist at all, but at least in this codebase, +-- the reality is, that we do have a number of flaky test cases. +-- +-- The code is basically copy-pasted from 'tasty-expected-failure' and if this +-- function proves useful in practice, we will upstream it. +knownFlakyBecause :: String -> TestTree -> TestTree +knownFlakyBecause info = wrapTest (fmap change) + where + change r + | resultSuccessful r + = r { resultDescription = resultDescription r <> " (flaky testcase: " <> info <> ")" + , resultShortDescription = resultShortDescription r <> " (flaky testcase)" + } + | otherwise + = r { resultOutcome = Runners.Success + , resultDescription = resultDescription r <> " (failed flaky testcase failed: " <> info <> ")" + , resultShortDescription = resultShortDescription r <> " (failed flaky testcase)" + } + data Expect = ExpectRange Range -- Both gotoDef and hover should report this range | ExpectLocation Location From 418171c11f69f4adc9280750750216d73592f8a6 Mon Sep 17 00:00:00 2001 From: Fendor Date: Sun, 7 May 2023 19:15:36 +0200 Subject: [PATCH 3/5] Mark "session-deps-are-picked-up" as flaky --- ghcide/test/exe/Main.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ghcide/test/exe/Main.hs b/ghcide/test/exe/Main.hs index 38f65fffc5..7d7de401ba 100644 --- a/ghcide/test/exe/Main.hs +++ b/ghcide/test/exe/Main.hs @@ -2895,9 +2895,9 @@ ifaceErrorTest3 = testCase "iface-error-test-3" $ runWithExtraFiles "recomp" $ \ expectNoMoreDiagnostics 2 sessionDepsArePickedUp :: TestTree -sessionDepsArePickedUp = testSession' - "session-deps-are-picked-up" - $ \dir -> do +sessionDepsArePickedUp = knownFlaky + "Regularly timeouts, likely due to a race in reloading session changes when hie.yaml changes." + $ testSession' "session-deps-are-picked-up" $ \dir -> do liftIO $ writeFileUTF8 (dir "hie.yaml") From 453efa24843f3d0430b01a03abacf4283d23cf29 Mon Sep 17 00:00:00 2001 From: Fendor Date: Mon, 8 May 2023 19:57:53 +0200 Subject: [PATCH 4/5] Add flaky test infrastructure to hls-test-utils --- hls-test-utils/src/Test/Hls/Util.hs | 57 ++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/hls-test-utils/src/Test/Hls/Util.hs b/hls-test-utils/src/Test/Hls/Util.hs index e654ee9660..2bc2756703 100644 --- a/hls-test-utils/src/Test/Hls/Util.hs +++ b/hls-test-utils/src/Test/Hls/Util.hs @@ -21,6 +21,8 @@ module Test.Hls.Util , knownBrokenForGhcVersions , knownBrokenInEnv , onlyWorkForGhcVersions + , knownFlakyInEnv + , knownFlaky -- * Extract code actions , fromAction , fromCommand @@ -71,9 +73,11 @@ import System.IO.Temp import System.Time.Extra (Seconds, sleep) import Test.Tasty (TestTree) import Test.Tasty.ExpectedFailure (expectFailBecause, - ignoreTestBecause) + ignoreTestBecause, wrapTest) import Test.Tasty.HUnit (Assertion, assertFailure, (@?=)) +import Test.Tasty.Runners (Result(..), resultSuccessful) +import qualified Test.Tasty.Runners as Runners noLiteralCaps :: C.ClientCapabilities noLiteralCaps = def & textDocument ?~ textDocumentCaps @@ -143,6 +147,57 @@ onlyRunForGhcVersions vers = then const id else ignoreTestBecause +-- | Mark test as flaky for a specific target. +-- This runs the test, but its result does not matter for the test suite. +-- +-- We define a test as flaky if it is fundamentally correct, but +-- fails occasionally due to lsp-test shenanigans. In particular, when the +-- test times out *sometimes*, but not always. +-- +-- A flaky test is a bug, which we are only not fixing right away +-- because it might be difficult to fix with lsp-test. +-- If your test isn't using lsp-test, then using this function +-- is not permitted. +knownFlakyInEnv :: [EnvSpec] -> String -> TestTree -> TestTree +knownFlakyInEnv envSpecs reason + | any matchesCurrentEnv envSpecs = knownFlakyBecause reason + | otherwise = id + +-- | Mark test as flaky for all supported GHC versions. +-- This runs the test, but its result does not matter for the test suite. +-- +-- We define a test as flaky if it is fundamentally correct, but +-- fails occasionally due to lsp-test shenanigans. In particular, when the +-- test times out *sometimes*, but not always. +-- +-- A flaky test is a bug, which we are only not fixing right away +-- because it might be difficult to fix with lsp-test. +-- If your test isn't using lsp-test, then using this function +-- is not permitted. +knownFlaky :: String -> TestTree -> TestTree +knownFlaky = knownFlakyInEnv (map GhcVer [minBound .. maxBound]) + +-- | The test can no longer fail and is marked as flaky. +-- Flaky tests are tests that sometimes fail but sometimes also succeed. +-- Ideally, such test cases should not exist at all, but at least in this codebase, +-- the reality is, that we do have a number of flaky test cases. +-- +-- The code is basically copy-pasted from 'tasty-expected-failure' and if this +-- function proves useful in practice, we will upstream it. +knownFlakyBecause :: String -> TestTree -> TestTree +knownFlakyBecause info = wrapTest (fmap change) + where + change r + | resultSuccessful r + = r { resultDescription = resultDescription r <> " (flaky testcase: " <> info <> ")" + , resultShortDescription = resultShortDescription r <> " (flaky testcase)" + } + | otherwise + = r { resultOutcome = Runners.Success + , resultDescription = resultDescription r <> " (failed flaky testcase failed: " <> info <> ")" + , resultShortDescription = resultShortDescription r <> " (failed flaky testcase)" + } + -- --------------------------------------------------------------------- -- | Like 'withCurrentDirectory', but will copy the directory over to the system From 85595688cadd5b96ce664c87d3e2b79497b2ae1d Mon Sep 17 00:00:00 2001 From: Fendor Date: Mon, 8 May 2023 20:49:32 +0200 Subject: [PATCH 5/5] Revert "CI: don't rerun failed testsuites" This reverts commit 729af3176ff11c45cc3c058165e829c67e828889. --- .github/workflows/test.yml | 60 +++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dd7a93540a..247795d811 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -118,14 +118,14 @@ jobs: - if: matrix.test name: Test hls-plugin-api - run: cabal test hls-plugin-api --test-options="$TEST_OPTS" + run: cabal test hls-plugin-api --test-options="$TEST_OPTS" || cabal test hls-plugin-api --test-options="$TEST_OPTS" - if: matrix.test name: Test func-test suite env: HLS_TEST_EXE: hls HLS_WRAPPER_TEST_EXE: hls-wrapper - run: cabal test func-test --test-options="$TEST_OPTS" + run: cabal test func-test --test-options="$TEST_OPTS" || cabal test func-test --test-options="$TEST_OPTS" - if: matrix.test name: Test wrapper-test suite @@ -136,116 +136,116 @@ jobs: - if: matrix.test name: Test hls-refactor-plugin - run: cabal test hls-refactor-plugin --test-options="$TEST_OPTS" + run: cabal test hls-refactor-plugin --test-options="$TEST_OPTS" || cabal test hls-refactor-plugin --test-options="$TEST_OPTS" - if: matrix.test && matrix.ghc != '9.6.1' name: Test hls-floskell-plugin - run: cabal test hls-floskell-plugin --test-options="$TEST_OPTS" + run: cabal test hls-floskell-plugin --test-options="$TEST_OPTS" || cabal test hls-floskell-plugin --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-class-plugin - run: cabal test hls-class-plugin --test-options="$TEST_OPTS" + run: cabal test hls-class-plugin --test-options="$TEST_OPTS" || cabal test hls-class-plugin --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-pragmas-plugin - run: cabal test hls-pragmas-plugin --test-options="$TEST_OPTS" + run: cabal test hls-pragmas-plugin --test-options="$TEST_OPTS" || cabal test hls-pragmas-plugin --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-eval-plugin - run: cabal test hls-eval-plugin --test-options="$TEST_OPTS" + run: cabal test hls-eval-plugin --test-options="$TEST_OPTS" || cabal test hls-eval-plugin --test-options="$TEST_OPTS" - if: matrix.test && matrix.ghc != '9.2.7' && matrix.ghc != '9.4.4' && matrix.ghc != '9.6.1' name: Test hls-haddock-comments-plugin - run: cabal test hls-haddock-comments-plugin --test-options="$TEST_OPTS" + run: cabal test hls-haddock-comments-plugin --test-options="$TEST_OPTS" || cabal test hls-haddock-comments-plugin --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-splice-plugin - run: cabal test hls-splice-plugin --test-options="$TEST_OPTS" + run: cabal test hls-splice-plugin --test-options="$TEST_OPTS" || cabal test hls-splice-plugin --test-options="$TEST_OPTS" - if: matrix.test && matrix.ghc != '9.6.1' name: Test hls-stylish-haskell-plugin - run: cabal test hls-stylish-haskell-plugin --test-options="$TEST_OPTS" + run: cabal test hls-stylish-haskell-plugin --test-options="$TEST_OPTS" || cabal test hls-stylish-haskell-plugin --test-options="$TEST_OPTS" - if: matrix.test && matrix.ghc != '9.6.1' name: Test hls-ormolu-plugin - run: cabal test hls-ormolu-plugin --test-options="$TEST_OPTS" + run: cabal test hls-ormolu-plugin --test-options="$TEST_OPTS" || cabal test hls-ormolu-plugin --test-options="$TEST_OPTS" - if: matrix.test && matrix.ghc != '9.6.1' name: Test hls-fourmolu-plugin - run: cabal test hls-fourmolu-plugin --test-options="$TEST_OPTS" + run: cabal test hls-fourmolu-plugin --test-options="$TEST_OPTS" || cabal test hls-fourmolu-plugin --test-options="$TEST_OPTS" - if: matrix.test && matrix.ghc != '9.2.7' && matrix.ghc != '9.4.4' && matrix.ghc != '9.6.1' name: Test hls-tactics-plugin test suite - run: cabal test hls-tactics-plugin --test-options="$TEST_OPTS" + run: cabal test hls-tactics-plugin --test-options="$TEST_OPTS" || cabal test hls-tactics-plugin --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-refine-imports-plugin test suite - run: cabal test hls-refine-imports-plugin --test-options="$TEST_OPTS" + run: cabal test hls-refine-imports-plugin --test-options="$TEST_OPTS" || cabal test hls-refine-imports-plugin --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-explicit-imports-plugin test suite - run: cabal test hls-explicit-imports-plugin --test-options="$TEST_OPTS" + run: cabal test hls-explicit-imports-plugin --test-options="$TEST_OPTS" || cabal test hls-explicit-imports-plugin --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-call-hierarchy-plugin test suite - run: cabal test hls-call-hierarchy-plugin --test-options="$TEST_OPTS" + run: cabal test hls-call-hierarchy-plugin --test-options="$TEST_OPTS" || cabal test hls-call-hierarchy-plugin --test-options="$TEST_OPTS" - if: matrix.test && matrix.os != 'windows-latest' name: Test hls-rename-plugin test suite - run: cabal test hls-rename-plugin --test-options="$TEST_OPTS" + run: cabal test hls-rename-plugin --test-options="$TEST_OPTS" || cabal test hls-rename-plugin --test-options="$TEST_OPTS" - if: matrix.test && matrix.ghc != '9.6.1' name: Test hls-hlint-plugin test suite - run: cabal test hls-hlint-plugin --test-options="$TEST_OPTS" + run: cabal test hls-hlint-plugin --test-options="$TEST_OPTS" || cabal test hls-hlint-plugin --test-options="$TEST_OPTS" - if: matrix.test && matrix.ghc != '9.0.2' && matrix.ghc != '9.2.7' && matrix.ghc != '9.4.4' && matrix.ghc != '9.6.1' name: Test hls-stan-plugin test suite - run: cabal test hls-stan-plugin --test-options="$TEST_OPTS" + run: cabal test hls-stan-plugin --test-options="$TEST_OPTS" || cabal test hls-stan-plugin --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-module-name-plugin test suite - run: cabal test hls-module-name-plugin --test-options="$TEST_OPTS" + run: cabal test hls-module-name-plugin --test-options="$TEST_OPTS" || cabal test hls-module-name-plugin --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-alternate-number-format-plugin test suite - run: cabal test hls-alternate-number-format-plugin --test-options="$TEST_OPTS" + run: cabal test hls-alternate-number-format-plugin --test-options="$TEST_OPTS" || cabal test hls-alternate-number-format-plugin --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-qualify-imported-names-plugin test suite - run: cabal test hls-qualify-imported-names-plugin --test-options="$TEST_OPTS" + run: cabal test hls-qualify-imported-names-plugin --test-options="$TEST_OPTS" || cabal test hls-qualify-imported-names-plugin --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-code-range-plugin test suite - run: cabal test hls-code-range-plugin --test-options="$TEST_OPTS" + run: cabal test hls-code-range-plugin --test-options="$TEST_OPTS" || cabal test hls-code-range-plugin --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-change-type-signature test suite - run: cabal test hls-change-type-signature-plugin --test-options="$TEST_OPTS" + run: cabal test hls-change-type-signature-plugin --test-options="$TEST_OPTS" || cabal test hls-change-type-signature-plugin --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-gadt-plugin test suit - run: cabal test hls-gadt-plugin --test-options="$TEST_OPTS" + run: cabal test hls-gadt-plugin --test-options="$TEST_OPTS" || cabal test hls-gadt-plugin --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-explicit-fixity-plugin test suite - run: cabal test hls-explicit-fixity-plugin --test-options="$TEST_OPTS" + run: cabal test hls-explicit-fixity-plugin --test-options="$TEST_OPTS" || cabal test hls-explicit-fixity-plugin --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-explicit-record-fields-plugin test suite - run: cabal test hls-explicit-record-fields-plugin --test-options="$TEST_OPTS" + run: cabal test hls-explicit-record-fields-plugin --test-options="$TEST_OPTS" || cabal test hls-explicit-record-fields-plugin --test-options="$TEST_OPTS" ## version needs to be limited since the tests depend on cabal-fmt which only builds using specific ghc versions - if: matrix.test && matrix.ghc == '8.10.7' name: Test hls-cabal-fmt-plugin test suite - run: cabal test hls-cabal-fmt-plugin --flag=isolateTests --test-options="$TEST_OPTS" + run: cabal test hls-cabal-fmt-plugin --flag=isolateTests --test-options="$TEST_OPTS" || cabal test hls-cabal-fmt-plugin --flag=isolateTests --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-cabal-plugin test suite - run: cabal test hls-cabal-plugin --test-options="$TEST_OPTS" + run: cabal test hls-cabal-plugin --test-options="$TEST_OPTS" || cabal test hls-cabal-plugin --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-retrie-plugin test suite - run: cabal test hls-retrie-plugin --test-options="$TEST_OPTS" + run: cabal test hls-retrie-plugin --test-options="$TEST_OPTS" || cabal test hls-retrie-plugin --test-options="$TEST_OPTS" test_post_job: if: always()