diff --git a/configuration-ghc-94.nix b/configuration-ghc-94.nix
index 7c6ca11a82..4a0cb10fcd 100644
--- a/configuration-ghc-94.nix
+++ b/configuration-ghc-94.nix
@@ -13,19 +13,14 @@ let
     with pkgs.haskell.lib;
     {
       hlsDisabledPlugins = disabledPlugins;
-      # YOLO
-      mkDerivation = args:
-        hsuper.mkDerivation (args // {
-          jailbreak = true;
-          doCheck = false;
-        });
     } // (builtins.mapAttrs (_: drv: disableLibraryProfiling drv) {
       # ptr-poker breaks on MacOS without SSE2 optimizations
       # https://github.com/nikita-volkov/ptr-poker/issues/11
       ptr-poker = hself.callCabal2nix "ptr-poker" inputs.ptr-poker { };
 
+      # Freezes in test for some reason.
       ghc-exactprint =
-        hself.callCabal2nix "ghc-exactprint" inputs.ghc-exactprint-160 { };
+        dontCheck (hself.callCabal2nix "ghc-exactprint" inputs.ghc-exactprint-160 { });
       # Hlint is still broken
       hlint = doJailbreak (hself.callCabal2nix "hlint" inputs.hlint { });
 
@@ -34,7 +29,9 @@ let
       # Re-generate HLS drv excluding some plugins
       haskell-language-server =
         hself.callCabal2nixWithOptions "haskell-language-server" ./.
-        (pkgs.lib.concatStringsSep " " [ "-fpedantic" "-f-hlint" ]) { };
+        # Pedantic cannot be used due to -Werror=unused-top-binds
+        # Check must be disabled due to some missing required files
+        (pkgs.lib.concatStringsSep " " [ "--no-check" "-f-pedantic" "-f-hlint" ]) { };
     });
 in {
   inherit disabledPlugins;
diff --git a/flake.lock b/flake.lock
index 2ac39dc862..86f6147966 100644
--- a/flake.lock
+++ b/flake.lock
@@ -1,22 +1,5 @@
 {
   "nodes": {
-    "all-cabal-hashes-unpacked": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1668997806,
-        "narHash": "sha256-HRTQuIO/MxV5OcbCNsHSCeULa7KAjxIBQk5sAVFzrKk=",
-        "owner": "commercialhaskell",
-        "repo": "all-cabal-hashes",
-        "rev": "934c06ca91eb6ceca8a7c484dfc2862e955489f8",
-        "type": "github"
-      },
-      "original": {
-        "owner": "commercialhaskell",
-        "ref": "current-hackage",
-        "repo": "all-cabal-hashes",
-        "type": "github"
-      }
-    },
     "flake-compat": {
       "flake": false,
       "locked": {
@@ -178,7 +161,6 @@
     },
     "root": {
       "inputs": {
-        "all-cabal-hashes-unpacked": "all-cabal-hashes-unpacked",
         "flake-compat": "flake-compat",
         "flake-utils": "flake-utils",
         "fourmolu": "fourmolu",
diff --git a/flake.nix b/flake.nix
index c78cd3a843..2ca316a77d 100644
--- a/flake.nix
+++ b/flake.nix
@@ -19,13 +19,6 @@
       flake = false;
     };
 
-    # cabal hashes contains all the version for different haskell packages, to update:
-    # nix flake lock --update-input all-cabal-hashes-unpacked
-    all-cabal-hashes-unpacked = {
-      url = "github:commercialhaskell/all-cabal-hashes/current-hackage";
-      flake = false;
-    };
-
     # List of hackage dependencies
     ghc-exactprint-160 = {
       url = "https://hackage.haskell.org/package/ghc-exactprint-1.6.1/ghc-exactprint-1.6.1.tar.gz";
@@ -61,7 +54,7 @@
     };
   };
   outputs =
-    inputs@{ self, nixpkgs, flake-compat, flake-utils, gitignore, all-cabal-hashes-unpacked, ... }:
+    inputs@{ self, nixpkgs, flake-compat, flake-utils, gitignore, ... }:
     {
       overlays.default = final: prev:
         with prev;
@@ -121,7 +114,10 @@
             with haskell.lib; {
               # Patches don't apply
               github = overrideCabal hsuper.github (drv: { patches = []; });
-              hiedb = hsuper.callCabal2nix "hiedb" inputs.hiedb {};
+              # Tests are broken on 9.2 and 9.4, and we wind up bypassing the
+              # nixpkgs fixes to the test suite by doing this override. So just
+              # turn it off.
+              hiedb = haskell.lib.dontCheck (hsuper.callCabal2nix "hiedb" inputs.hiedb {});
 
               # https://github.com/NixOS/nixpkgs/issues/140774
               ormolu =
@@ -142,29 +138,28 @@
             builtins.mapAttrs (_: haskell.lib.dontCheck)
               (overlay hself hsuper);
 
-          extended = hpkgs: hpkgs.override (old: {
+          applyHaskellOverlays = overlays: hpkgs: hpkgs.override (old: {
             overrides =
               lib.fold
                 lib.composeExtensions
                 (old.overrides or (_: _: { }))
-                [ haskellOverrides
-                  (dontCheck (haskell.lib.packageSourceOverrides hlsSources))
-                  tweaks
-                ];
+                overlays;
           });
+
+          extended = forHlsCI:
+            applyHaskellOverlays
+              (prev.lib.optional forHlsCI haskellOverrides
+               ++ [ (dontCheck (haskell.lib.packageSourceOverrides hlsSources))
+                    tweaks
+                  ]
+              );
         in {
           inherit hlsSources;
 
-          all-cabal-hashes = prev.runCommand "all-cabal-hashes.tar.gz"
-            { }
-            ''
-              cd ${all-cabal-hashes-unpacked}
-              cd ..
-              tar czf $out $(basename ${all-cabal-hashes-unpacked})
-            '';
-
           # Haskell packages extended with our packages
-          hlsHpkgs = compiler: extended haskell.packages.${compiler};
+          hlsHpkgs = compiler: extended true haskell.packages.${compiler};
+          # Haskell packages extended with our packages; reusing the nixpkgs set as much as possible
+          hlsHpkgsNixpkgs = compiler: extended false haskell.packages.${compiler};
 
           # Support of GenChangelogs.hs
           gen-hls-changelogs = hpkgs:
diff --git a/plugins/hls-cabal-plugin/hls-cabal-plugin.cabal b/plugins/hls-cabal-plugin/hls-cabal-plugin.cabal
index f202b633c3..19ef99c3d0 100644
--- a/plugins/hls-cabal-plugin/hls-cabal-plugin.cabal
+++ b/plugins/hls-cabal-plugin/hls-cabal-plugin.cabal
@@ -44,8 +44,7 @@ library
     -- automatically, forcing us to manually update the packages revision id.
     -- This is a lot of work for almost zero benefit, so we just allow more versions here
     -- and we eventually completely drop support for building HLS with stack.
-    , Cabal                 ^>=3.2 || ^>=3.4 || ^>=3.6 || ^>= 3.8
-    , Cabal-syntax          ^>= 3.6
+    , Cabal-syntax          ^>= 3.6 || ^>= 3.8
     , deepseq
     , directory
     , extra                 >=1.7.4