Add getComponent to project and package (#1060)

* Delays cabal configure errors until after evaluation when possible.

* Avoids rerunning long `cabal configure` if we already know it will fail.

* Provides a way to get a component from a package or project using a `cabal` like reference.

Code using the `tool` functions will automatically use `getComponent`.

For `(hackage-package {...}).components.library` is also ok.

```
# Consider changing hackage-package use:
(pkgs.haskell-nix.hackage-package {...}).components.exes.something
(pkgs.haskell-nix.hackage-package {...}).getComponent "exe:something"

# For any cabal project:
project.hsPkgs.somepackage.components.exes.something
project.getComponent "somepackage:exe:something"
# or do it in two steps
(project.getPackage "somepackage").getComponent "exe:something"
```

The reason for the new function is that we cannot provide the attribute interface without knowing that packages are in the project first.

Here is how the `cabal configure` error output is handled:

* The `plan-nix` derivation builds even if `cabal configure` fails.

* When it fails, it copies `failed-cabal-configure.nix` to the `$out/default.nix` along with a copy of the `cabal configure` output.

* When `failed-cabal-configure.nix` is imported and used in any way it writes the `cabal configure` output with `__trace` so it will always be visible.

* Instead of a `plan` the imported nix contains a `configurationError` pointing the `cabal configure` output.

* The intermediate functions `configurationError` and bubble it up to where it is needed.

* `getPackage` returns a mostly empty proxy for a real package when there is a `configurationError`

* The `getComponent` function always returns a derivation, but the version in the proxy writes the `cabal configure` output to stdout and calls `exit 1` (so that it will never build).

Author: hamishmack

ci.nix

@@ -80,9 +80,9 @@ dimension "Nixpkgs version" nixpkgsVersions (nixpkgsName: nixpkgs-pin:
         } // pkgs.lib.optionalAttrs runTests {
           inherit (build) tests tools maintainer-scripts maintainer-script-cache;
         } // pkgs.lib.optionalAttrs (ifdLevel >= 1) {
-          iserv-proxy = pkgs.ghc-extra-packages."${compiler-nix-name}".iserv-proxy.components.exes.iserv-proxy;
+          iserv-proxy = pkgs.ghc-extra-projects."${compiler-nix-name}".getComponent "iserv-proxy:exe:iserv-proxy";
         } // pkgs.lib.optionalAttrs (ifdLevel >= 3) {
-          hello = (pkgs.haskell-nix.hackage-package { name = "hello"; version = "1.0.0.2"; inherit compiler-nix-name; }).components.exes.hello;
+          hello = (pkgs.haskell-nix.hackage-package { name = "hello"; version = "1.0.0.2"; inherit compiler-nix-name; }).getComponent "exe:hello";
         });
       }
       //
@@ -102,10 +102,10 @@ dimension "Nixpkgs version" nixpkgsVersions (nixpkgsName: nixpkgs-pin:
             inherit (build) tests;
         }) // pkgs.lib.optionalAttrs (ifdLevel >= 2 && crossSystemName != "ghcjs") {
           # GHCJS builds its own template haskell runner.
-          remote-iserv = pkgs.ghc-extra-packages."${compiler-nix-name}".remote-iserv.components.exes.remote-iserv;
-          iserv-proxy = pkgs.ghc-extra-packages."${compiler-nix-name}".iserv-proxy.components.exes.iserv-proxy;
+          remote-iserv = pkgs.ghc-extra-projects."${compiler-nix-name}".getComponent "remote-iserv:exe:remote-iserv";
+          iserv-proxy = pkgs.ghc-extra-projects."${compiler-nix-name}".getComponent "iserv-proxy:exe:iserv-proxy";
         } // pkgs.lib.optionalAttrs (ifdLevel >= 3) {
-          hello = (pkgs.haskell-nix.hackage-package { name = "hello"; version = "1.0.0.2"; inherit compiler-nix-name; }).components.exes.hello;
+          hello = (pkgs.haskell-nix.hackage-package { name = "hello"; version = "1.0.0.2"; inherit compiler-nix-name; }).getComponent "exe:hello";
         })
       ))
     )

compiler/ghcjs/ghcjs.nix

@@ -24,21 +24,21 @@ let
     all-ghcjs = pkgs.buildPackages.symlinkJoin {
         name = "ghcjs-${ghcjsVersion}-symlinked";
         paths = [
-            ghcjs.components.exes.ghcjs
-            ghcjs.components.exes.ghcjs-pkg
-            ghcjs.components.exes.ghcjs-boot
-            ghcjs.components.exes.ghcjs-dumparchive
+            (ghcjs.getComponent "exe:ghcjs")
+            (ghcjs.getComponent "exe:ghcjs-pkg")
+            (ghcjs.getComponent "exe:ghcjs-boot")
+            (ghcjs.getComponent "exe:ghcjs-dumparchive")
         ] ++ (if isGhcjs88
           then [
-            ghcjs.components.exes.haddock
-            ghcjs.components.exes.private-ghcjs-run
-            ghcjs.components.exes.private-ghcjs-unlit
-            ghcjs.components.exes.private-ghcjs-hsc2hs
+            (ghcjs.getComponent "exe:haddock")
+            (ghcjs.getComponent "exe:private-ghcjs-run")
+            (ghcjs.getComponent "exe:private-ghcjs-unlit")
+            (ghcjs.getComponent "exe:private-ghcjs-hsc2hs")
           ]
           else [
-            ghcjs.components.exes.haddock-ghcjs
-            ghcjs.components.exes.hsc2hs-ghcjs
-            ghcjs.components.exes.ghcjs-run
+            (ghcjs.getComponent "exe:haddock-ghcjs")
+            (ghcjs.getComponent "exe:hsc2hs-ghcjs")
+            (ghcjs.getComponent "exe:ghcjs-run")
           ]);
     };
     libexec = "libexec/${builtins.replaceStrings ["darwin" "i686"] ["osx" "i386"] pkgs.stdenv.buildPlatform.system}-${ghc.name}/ghcjs-${ghcVersion}";
@@ -68,9 +68,9 @@ let
             lndir ${all-ghcjs}/bin $out/bin
             chmod -R +w $out/bin
             rm $out/bin/ghcjs-boot
-            cp ${ghcjs.components.exes.ghcjs-boot}/bin/ghcjs-boot $out/bin
+            cp ${ghcjs.getComponent "exe:ghcjs-boot"}/bin/ghcjs-boot $out/bin
             rm $out/bin/haddock
-            cp ${ghcjs.components.exes.haddock}/bin/haddock $out/bin
+            cp ${ghcjs.getComponent "exe:haddock"}/bin/haddock $out/bin
 
             wrapProgram $out/bin/ghcjs --add-flags "-B$out/lib"
             wrapProgram $out/bin/haddock --add-flags "-B$out/lib"

lib/call-cabal-project-to-nix.nix

@@ -44,6 +44,7 @@ in
 
 let
   forName = pkgs.lib.optionalString (name != null) (" for " + name);
+  nameAndSuffix = suffix: if name == null then suffix else name + "-" + suffix;
 
   ghc' =
     if ghcOverride != null
@@ -378,18 +379,40 @@ let
         else null;
   } // pkgs.lib.optionalAttrs (checkMaterialization != null) {
     inherit checkMaterialization;
-  }) (pkgs.evalPackages.runCommand (if name == null then "plan-to-nix-pkgs" else name + "-plan-to-nix-pkgs") {
+  }) (pkgs.evalPackages.runCommand (nameAndSuffix "plan-to-nix-pkgs") {
     nativeBuildInputs = [ nix-tools dummy-ghc dummy-ghc-pkg cabal-install pkgs.evalPackages.rsync ];
     # Needed or stack-to-nix will die on unicode inputs
     LOCALE_ARCHIVE = pkgs.lib.optionalString (pkgs.evalPackages.stdenv.buildPlatform.libc == "glibc") "${pkgs.evalPackages.glibcLocales}/lib/locale/locale-archive";
     LANG = "en_US.UTF-8";
     meta.platforms = pkgs.lib.platforms.all;
     preferLocalBuild = false;
     outputs = [
-      "out"     # The results of plan-to-nix
-      "json"    # The `plan.json` file generated by cabal and used for `plan-to-nix` input
-      "freeze"  # The `cabal.project.freeze` file created by `cabal v2-freeze`
+      "out"           # The results of plan-to-nix
+      # These two output will be present if in cabal configure failed.
+      # They are used to provide passthru.json and passthru.freeze that
+      # check first for cabal configure failure.
+      "maybeJson"    # The `plan.json` file generated by cabal and used for `plan-to-nix` input
+      "maybeFreeze"  # The `cabal.project.freeze` file created by `cabal v2-freeze`
     ];
+    passthru =
+      let
+        checkCabalConfigure = ''
+          if [[ -f ${plan-nix}/cabal-configure.out ]]; then
+            cat ${plan-nix}/cabal-configure.out
+            exit 1
+          fi
+        '';
+      in {
+        # These check for cabal configure failure
+        json = pkgs.evalPackages.runCommand (nameAndSuffix "plan-json") {} ''
+          ${checkCabalConfigure}
+          cp ${plan-nix.maybeJson} $out
+        '';
+        freeze = pkgs.evalPackages.runCommand (nameAndSuffix "plan-freeze") {} ''
+          ${checkCabalConfigure}
+          cp ${plan-nix.maybeFreeze} $out
+        '';
+      };
   } ''
     tmp=$(mktemp -d)
     cd $tmp
@@ -430,12 +453,14 @@ let
     export SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt
     export GIT_SSL_CAINFO=${cacert}/etc/ssl/certs/ca-bundle.crt
 
+    mkdir -p $out
+
     # Using `cabal v2-freeze` will configure the project (since
     # it is not configured yet), taking the existing `cabal.project.freeze`
     # file into account.  Then it "writes out a freeze file which
     # records all of the versions and flags that are picked" (from cabal docs).
     echo "Using index-state ${index-state-found}"
-    HOME=${
+    if(HOME=${
       # This creates `.cabal` directory that is as it would have
       # been at the time `cached-index-state`.  We may include
       # some packages that will be excluded by `index-state-found`
@@ -460,14 +485,12 @@ let
         --enable-benchmarks \
         ${pkgs.lib.optionalString (ghc.targetPrefix == "js-unknown-ghcjs-")
             "--ghcjs --with-ghcjs=js-unknown-ghcjs-ghc --with-ghcjs-pkg=js-unknown-ghcjs-ghc-pkg"} \
-        ${configureArgs}
+        ${configureArgs} 2>&1 | tee -a cabal-configure.out); then
 
-    cp cabal.project.freeze $freeze
+    cp cabal.project.freeze $maybeFreeze
     # Not needed any more (we don't want it to wind up in the $out hash)
     rm cabal.project.freeze
 
-    mkdir -p $out
-
     # ensure we have all our .cabal files (also those generated from package.yaml) files.
     # otherwise we'd need to be careful about putting the `cabal-generator = hpack` into
     # the nix expression.  As we already called `hpack` on all `package.yaml` files we can
@@ -489,7 +512,7 @@ let
     (cd $out${subDir'} && plan-to-nix --full --plan-json $tmp${subDir'}/dist-newstyle/cache/plan.json -o .)
 
     # Make the plan.json file available in case we need to debug plan-to-nix
-    cp $tmp${subDir'}/dist-newstyle/cache/plan.json $json
+    cp $tmp${subDir'}/dist-newstyle/cache/plan.json $maybeJson
 
     # Remove the non nix files ".project" ".cabal" "package.yaml" files
     # as they should not be in the output hash (they may change slightly
@@ -504,6 +527,18 @@ let
 
     # move pkgs.nix to default.nix ensure we can just nix `import` the result.
     mv $out${subDir'}/pkgs.nix $out${subDir'}/default.nix
+    else
+      # When cabal configure fails copy the output that we captured above and
+      # use `failed-cabal-configure.nix` to make a suitable derviation with.
+      cp cabal-configure.out $out
+      cp ${./failed-cabal-configure.nix} $out/default.nix
+
+      # These should only be used indirectly by `passthru.json` and `passthru.freeze`.
+      # Those derivations will check for `cabal-configure.out` out first to see if
+      # it is ok to use these files.
+      echo "Cabal configure failed see $out/cabal-configure.out for details" > $maybeJson
+      echo "Cabal configure failed see $out/cabal-configure.out for details" > $maybeFreeze
+    fi
   '');
 in {
   projectNix = plan-nix;

lib/default.nix

@@ -62,6 +62,12 @@ in {
     benchmarks = "bench";
   };
 
+  # For looking up the components attribute based on the cabal component type
+  prefixComponent =
+    lib.listToAttrs (
+      lib.mapAttrsToList (value: name: { inherit name value; })
+        componentPrefix);
+
   applyComponents = f: config:
     let
       comps = config.components;

lib/failed-cabal-configure.nix

@@ -0,0 +1,9 @@
+let
+  configurationError = ./cabal-configure.out;
+in
+  # Trace the error output to make sure the user has a chance to see it
+  # (even if the choose not to build anything from the project)
+  __trace ''
+    ERROR: cabal configure failed with:
+    ${__readFile configurationError}
+  '' { inherit configurationError; }

overlays/bootstrap.nix

@@ -581,7 +581,7 @@ in {
         cabal-install = final.evalPackages.haskell-nix.cabal-install-unchecked.${compiler-nix-name};
         nix-tools = final.evalPackages.haskell-nix.nix-tools-unchecked.${compiler-nix-name};
         materialized = ../materialized + "/${compiler-nix-name}/cabal-install";
-      } // args)).components.exes.cabal;
+      } // args)).getComponent "exe:cabal";
     nix-tools-set = { compiler-nix-name, ... }@args:
       let
         project =
@@ -617,7 +617,22 @@ in {
                 ];
             }];
           } // args);
-        exes = project.nix-tools.components.exes // project.hpack.components.exes;
+        exes =
+          let
+            package = project.getPackage "nix-tools";
+          in (builtins.map (name: package.getComponent "exe:${name}") [
+            "cabal-to-nix"
+            "hashes-to-nix"
+            "plan-to-nix"
+            "hackage-to-nix"
+            "lts-to-nix"
+            "stack-to-nix"
+            "truncate-index"
+            "stack-repos"
+            "cabal-name"
+          ]) ++ [
+            (project.getComponent "hpack:exe:hpack")
+          ];
         tools = [
           final.buildPackages.nix
           # Double buildPackages is intentional, see comment in lib/default.nix for details.
@@ -626,7 +641,7 @@ in {
     in
       final.symlinkJoin {
         name = "nix-tools";
-        paths = builtins.attrValues exes;
+        paths = exes;
         buildInputs = [ final.makeWrapper ];
         meta.platforms = final.lib.platforms.all;
         # We wrap the -to-nix executables with the executables from `tools` (e.g. nix-prefetch-git)
@@ -774,7 +789,7 @@ in {
                 version = "1.24.4";
                 inherit ghcOverride nix-tools cabal-install index-state;
                 materialized = ../materialized/bootstrap + "/${buildBootstrapper.compilerNixName}/hscolour";
-            } // args)).components.exes.HsColour;
+            } // args)).getComponent "exe:HsColour";
             hscolour = bootstrap.packages.hscolour-tool {};
             hscolour-unchecked = bootstrap.packages.hscolour-tool { checkMaterialization = false; };
         };

overlays/haskell.nix

@@ -457,7 +457,7 @@ final: prev: {
         hackage-package =
           { name, compiler-nix-name, ... }@args':
           let args = { caller = "hackage-package"; } // args';
-          in (hackage-project args).hsPkgs.${name};
+          in (hackage-project args).getPackage name;
         hackage-project =
             { name
             , compiler-nix-name
@@ -500,11 +500,19 @@ final: prev: {
             let
               args = { caller = "cabalProject'"; } // args';
               callProjectResults = callCabalProjectToNix args;
-            in let pkg-set = mkCabalProjectPkgSet
-                { inherit compiler-nix-name;
-                  plan-pkgs = importAndFilterProject {
-                    inherit (callProjectResults) projectNix sourceRepos src;
+              plan-pkgs = importAndFilterProject {
+                inherit (callProjectResults) projectNix sourceRepos src;
+              };
+              pkg-set = if plan-pkgs ? configurationError
+                then {
+                  inherit (plan-pkgs) configurationError;
+                  config = {
+                    compiler.nix-name = compiler-nix-name;
+                    hsPkgs = {};
                   };
+                }
+                else mkCabalProjectPkgSet
+                { inherit compiler-nix-name plan-pkgs;
                   pkg-def-extras = args.pkg-def-extras or [];
                   modules = (args.modules or [])
                           ++ final.lib.optional (args ? ghcOverride || args ? ghc)
@@ -514,7 +522,7 @@ final: prev: {
                   extra-hackages = args.extra-hackages or [];
                 };
 
-                project = addProjectAndPackageAttrs rec {
+              project = addProjectAndPackageAttrs rec {
                   inherit (pkg-set.config) hsPkgs;
                   inherit pkg-set;
                   plan-nix = callProjectResults.projectNix;
@@ -535,7 +543,7 @@ final: prev: {
           final.lib.fix (project':
             let project = project' // { recurseForDerivations = false; };
             in rawProject // rec {
-              hsPkgs = final.lib.mapAttrs (n: package':
+              hsPkgs = final.lib.mapAttrs (packageName: package':
                 if package' == null
                   then null
                   else
@@ -548,6 +556,16 @@ final: prev: {
                       ) package'.components;
                       inherit project;
 
+                      # Look up a component in the package based on ctype:name
+                      getComponent = componentName:
+                        let m = builtins.match "(lib|flib|exe|test|bench):([^:]*)" componentName;
+                        in
+                          assert final.lib.asserts.assertMsg (m != null)
+                            "Invalid package component name ${componentName}.  Expected it to start with one of lib: flib: exe: test: or bench:";
+                          if builtins.elemAt m 0 == "lib" && builtins.elemAt m 1 == packageName
+                            then components.library
+                            else components.${haskellLib.prefixComponent.${builtins.elemAt m 0}}.${builtins.elemAt m 1};
+
                       coverageReport = haskellLib.coverageReport (rec {
                         name = package.identifier.name + "-" + package.identifier.version;
                         library = if components ? library then components.library else null;
@@ -568,6 +586,48 @@ final: prev: {
                 rawProject.projectFunction pkgs.haskell-nix rawProject.projectArgs
               ) final.pkgsCross) // { recurseForDerivations = false; };
 
+            # Like `.hsPkgs.${packageName}` but when compined with `getComponent` any
+            # cabal configure errors are defered until the components derivation builds.
+            getPackage = packageName:
+              if rawProject.pkg-set ? configurationError
+                then
+                  # A minimal proxy for a package when cabal configure failed
+                  let package = {
+                    # Including the project so that things like:
+                    #  (p.getPackage "hello").project.tool "hlint" "latest"
+                    # will still work even if "hello" failed to configure.
+                    inherit project;
+
+                    # Defer configure time errors for the library component
+                    #  (p.getPackage "hello").components.library
+                    components.library = package.getComponent "lib:${packageName}";
+
+                    # This procide a derivation (even though the component may
+                    # not exist at all).  The derivation will never build
+                    # and simple outputs the result of cabal configure.
+                    getComponent = componentName:
+                      final.evalPackages.runCommand "cabal-configure-error" {
+                        passthru = {
+                          inherit project package;
+                        };
+                      } ''
+                        cat ${rawProject.pkg-set.configurationError}
+                        echo Unable to find component ${packageName}:${componentName}  \
+                          due to the above cabal configuration error
+                        exit 1
+                      '';
+                  };
+                  in package
+                else project.hsPkgs.${packageName};
+
+            # Look a component in the project based on `pkg:ctype:name`
+            getComponent = componentName:
+              let m = builtins.match "([^:]*):(lib|flib|exe|test|bench):([^:]*)" componentName;
+              in
+                assert final.lib.asserts.assertMsg (m != null)
+                  "Invalid package component name ${componentName}.  Expected package:ctype:component (where ctype is one of lib, flib, exe, test, or bench)";
+                (getPackage (builtins.elemAt m 0)).getComponent "${builtins.elemAt m 1}:${builtins.elemAt m 2}";
+
             # Helper function that can be used to make a Nix Flake out of a project
             # by including a flake.nix.  See docs/tutorials/getting-started-flakes.md
             # for an example flake.nix file.

overlays/tools.nix

@@ -66,7 +66,7 @@ in { haskell-nix = prev.haskell-nix // {
           { configureArgs = "--disable-benchmarks --disable-tests"; }
           // args
           // { name = final.haskell-nix.toolPackageName.${name} or name; }))
-          .components.exes."${final.haskell-nix.packageToolName.${name} or name}";
+          .getComponent "exe:${final.haskell-nix.packageToolName.${name} or name}";
 
   tool = compiler-nix-name: name: versionOrArgs:
     let

scripts/check-compiler-materialization/default.nix

@@ -4,12 +4,12 @@
 let
   eval = (import ../../. {}).pkgs;
   linux = (import ../../. { checkMaterialization = true; system = "x86_64-linux"; }).pkgs;
-  darwin = (import ../../. { checkMaterialization = true; system = "x86_64-darwin"; }).pkgs;
+  darwin = (import ../../. { checkMaterialization = true; system = "x86_64-darwin"; }).pkgs-unstable;
 in eval.linkFarm "check-${compiler-nix-name}" [
   # This set of derivations should be enough to ensure all the materialized files for a
   # given GHC version are checked.
   { name = "linux-cabal-install";  path = linux.haskell-nix.cabal-install.${compiler-nix-name}; }
-  # { name = "darwin-cabal-install"; path = darwin.haskell-nix.cabal-install.${compiler-nix-name}; }
+  { name = "darwin-cabal-install"; path = darwin.haskell-nix.cabal-install.${compiler-nix-name}; }
   { name = "linux-nix-tools";      path = linux.haskell-nix.nix-tools.${compiler-nix-name}; }
   { name = "linux";                path = linux.ghc-extra-projects.${compiler-nix-name}.plan-nix; }
   # In some cased you may need comment out one or more of these if the GHC version needed cannot be built.