diff --git a/builder/comp-builder.nix b/builder/comp-builder.nix index 0e7bce3636..bb5dd23e5a 100644 --- a/builder/comp-builder.nix +++ b/builder/comp-builder.nix @@ -53,6 +53,9 @@ let self = , enableExecutableProfiling ? component.enableExecutableProfiling , profilingDetail ? component.profilingDetail +# Coverage +, doCoverage ? component.doCoverage + # Data , enableSeparateDataOutput ? component.enableSeparateDataOutput @@ -118,6 +121,7 @@ let (enableFeature enableExecutableProfiling "executable-profiling") (enableFeature enableStatic "static") (enableFeature enableShared "shared") + (enableFeature doCoverage "coverage") ] ++ lib.optionals (stdenv.hostPlatform.isMusl && (haskellLib.isExecutableType componentId)) [ # These flags will make sure the resulting executable is statically linked. # If it uses other libraries it may be necessary for to add more @@ -358,6 +362,11 @@ let fi done '') + + (lib.optionalString doCoverage '' + mkdir -p $out/share + cp -r dist/hpc $out/share + cp dist/setup-config $out/ + '') } runHook postInstall '' + (lib.optionalString keepSource '' diff --git a/changelog.md b/changelog.md index 5671e901c2..285185cb70 100644 --- a/changelog.md +++ b/changelog.md @@ -1,6 +1,15 @@ This file contains a summary of changes to Haskell.nix and `nix-tools` that will impact users. +## Sep 8, 2020 +* Added the ability to generate coverage reports for packages and + projects. +* Added the `doCoverage` module option that allows users to choose + packages to enable coverage for. +* Added a `doCoverage` flag to the component builder that outputs HPC + information when coverage is enabled. +* Added test for coverage. + ## July 21, 2020 * Removed `components.all`, use `symlinkJoin` on components.exes or `shellFor` if you need a shell. diff --git a/docs/dev/coverage.md b/docs/dev/coverage.md new file mode 100644 index 0000000000..954703f7dd --- /dev/null +++ b/docs/dev/coverage.md @@ -0,0 +1,180 @@ +# Developer Coverage Overview + +## Building + +The implementation of coverage starts with the "doCoverage" flag on +the builder in `comp-builder.nix`. The doCoverage flag enables and +disables the Cabal coverage flag and copies any generated coverage +data to "$out/share/hpc". + +### Mix and tix files + +The coverage information for any derivation consists of "mix" and +"tix" files. + +Mix files record static information about a source file and are +generated at build time. They primarily contain a path to the source +file and information about expressions and regions of the source file, +which are later referenced by tix files. + +Tix files contain dynamic information about a test run, recording when +a portion of a source file is touched by a test. These are generated +when the test is run. + +### Multiple local packages + +In the context of multiple local packages, there are a few types of +coverage we might be interested in: + - How well does the tests for this package cover the package library? + - How well does the tests for this package cover the libraries of + other packages in this project? + - Both of the above. + +To facilitate expressing any of these classifications of coverage, the +`lib/cover.nix` function provides the `mixLibraries` argument. If +you're just interested in how the tests cover the package library, you +provide that library as an argument to `mixLibraries`. If you're +interested in how the tests also cover other local packages in the +project, you can also provide those libraries as arguments to +mixLibraries. + +The `projectCoverageReport` and `coverageReport` attributes that are +provided by default on projects and packages respectively provide +coverage information for *all* local packages in the project. This is +to mimic the behaviour of Stack, which seems to be the expectation of +most people. Of course, you can use the `projectCoverageReport` and +`coverageReport` functions to construct your own custom coverage +reports (as detailed in the [coverage tutorial](../tutorials/coverage.md#custom)). + +## Coverage reports + +### Package reports + +The coverage information generated will look something like this: + +```bash +/nix/store/...-my-project-0.1.0.0-coverage-report/ +└── share + └── hpc + └── vanilla + ├── html + │   └── my-library-0.1.0.0 + │   ├── my-library-0.1.0.0-48EVZBwW9Kj29VTaRMhBDf + │ │ ├── My.Lib.Config.hs.html + │ │ ├── My.Lib.Types.hs.html + │ │ └── My.Lib.Util.hs.html + │   ├── hpc_index_alt.html + │   ├── hpc_index_exp.html + │   ├── hpc_index_fun.html + │   └── hpc_index.html + ├── mix + │   └── my-library-0.1.0.0 + │   └── my-library-0.1.0.0-48EVZBwW9Kj29VTaRMhBDf + │      ├── My.Lib.Config.mix + │      ├── My.Lib.Types.mix + │      └── My.Lib.Util.mix + └── tix + └── my-library-0.1.0.0 + ├── my-library-0.1.0.0.tix + ├── my-test-1 + │   └── my-test-1.tix + └── unit-test + └── unit-test.tix +``` + +- The mix files are copied verbatim from the library built with + coverage. +- The tix files for each test are copied from the check run verbatim + and are output to ".../tix///.tix". +- The tix files for each library are generated by summing the tix + files for each test, but excluding any test modules. This tix file + is output to ".../tix//.tix". + - Test modules are determined by inspecting the plan for the project + (i.e. for the project "my-project" and test-suite "my-test-1", the + test modules are read from: + `my-project.checks.my-test-1.config.modules`) +- The hpc HTML reports for each library are generated from their + respective tix files (i.e. the + `share/hpc/vanilla/html/my-library-0.1.0.0` report is generated from + the + `share/hpc/vanilla/tix/my-library-0.1.0.0/my-library-0.1.0.0.tix` + file) + +### Project-wide reports + +The coverage information for an entire project will look something +like this: + +```bash +/nix/store/...-coverage-report +└── share + └── hpc + └── vanilla + ├── html + │   ├── index.html + │   ├── all + │   │   ├── my-library-0.1.0.0-ERSaOroBZhe9awsoBkhmcV + │ │   │ ├── My.Lib.Config.hs.html + │ │   │ ├── My.Lib.Types.hs.html + │ │   │ └── My.Lib.Util.hs.html + │ │ ├── other-library-0.1.0.0-48EVZBwW9Kj29VTaRMhBDf + │ │ │   ├── Other.Lib.A.hs.html + │ │ │   └── Other.Lib.B.hs.html + │   │   ├── hpc_index_alt.html + │   │   ├── hpc_index_exp.html + │   │   ├── hpc_index_fun.html + │   │   └── hpc_index.html + │   ├── my-library-0.1.0.0 + │   │   ├── my-library-0.1.0.0-ERSaOroBZhe9awsoBkhmcV + │ │   │ ├── My.Lib.Config.hs.html + │ │   │ ├── My.Lib.Types.hs.html + │ │   │ └── My.Lib.Util.hs.html + │   │   ├── hpc_index_alt.html + │   │   ├── hpc_index_exp.html + │   │   ├── hpc_index_fun.html + │   │   └── hpc_index.html + │ └── other-libray-0.1.0.0 + │ ├── other-library-0.1.0.0-48EVZBwW9Kj29VTaRMhBDf + │ │   ├── Other.Lib.A.hs.html + │ │   └── Other.Lib.B.hs.html + │ ├── hpc_index_alt.html + │ ├── hpc_index_exp.html + │ ├── hpc_index_fun.html + │ └── hpc_index.html + ├── mix + │   ├── my-library-0.1.0.0-ERSaOroBZhe9awsoBkhmcV + │   │   ├── My.Lib.Config.mix + │   │   ├── My.Lib.Types.mix + │   │   └── My.Lib.Util.mix + │   └── other-library-0.1.0.0-48EVZBwW9Kj29VTaRMhBDf + │   ├── Other.Lib.A.mix + │   └── Other.Lib.B.mix + └── tix + ├── all + │   └── all.tix + ├── my-library-0.1.0.0 + │ ├── my-library-0.1.0.0.tix + │ ├── my-test-1 + │ │   └── my-test-1.tix + │ └── unit-test + │ └── unit-test.tix + └── another-library-0.1.0.0 + ├── another-library-0.1.0.0.tix + ├── my-test-2 + │   └── my-test-2.tix + └── unit-test + └── unit-test.tix +``` + +All of the coverage information is copied verbatim from the coverage +reports for each of the constituent packages. A few additions are +made: + - `tix/all/all.tix` is generated from the union of all the library + tix files. + - We use this file when generating coverage reports for + "coveralls.io". + - An index page (`html/index.html`) is generated which links to the + HTML coverage reports of the constituent packages. + - A synthetic HTML report is generated from the `tix/all/all.tix` + file. This shows the union of all the coverage information + generated by each constituent coverage report. diff --git a/docs/tutorials/coverage.md b/docs/tutorials/coverage.md new file mode 100644 index 0000000000..4a1384f0a3 --- /dev/null +++ b/docs/tutorials/coverage.md @@ -0,0 +1,123 @@ +# Coverage + +haskell.nix can generate coverage information for your package or +project using Cabal's inbuilt hpc support. + +## Prerequisites + +To get a sensible coverage report, you need to enable coverage on each +of the packages of your project: + +```nix +pkgs.haskell-nix.project { + src = pkgs.haskell-nix.haskellLib.cleanGit { + name = "haskell-nix-project"; + src = ./.; + }; + compiler-nix-name = "ghc884"; + + modules = [{ + packages.$pkg.components.library.doCoverage = true; + }]; +} +``` + +If you would like to make coverage optional, add an argument to your nix expression: + +```nix +{ withCoverage ? false }: + +pkgs.haskell-nix.project { + src = pkgs.haskell-nix.haskellLib.cleanGit { + name = "haskell-nix-project"; + src = ./.; + }; + compiler-nix-name = "ghc884"; + + modules = pkgs.lib.optional withCoverage [{ + packages.$pkg.components.library.doCoverage = true; + }]; +} +``` + +## Per-package + +```bash +nix-build default.nix -A "projectWithCoverage.$pkg.coverageReport" +``` + +This will generate a coverage report for the package you requested. +All tests that are enabled (configured with `doCheck == true`) are +included in the coverage report. + +See the [developer coverage docs](../dev/coverage.md#package-reports) for more information. + +## Project-wide + +```bash +nix-build default.nix -A "projectWithCoverage.projectCoverageReport" +``` + +This will generate a coverage report for all the local packages in +your project. + +See the [developer coverage docs](../dev/coverage.md#project-wide-reports) for more information. + +## Custom + +By default, the behaviour of the `coverageReport` attribute is to +generate a coverage report that describes how that package affects the +coverage of all local packages (including itself) in the project. + +The default behaviour of `projectCoverageReport` is to sum the +default coverage reports (produced by the above process) of all local +packages in the project. + +You can modify this behaviour by using the `coverageReport` and +`projectCoverageReport` functions found in the haskell.nix library: + +```nix +let + inherit (pkgs.haskell-nix) haskellLib; + + project = haskellLib.project { + src = pkgs.haskell-nix.haskellLib.cleanGit { + name = "haskell-nix-project"; + src = ./.; + }; + compiler-nix-name = "ghc884"; + + modules = [{ + packages.$pkgA.components.library.doCoverage = true; + packages.$pkgB.components.library.doCoverage = true; + }]; + }; + + # Generate a coverage report for $pkgA that only includes the + # unit-test check and only shows coverage information for $pkgA, not + # $pkgB. + custom$pkgACoverageReport = haskellLib.coverageReport rec { + name = "$pkgA-unit-tests-only" + inherit (project.$pkgA.components) library; + checks = [project.$pkgA.components.checks.unit-test]; + # Note that this is the default value of the "mixLibraries" + # argument and so this line isn't really necessary. + mixLibraries = [project.$pkgA.components.library]; + }; + + custom$pkgBCoverageReport = haskellLib.coverageReport rec { + name = "$pkgB-unit-tests-only" + inherit (project.$pkgB.components) library; + checks = [project.$pkgB.components.checks.unit-test]; + mixLibraries = [project.$pkgB.components.library]; + }; + + # Generate a project coverage report that only includes the unit + # tests of the project, and only shows how each unit test effects + # the coverage of it's package, and not other packages in the + # project. + allUnitTestsProjectReport = haskellLib.projectCoverageReport [custom$pkgACoverageReport custom$pkgBCoverageReport]; +in { + inherit project custom$pkgACoverageReport custom$pkgBCoverageReport allUnitTestsProjectCoverageReport; +} +``` diff --git a/lib/check.nix b/lib/check.nix index 24f68d88a5..2fa2f41ebe 100644 --- a/lib/check.nix +++ b/lib/check.nix @@ -15,7 +15,7 @@ in stdenv.mkDerivation ({ src = drv.source or (srcOnly drv); passthru = { - inherit (drv) identifier config configFiles executableToolDepends cleanSrc env; + inherit (drv) identifier config configFiles executableToolDepends cleanSrc env exeName; }; inherit (drv) meta LANG LC_ALL buildInputs nativeBuildInputs; @@ -27,11 +27,14 @@ in stdenv.mkDerivation ({ # If doCheck or doCrossCheck are false we may still build this # component and we want it to quietly succeed. buildPhase = '' - touch $out + mkdir $out runHook preCheck - ${toString component.testWrapper} ${drv}/bin/${drv.exeName} ${lib.concatStringsSep " " component.testFlags} | tee $out + ${toString component.testWrapper} ${drv}/bin/${drv.exeName} ${lib.concatStringsSep " " component.testFlags} | tee $out/test-stdout + + # Copy over tix files, if they exist + find . -iname '${drv.exeName}.tix' -exec mkdir -p $out/share/hpc/vanilla/tix/${drv.exeName} \; -exec cp {} $out/share/hpc/vanilla/tix/${drv.exeName}/ \; runHook postCheck ''; diff --git a/lib/cover-project.nix b/lib/cover-project.nix new file mode 100644 index 0000000000..64b12aad29 --- /dev/null +++ b/lib/cover-project.nix @@ -0,0 +1,150 @@ +# A project coverage report is a composition of package coverage +# reports +{ stdenv, pkgs, lib, haskellLib }: + +# List of coverage reports to accumulate +coverageReports: + +let + toBashArray = arr: "(" + (lib.concatStringsSep " " arr) + ")"; + + # Create table rows for a project coverage index page that look something like: + # + # | Package | + # |------------------| + # | cardano-shell | + # | cardano-launcher | + coverageTableRows = coverageReport: + '' + + + ${coverageReport.passthru.name} + + + ''; + + projectIndexHtml = pkgs.writeText "index.html" '' + + + + + + + + + + + + ${with lib; concatStringsSep "\n" (map coverageTableRows coverageReports)} + + +
Report
+ + + ''; + + ghc = + if (builtins.length coverageReports) > 0 + then (builtins.head coverageReports).library.project.pkg-set.config.ghc.package or pkgs.ghc + else pkgs.ghc; + + libs = map (r: r.library) coverageReports; + + projectLibs = map (pkg: pkg.components.library) (lib.attrValues (haskellLib.selectProjectPackages ((lib.head libs).project.hsPkgs))); + + mixDirs = + map + (l: "${l}/share/hpc/vanilla/mix/${l.identifier.name}-${l.identifier.version}") + (projectLibs); + + srcDirs = map (l: l.src.outPath) (projectLibs); + +in pkgs.runCommand "project-coverage-report" + ({ buildInputs = [ghc]; + LANG = "en_US.UTF-8"; + LC_ALL = "en_US.UTF-8"; + } // lib.optionalAttrs (stdenv.buildPlatform.libc == "glibc") { + LOCALE_ARCHIVE = "${pkgs.buildPackages.glibcLocales}/lib/locale/locale-archive"; + }) + '' + function markup() { + local -n srcDs=$1 + local -n mixDs=$2 + local -n includedModules=$3 + local destDir=$4 + local tixFile=$5 + + local hpcMarkupCmd=("hpc" "markup" "--destdir=$destDir") + for srcDir in "''${srcDs[@]}"; do + hpcMarkupCmd+=("--srcdir=$srcDir") + done + + for mixDir in "''${mixDs[@]}"; do + hpcMarkupCmd+=("--hpcdir=$mixDir") + done + + for module in "''${includedModules[@]}"; do + hpcMarkupCmd+=("--include=$module") + done + + hpcMarkupCmd+=("$tixFile") + + echo "''${hpcMarkupCmd[@]}" + eval "''${hpcMarkupCmd[@]}" + } + + function findModules() { + local searchDir=$2 + local pattern=$3 + + pushd $searchDir + mapfile -d $'\0' $1 < <(find ./ -type f \ + -wholename "$pattern" -not -name "Paths*" \ + -exec basename {} \; \ + | sed "s/\.mix$//" \ + | tr "\n" "\0") + popd + } + + mkdir -p $out/share/hpc/vanilla/tix/all + mkdir -p $out/share/hpc/vanilla/mix/ + mkdir -p $out/share/hpc/vanilla/html/ + + # Find all tix files in each package + tixFiles=() + ${with lib; concatStringsSep "\n" (map (coverageReport: '' + identifier="${coverageReport.name}" + report=${coverageReport} + tix="$report/share/hpc/vanilla/tix/$identifier/$identifier.tix" + if test -f "$tix"; then + tixFiles+=("$tix") + fi + + # Copy mix, tix, and html information over from each report + cp -Rn $report/share/hpc/vanilla/mix/$identifier/* $out/share/hpc/vanilla/mix/ + cp -R $report/share/hpc/vanilla/tix/* $out/share/hpc/vanilla/tix/ + cp -R $report/share/hpc/vanilla/html/* $out/share/hpc/vanilla/html/ + '') coverageReports)} + + if [ ''${#tixFiles[@]} -ne 0 ]; then + # Create tix file with test run information for all packages + tixFile="$out/share/hpc/vanilla/tix/all/all.tix" + hpcSumCmd=("hpc" "sum" "--union" "--output=$tixFile") + hpcSumCmd+=("''${tixFiles[@]}") + echo "''${hpcSumCmd[@]}" + eval "''${hpcSumCmd[@]}" + + # Markup a HTML coverage report for the entire project + cp ${projectIndexHtml} $out/share/hpc/vanilla/html/index.html + + local markupOutDir="$out/share/hpc/vanilla/html/all" + local srcDirs=${toBashArray srcDirs} + local mixDirs=${toBashArray mixDirs} + local allMixModules=() + + mkdir $markupOutDir + findModules allMixModules "$out/share/hpc/vanilla/mix/" "*.mix" + + markup srcDirs mixDirs allMixModules "$markupOutDir" "$tixFile" + fi + '' diff --git a/lib/cover.nix b/lib/cover.nix new file mode 100644 index 0000000000..010e2d3dda --- /dev/null +++ b/lib/cover.nix @@ -0,0 +1,165 @@ +{ stdenv, lib, haskellLib, pkgs }: + +# Name of the coverage report, which should be unique +{ name +# Library to check coverage of +, library +# List of check derivations that generate coverage +, checks +# List of other libraries to include in the coverage report. The +# default value if just the derivation provided as the `library` +# argument. Use a larger list of libraries if you would like the tests +# of one local package to generate coverage for another. +, mixLibraries ? [library] +# hack for project-less projects +, ghc ? library.project.pkg-set.config.ghc.package +}: + +let + toBashArray = arr: "(" + (lib.concatStringsSep " " arr) + ")"; + + mixDir = l: "${l}/share/hpc/vanilla/mix/${l.identifier.name}-${l.identifier.version}"; + mixDirs = map mixDir mixLibraries; + + srcDirs = map (l: l.src.outPath) mixLibraries; + +in pkgs.runCommand (name + "-coverage-report") + ({ buildInputs = [ ghc ]; + passthru = { + inherit name library checks; + }; + # HPC will fail if the Haskell file contains non-ASCII characters, + # unless our locale is set correctly. This has been fixed, but we + # don't know what version of HPC we will be using, hence we should + # always use the workaround. + # https://gitlab.haskell.org/ghc/ghc/-/issues/17073 + LANG = "en_US.UTF-8"; + LC_ALL = "en_US.UTF-8"; + } // lib.optionalAttrs (stdenv.buildPlatform.libc == "glibc") { + LOCALE_ARCHIVE = "${pkgs.buildPackages.glibcLocales}/lib/locale/locale-archive"; + }) + '' + function markup() { + local -n srcDs=$1 + local -n mixDs=$2 + local -n includedModules=$3 + local destDir=$4 + local tixFile=$5 + + local hpcMarkupCmd=("hpc" "markup" "--destdir=$destDir") + for srcDir in "''${srcDs[@]}"; do + hpcMarkupCmd+=("--srcdir=$srcDir") + done + + for mixDir in "''${mixDs[@]}"; do + hpcMarkupCmd+=("--hpcdir=$mixDir") + done + + for module in "''${includedModules[@]}"; do + hpcMarkupCmd+=("--include=$module") + done + + hpcMarkupCmd+=("$tixFile") + + echo "''${hpcMarkupCmd[@]}" + eval "''${hpcMarkupCmd[@]}" + } + + function sumTix() { + local -n includedModules=$1 + local -n tixFs=$2 + local outFile="$3" + + local hpcSumCmd=("hpc" "sum" "--union" "--output=$outFile") + + for module in "''${includedModules[@]}"; do + hpcSumCmd+=("--include=$module") + done + + for tixFile in "''${tixFs[@]}"; do + hpcSumCmd+=("$tixFile") + done + + echo "''${hpcSumCmd[@]}" + eval "''${hpcSumCmd[@]}" + } + + function findModules() { + local searchDir=$2 + local pattern=$3 + + pushd $searchDir + mapfile -d $'\0' $1 < <(find ./ -type f \ + -wholename "$pattern" -not -name "Paths*" \ + -exec basename {} \; \ + | sed "s/\.mix$//" \ + | tr "\n" "\0") + popd + } + + local mixDirs=${toBashArray mixDirs} + + mkdir -p $out/share/hpc/vanilla/mix/${name} + mkdir -p $out/share/hpc/vanilla/tix/${name} + mkdir -p $out/share/hpc/vanilla/html/${name} + + # Copy over mix files verbatim + for dir in "''${mixDirs[@]}"; do + if [ -d "$dir" ]; then + cp -R "$dir"/* $out/share/hpc/vanilla/mix/${name} + fi + done + + local srcDirs=${toBashArray srcDirs} + local allMixModules=() + local pkgMixModules=() + + # The behaviour of stack coverage reports is to provide tix files + # that include coverage information for every local package, but + # to provide HTML reports that only include coverage info for the + # current package. We emulate the same behaviour here. If the user + # includes all local packages in the mix libraries argument, they + # will get a coverage report very similar to stack. + + # All mix modules + findModules allMixModules "$out/share/hpc/vanilla/mix/${name}" "*.mix" + # Only mix modules corresponding to this package + findModules pkgMixModules "$out/share/hpc/vanilla/mix/${name}" "*${name}*/*.mix" + + # For each test + local tixFiles=() + ${lib.concatStringsSep "\n" (builtins.map (check: '' + if [ -d "${check}/share/hpc/vanilla/tix" ]; then + pushd ${check}/share/hpc/vanilla/tix + + tixFile="$(find . -iwholename "*.tix" -type f -print -quit)" + local newTixFile=$out/share/hpc/vanilla/tix/${name}/"$tixFile" + + mkdir -p "$(dirname $newTixFile)" + # Copy over the tix file verbatim + cp "$tixFile" "$newTixFile" + + # Add the tix file to our list + tixFiles+=("$newTixFile") + + # Create a coverage report for *just that test* + markup srcDirs mixDirs pkgMixModules "$out/share/hpc/vanilla/html/${name}/${check.exeName}/" "$newTixFile" + + popd + fi + '') checks) + } + + # Sum tix files to create a tix file with all relevant tix + # information and markup a HTML report from this info. + if (( "''${#tixFiles[@]}" > 0 )); then + local sumTixFile="$out/share/hpc/vanilla/tix/${name}/${name}.tix" + local markupOutDir="$out/share/hpc/vanilla/html/${name}" + + # Sum all of our tix file, including modules from any local package + sumTix allMixModules tixFiles "$sumTixFile" + + # Markup a HTML report, included modules from only this package + markup srcDirs mixDirs pkgMixModules "$markupOutDir" "$sumTixFile" + fi + '' diff --git a/lib/default.nix b/lib/default.nix index 6c23fdaad4..fea665d8b2 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -232,6 +232,16 @@ in { inherit stdenv lib haskellLib srcOnly; }; + # Do coverage of a package + coverageReport = import ./cover.nix { + inherit stdenv lib haskellLib pkgs; + }; + + # Do coverage of a project + projectCoverageReport = import ./cover-project.nix { + inherit stdenv lib haskellLib pkgs; + }; + # Use `isCrossHost` to identify when we are cross compiling and # the code we are producing will not run on the build system # without an emulator. diff --git a/mkdocs.yml b/mkdocs.yml index 9dfd6b3204..b0b0090b4a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -35,6 +35,7 @@ pages: - 'Bumping Hackage and Stackage snapshots': tutorials/hackage-stackage.md - 'Materialization: Speeding up Nix evaluation': tutorials/materialization.md - 'Cross-compiling your project': tutorials/cross-compilation.md + - 'Generating coverage information': tutorials/coverage.md - 'Reference': - 'Suported GHC versions': reference/supported-ghc-versions.md - 'Command-line tools': reference/commands.md @@ -52,4 +53,5 @@ pages: - 'Removing withPackage wrapper': dev/removing-with-package-wrapper.md - 'Test Suite': dev/tests.md - 'Adding a new GHC version': dev/adding-new-ghc.md + - 'Coverage': dev/coverage.md - 'ChangeLog': changelog.md diff --git a/modules/plan.nix b/modules/plan.nix index 14a0ac45d6..857637392f 100644 --- a/modules/plan.nix +++ b/modules/plan.nix @@ -108,6 +108,11 @@ let type = bool; default = (def.doQuickjump or true); }; + doCoverage = mkOption { + description = "Enable production of test coverage reports."; + type = bool; + default = (def.doCoverage or false); + }; dontPatchELF = mkOption { description = "If set, the patchelf command is not used to remove unnecessary RPATH entries. Only applies to Linux."; type = bool; diff --git a/overlays/haskell.nix b/overlays/haskell.nix index d01082020a..99ba6fc8c8 100644 --- a/overlays/haskell.nix +++ b/overlays/haskell.nix @@ -482,15 +482,18 @@ final: prev: { { compiler.nix-name = args.compiler-nix-name; }; extra-hackages = args.extra-hackages or []; }; - in addProjectAndPackageAttrs { - inherit (pkg-set.config) hsPkgs; - inherit pkg-set; - plan-nix = callProjectResults.projectNix; - inherit (callProjectResults) index-state; - tool = final.buildPackages.haskell-nix.tool pkg-set.config.compiler.nix-name; - tools = final.buildPackages.haskell-nix.tools pkg-set.config.compiler.nix-name; - roots = final.haskell-nix.roots pkg-set.config.compiler.nix-name; - }; + + project = addProjectAndPackageAttrs rec { + inherit (pkg-set.config) hsPkgs; + inherit pkg-set; + plan-nix = callProjectResults.projectNix; + inherit (callProjectResults) index-state; + tool = final.buildPackages.haskell-nix.tool pkg-set.config.compiler.nix-name; + tools = final.buildPackages.haskell-nix.tools pkg-set.config.compiler.nix-name; + roots = final.haskell-nix.roots pkg-set.config.compiler.nix-name; + }; + in project; + # Take `hsPkgs` from the `rawProject` and update all the packages and # components so they have a `.project` attribute and as well as @@ -498,25 +501,34 @@ final: prev: { addProjectAndPackageAttrs = rawProject: final.lib.fix (project': let project = project' // { recurseForDerivations = false; }; - in rawProject // { + in rawProject // rec { hsPkgs = (final.lib.mapAttrs (n: package': if package' == null then null else let package = package' // { recurseForDerivations = false; }; - in package' // { + in package' // rec { components = final.lib.mapAttrs (n: v: if n == "library" || n == "all" then v // { inherit project package; } else final.lib.mapAttrs (_: c: c // { inherit project package; }) v ) package'.components; inherit project; + + coverageReport = haskellLib.coverageReport (rec { + name = package.identifier.name + "-" + package.identifier.version; + inherit (components) library; + checks = final.lib.filter (final.lib.isDerivation) (final.lib.attrValues package'.checks); + mixLibraries = map (pkg: pkg.components.library) (final.lib.attrValues (haskellLib.selectProjectPackages project.hsPkgs)); + }); } ) rawProject.hsPkgs // { # These are functions not packages inherit (rawProject.hsPkgs) shellFor ghcWithHoogle ghcWithPackages; }); + + projectCoverageReport = haskellLib.projectCoverageReport (map (pkg: pkg.coverageReport) (final.lib.attrValues (haskellLib.selectProjectPackages hsPkgs))); }); cabalProject = @@ -525,7 +537,7 @@ final: prev: { args = { caller = "hackage-package"; } // args'; p = cabalProject' args; in p.hsPkgs // { - inherit (p) plan-nix index-state tool tools roots; + inherit (p) plan-nix index-state tool tools roots projectCoverageReport; # Provide `nix-shell -A shells.ghc` for users migrating from the reflex-platform. # But we should encourage use of `nix-shell -A shellFor` shells.ghc = p.hsPkgs.shellFor {}; @@ -543,18 +555,20 @@ final: prev: { ++ (args.modules or []) ++ final.lib.optional (args ? ghc) { ghc.package = args.ghc; }; }; - in addProjectAndPackageAttrs { - inherit (pkg-set.config) hsPkgs; - inherit pkg-set; - stack-nix = callProjectResults.projectNix; - tool = final.buildPackages.haskell-nix.tool pkg-set.config.compiler.nix-name; - tools = final.buildPackages.haskell-nix.tools pkg-set.config.compiler.nix-name; - roots = final.haskell-nix.roots pkg-set.config.compiler.nix-name; - }; + + project = addProjectAndPackageAttrs { + inherit (pkg-set.config) hsPkgs; + inherit pkg-set; + stack-nix = callProjectResults.projectNix; + tool = final.buildPackages.haskell-nix.tool pkg-set.config.compiler.nix-name; + tools = final.buildPackages.haskell-nix.tools pkg-set.config.compiler.nix-name; + roots = final.haskell-nix.roots pkg-set.config.compiler.nix-name; + }; + in project; stackProject = args: let p = stackProject' args; in p.hsPkgs // { - inherit (p) stack-nix tool tools roots; + inherit (p) stack-nix tool tools roots projectCoverageReport; # Provide `nix-shell -A shells.ghc` for users migrating from the reflex-platform. # But we should encourage use of `nix-shell -A shellFor` shells.ghc = p.hsPkgs.shellFor {}; diff --git a/test/buildable/default.nix b/test/buildable/default.nix index 5c65fb0ac1..e622fe0af2 100644 --- a/test/buildable/default.nix +++ b/test/buildable/default.nix @@ -21,7 +21,7 @@ in recurseIntoAttrs { buildCommand = (concatStrings (mapAttrsToList (name: value: '' printf "checking whether executable runs... " >& 2 - cat ${haskellLib.check value} + cat ${haskellLib.check value}/test-stdout '') packages.buildable-test.components.exes)) + '' touch $out ''; diff --git a/test/cabal-22/default.nix b/test/cabal-22/default.nix index e16f2d92d9..3a5694659f 100644 --- a/test/cabal-22/default.nix +++ b/test/cabal-22/default.nix @@ -26,7 +26,7 @@ in recurseIntoAttrs { # fixme: run on target platform when cross-compiled printf "checking whether executable runs... " >& 2 - cat ${haskellLib.check packages.project.components.exes.project} + cat ${haskellLib.check packages.project.components.exes.project}/test-stdout '' + # Aarch is statically linked and does not produce a .so file. @@ -56,10 +56,10 @@ in recurseIntoAttrs { touch $out printf "checking whether benchmark ran... " >& 2 - cat ${haskellLib.check packages.project.components.benchmarks.project-bench} + cat ${haskellLib.check packages.project.components.benchmarks.project-bench}/test-stdout printf "checking whether tests ran... " >& 2 - cat ${haskellLib.check packages.project.components.tests.unit} + cat ${haskellLib.check packages.project.components.tests.unit}/test-stdout ''; meta.platforms = platforms.all; diff --git a/test/cabal-hpack/default.nix b/test/cabal-hpack/default.nix index c169894b28..2bb66e15a6 100644 --- a/test/cabal-hpack/default.nix +++ b/test/cabal-hpack/default.nix @@ -36,7 +36,7 @@ in recurseIntoAttrs { # fixme: run on target platform when cross-compiled printf "checking whether executable runs... " >& 2 - cat ${haskellLib.check packages.cabal-hpack.components.exes.cabal-hpack} + cat ${haskellLib.check packages.cabal-hpack.components.exes.cabal-hpack}/test-stdout '' + (if stdenv.hostPlatform.isMusl then '' printf "checking that executable is statically linked... " >& 2 diff --git a/test/cabal-simple/default.nix b/test/cabal-simple/default.nix index d1a6486f2e..4af00f9e5a 100644 --- a/test/cabal-simple/default.nix +++ b/test/cabal-simple/default.nix @@ -39,7 +39,7 @@ in recurseIntoAttrs { # fixme: run on target platform when cross-compiled printf "checking whether executable runs... " >& 2 - cat ${haskellLib.check packages.cabal-simple.components.exes.cabal-simple} + cat ${haskellLib.check packages.cabal-simple.components.exes.cabal-simple}/test-stdout '' + (if stdenv.hostPlatform.isMusl then '' printf "checking that executable is statically linked... " >& 2 diff --git a/test/cabal-source-repo-comments/default.nix b/test/cabal-source-repo-comments/default.nix index b72f656644..ce97b1b284 100644 --- a/test/cabal-source-repo-comments/default.nix +++ b/test/cabal-source-repo-comments/default.nix @@ -20,7 +20,7 @@ in recurseIntoAttrs { exe="${packages.use-cabal-simple.components.exes.use-cabal-simple}/bin/use-cabal-simple${stdenv.hostPlatform.extensions.executable}" printf "checking whether executable runs... " >& 2 - cat ${haskellLib.check packages.use-cabal-simple.components.exes.use-cabal-simple} + cat ${haskellLib.check packages.use-cabal-simple.components.exes.use-cabal-simple}/test-stdout touch $out ''; diff --git a/test/cabal-source-repo/default.nix b/test/cabal-source-repo/default.nix index be1a9dd8e7..67d373df68 100644 --- a/test/cabal-source-repo/default.nix +++ b/test/cabal-source-repo/default.nix @@ -20,7 +20,7 @@ in recurseIntoAttrs { exe="${packages.use-cabal-simple.components.exes.use-cabal-simple}/bin/use-cabal-simple${stdenv.hostPlatform.extensions.executable}" printf "checking whether executable runs... " >& 2 - cat ${haskellLib.check packages.use-cabal-simple.components.exes.use-cabal-simple} + cat ${haskellLib.check packages.use-cabal-simple.components.exes.use-cabal-simple}/test-stdout touch $out ''; diff --git a/test/cabal-sublib/default.nix b/test/cabal-sublib/default.nix index c174d32f33..d0464b28b4 100644 --- a/test/cabal-sublib/default.nix +++ b/test/cabal-sublib/default.nix @@ -36,7 +36,7 @@ in recurseIntoAttrs { # fixme: run on target platform when cross-compiled printf "checking whether executable runs... " >& 2 - cat ${haskellLib.check packages.cabal-sublib.components.exes.cabal-sublib} + cat ${haskellLib.check packages.cabal-sublib.components.exes.cabal-sublib}/test-stdout '' + # Musl and Aarch are statically linked.. diff --git a/test/call-cabal-project-to-nix/default.nix b/test/call-cabal-project-to-nix/default.nix index 8d3ecaa0bb..e49abf6731 100644 --- a/test/call-cabal-project-to-nix/default.nix +++ b/test/call-cabal-project-to-nix/default.nix @@ -29,7 +29,7 @@ in recurseIntoAttrs { exe="${packages.cabal-simple.components.exes.cabal-simple}/bin/cabal-simple${stdenv.hostPlatform.extensions.executable}" printf "checking whether executable runs... " >& 2 - cat ${haskellLib.check packages.cabal-simple.components.exes.cabal-simple} + cat ${haskellLib.check packages.cabal-simple.components.exes.cabal-simple}/test-stdout touch $out ''; diff --git a/test/call-stack-to-nix/default.nix b/test/call-stack-to-nix/default.nix index 1155299863..e840f43d78 100644 --- a/test/call-stack-to-nix/default.nix +++ b/test/call-stack-to-nix/default.nix @@ -24,7 +24,7 @@ in recurseIntoAttrs { exe="${packages.stack-simple.components.exes.stack-simple-exe}/bin/stack-simple-exe${stdenv.hostPlatform.extensions.executable}" printf "checking whether executable runs... " >& 2 - cat ${haskellLib.check packages.stack-simple.components.exes.stack-simple-exe} + cat ${haskellLib.check packages.stack-simple.components.exes.stack-simple-exe}/test-stdout touch $out ''; diff --git a/test/coverage/cabal.project b/test/coverage/cabal.project new file mode 100644 index 0000000000..f035a92f6d --- /dev/null +++ b/test/coverage/cabal.project @@ -0,0 +1,2 @@ +packages: pkga + pkgb diff --git a/test/coverage/conduit.hs b/test/coverage/conduit.hs new file mode 100644 index 0000000000..061470e92d --- /dev/null +++ b/test/coverage/conduit.hs @@ -0,0 +1,19 @@ +-- https://github.com/snoyberg/conduit#readme + +import Conduit +import System.Directory (removeFile) + +main = do + -- Pure operations: summing numbers. + print $ runConduitPure $ yieldMany [1..10] .| sumC + + -- Exception safe file access: copy a file. + writeFile "input.txt" "This is a test." -- create the source file + runConduitRes $ sourceFileBS "input.txt" .| sinkFile "output.txt" -- actual copying + readFile "output.txt" >>= putStrLn -- prove that it worked + + -- Perform transformations. + print $ runConduitPure $ yieldMany [1..10] .| mapC (+ 1) .| sinkList + + removeFile "input.txt" + removeFile "output.txt" diff --git a/test/coverage/default.nix b/test/coverage/default.nix new file mode 100644 index 0000000000..fb0b041901 --- /dev/null +++ b/test/coverage/default.nix @@ -0,0 +1,127 @@ +{ stdenv, cabal-install, cabalProject', stackProject', recurseIntoAttrs, runCommand, testSrc, compiler-nix-name }: + +with stdenv.lib; + +let + projectArgs = { + src = testSrc "coverage"; + inherit compiler-nix-name; + modules = [{ + # Package has no exposed modules which causes + # haddock: No input file(s) + packages.bytestring-builder.doHaddock = false; + + # Coverage + packages.pkga.components.library.doCoverage = true; + packages.pkgb.components.library.doCoverage = true; + }]; + }; + + cabalProj = (cabalProject' projectArgs); + stackProj = (stackProject' projectArgs); + +in recurseIntoAttrs ({ + run = stdenv.mkDerivation { + name = "coverage-test"; + + buildCommand = '' + ######################################################################## + # test coverage reports with an example project + + fileExistsNonEmpty() { + local file=$1 + if [ ! -f "$file" ]; then + echo "Missing: $file" + exit 1 + fi + local filesize=$(command stat --format '%s' "$file") + if [ $filesize -eq 0 ]; then + echo "File must not be empty: $file" + exit 1 + fi + } + findFileExistsNonEmpty() { + local searchDir=$1 + local filePattern=$2 + + local file="$(find $searchDir -name $filePattern -print -quit)" + + if [ -z $file ]; then + echo "Couldn't find file \"$filePattern\" in directory \"$searchDir\"." + exit 1 + fi + + local filesize=$(command stat --format '%s' "$file") + if [ $filesize -eq 0 ]; then + echo "File must not be empty: $file" + exit 1 + fi + } + dirExistsEmpty() { + local dir=$1 + if [ ! -d "$dir" ]; then + echo "Missing: $dir" + exit 1 + fi + if [ "$(ls -A $dir)" ]; then + echo "Dir should be empty: $dir" + exit 1 + fi + } + dirExists() { + local dir=$1 + if [ ! -d "$dir" ]; then + echo "Missing: $dir" + exit 1 + fi + } + + ${concatStringsSep "\n" (map (project: '' + pkga_basedir="${project.hsPkgs.pkga.coverageReport}/share/hpc/vanilla" + findFileExistsNonEmpty "$pkga_basedir/mix/pkga-0.1.0.0/" "PkgA.mix" + dirExistsEmpty "$pkga_basedir/tix/pkga-0.1.0.0" + dirExistsEmpty "$pkga_basedir/html/pkga-0.1.0.0" + + pkgb_basedir="${project.hsPkgs.pkgb.coverageReport}/share/hpc/vanilla" + testTix="$pkgb_basedir/tix/pkgb-0.1.0.0/tests/tests.tix" + libTix="$pkgb_basedir/tix/pkgb-0.1.0.0/pkgb-0.1.0.0.tix" + fileExistsNonEmpty "$testTix" + fileExistsNonEmpty "$libTix" + findFileExistsNonEmpty "$pkgb_basedir/mix/pkgb-0.1.0.0/" "ConduitExample.mix" + findFileExistsNonEmpty "$pkgb_basedir/mix/pkgb-0.1.0.0/" "PkgB.mix" + fileExistsNonEmpty "$pkgb_basedir/html/pkgb-0.1.0.0/hpc_index.html" + + filesizeTestsTix=$(command stat --format '%s' "$testTix") + filesizeLibTix=$(command stat --format '%s' "$libTix") + if (( filesizeTestsTix <= filesizeLibTix )); then + echo "Filesize of \"$testTix\" ($filesizeTestsTix) should be greather than that of \"$libTix\" ($filesizeLibTix). Did you forget to exclude test modules when creating \"$libTix\"?" + exit 1 + fi + + project_basedir="${project.projectCoverageReport}/share/hpc/vanilla" + fileExistsNonEmpty "$project_basedir/html/index.html" + dirExists "$project_basedir/html/pkga-0.1.0.0" + dirExists "$project_basedir/html/pkgb-0.1.0.0" + findFileExistsNonEmpty "$project_basedir/mix/" "PkgA.mix" + findFileExistsNonEmpty "$project_basedir/mix/" "PkgB.mix" + findFileExistsNonEmpty "$project_basedir/mix/" "ConduitExample.mix" + dirExists "$project_basedir/tix/all" + fileExistsNonEmpty "$project_basedir/tix/all/all.tix" + dirExists "$project_basedir/tix/pkga-0.1.0.0" + dirExists "$project_basedir/tix/pkgb-0.1.0.0" + fileExistsNonEmpty "$project_basedir/tix/pkgb-0.1.0.0/pkgb-0.1.0.0.tix" + dirExists "$project_basedir/tix/pkgb-0.1.0.0/tests" + fileExistsNonEmpty "$project_basedir/tix/pkgb-0.1.0.0/tests/tests.tix" + '') [ cabalProj stackProj ])} + + touch $out + ''; + + meta.platforms = platforms.all; + + passthru = { + # Used for debugging with nix repl + inherit cabalProj stackProj; + }; + }; +}) diff --git a/test/coverage/pkga/MainA.hs b/test/coverage/pkga/MainA.hs new file mode 100644 index 0000000000..c9ba2faf0c --- /dev/null +++ b/test/coverage/pkga/MainA.hs @@ -0,0 +1,4 @@ +module Main where + +main :: IO () +main = putStrLn "This is MainA" diff --git a/test/coverage/pkga/PkgA.hs b/test/coverage/pkga/PkgA.hs new file mode 100644 index 0000000000..0acf8dbfbd --- /dev/null +++ b/test/coverage/pkga/PkgA.hs @@ -0,0 +1,16 @@ +module PkgA (decode) where + +import Control.Lens +import Data.Text.Lens +import Data.Char +import Data.Text (Text) + +decode :: Text -> Text +decode = unpacked . mapped %~ rot 13 + +rot :: Int -> Char -> Char +rot n c | c >= 'a' && c <= 'z' = r 'a' 'z' + | c >= 'A' && c <= 'Z' = r 'A' 'Z' + | otherwise = c + where + r a b = chr $ ord a + ((ord c - ord a + n) `mod` (ord b - ord a + 1)) diff --git a/test/coverage/pkga/Setup.hs b/test/coverage/pkga/Setup.hs new file mode 100644 index 0000000000..9a994af677 --- /dev/null +++ b/test/coverage/pkga/Setup.hs @@ -0,0 +1,2 @@ +import Distribution.Simple +main = defaultMain diff --git a/test/coverage/pkga/pkga.cabal b/test/coverage/pkga/pkga.cabal new file mode 100644 index 0000000000..1704efde31 --- /dev/null +++ b/test/coverage/pkga/pkga.cabal @@ -0,0 +1,26 @@ +cabal-version: 2.2 +-- Initial package description 'pkga.cabal' generated by 'cabal init'. For +-- further documentation, see http://haskell.org/cabal/users-guide/ + +name: pkga +version: 0.1.0.0 +-- synopsis: +-- description: +-- bug-reports: +license: LicenseRef-PublicDomain +author: Rodney Lorrimar +maintainer: rodney.lorrimar@iohk.io +category: Testing + +library + exposed-modules: PkgA + build-depends: base + , lens + , text + default-language: Haskell2010 + +executable pkga-exe + main-is: MainA.hs + build-depends: base + hs-source-dirs: . + default-language: Haskell2010 diff --git a/test/coverage/pkgb/Setup.hs b/test/coverage/pkgb/Setup.hs new file mode 100644 index 0000000000..9a994af677 --- /dev/null +++ b/test/coverage/pkgb/Setup.hs @@ -0,0 +1,2 @@ +import Distribution.Simple +main = defaultMain diff --git a/test/coverage/pkgb/app/Main.hs b/test/coverage/pkgb/app/Main.hs new file mode 100644 index 0000000000..9d381209ca --- /dev/null +++ b/test/coverage/pkgb/app/Main.hs @@ -0,0 +1,10 @@ +module Main where + +import ConduitExample (example) +import PkgB (message) +import qualified Data.Text.IO as T + +main :: IO () +main = do + T.putStrLn message + example diff --git a/test/coverage/pkgb/app/tests.hs b/test/coverage/pkgb/app/tests.hs new file mode 100644 index 0000000000..70e2077fd8 --- /dev/null +++ b/test/coverage/pkgb/app/tests.hs @@ -0,0 +1,7 @@ +module Main where + +import System.Process +import ConduitExample + +main :: IO () +main = example diff --git a/test/coverage/pkgb/pkgb.cabal b/test/coverage/pkgb/pkgb.cabal new file mode 100644 index 0000000000..17820f5ce6 --- /dev/null +++ b/test/coverage/pkgb/pkgb.cabal @@ -0,0 +1,43 @@ +cabal-version: 2.2 +-- Initial package description 'pkgb.cabal' generated by 'cabal init'. For +-- further documentation, see http://haskell.org/cabal/users-guide/ + +name: pkgb +version: 0.1.0.0 +-- synopsis: +-- description: +-- bug-reports: +license: LicenseRef-PublicDomain +author: Rodney Lorrimar +maintainer: rodney.lorrimar@iohk.io +category: Testing + +library + exposed-modules: ConduitExample + , PkgB + build-depends: base + , pkga + , conduit + , conduit-extra + , directory + , resourcet + hs-source-dirs: src + default-language: Haskell2010 + +executable pkgb + main-is: Main.hs + build-depends: base + , pkgb + , optparse-applicative + , text + hs-source-dirs: app + default-language: Haskell2010 + +test-suite tests + type: exitcode-stdio-1.0 + main-is: tests.hs + hs-source-dirs: app + build-depends: base + , pkgb + , process + build-tools: pkga diff --git a/test/coverage/pkgb/src/ConduitExample.hs b/test/coverage/pkgb/src/ConduitExample.hs new file mode 100644 index 0000000000..a1b1da3955 --- /dev/null +++ b/test/coverage/pkgb/src/ConduitExample.hs @@ -0,0 +1,21 @@ +-- https://github.com/snoyberg/conduit#readme + +module ConduitExample (example) where + +import Conduit +import System.Directory (removeFile) + +example = do + -- Pure operations: summing numbers. + print $ runConduitPure $ yieldMany [1..10] .| sumC + + -- Exception safe file access: copy a file. + writeFile "input.txt" "This is a test." -- create the source file + runConduitRes $ sourceFileBS "input.txt" .| sinkFile "output.txt" -- actual copying + readFile "output.txt" >>= putStrLn -- prove that it worked + + -- Perform transformations. + print $ runConduitPure $ yieldMany [1..10] .| mapC (+ 1) .| sinkList + + removeFile "input.txt" + removeFile "output.txt" diff --git a/test/coverage/pkgb/src/PkgB.hs b/test/coverage/pkgb/src/PkgB.hs new file mode 100644 index 0000000000..1802960dd7 --- /dev/null +++ b/test/coverage/pkgb/src/PkgB.hs @@ -0,0 +1,7 @@ +{-# LANGUAGE OverloadedStrings #-} + +module PkgB (message) where + +import PkgA (decode) + +message = decode "Guvf vf n pnony cebwrpg!" diff --git a/test/coverage/pkgb/src/conduit-test.hs b/test/coverage/pkgb/src/conduit-test.hs new file mode 100644 index 0000000000..45a0361579 --- /dev/null +++ b/test/coverage/pkgb/src/conduit-test.hs @@ -0,0 +1,5 @@ +module Main where + +import ConduitExample + +main = example diff --git a/test/coverage/stack.yaml b/test/coverage/stack.yaml new file mode 100644 index 0000000000..a6509114e9 --- /dev/null +++ b/test/coverage/stack.yaml @@ -0,0 +1,5 @@ +resolver: lts-14.13 + +packages: +- pkga/ +- pkgb/ \ No newline at end of file diff --git a/test/default.nix b/test/default.nix index 9a61d0a94c..5c59132a1d 100644 --- a/test/default.nix +++ b/test/default.nix @@ -181,6 +181,7 @@ let hls-stack = callTest ./haskell-language-server/stack.nix { inherit compiler-nix-name; }; cabal-hpack = callTest ./cabal-hpack { inherit util compiler-nix-name; }; index-state = callTest ./index-state { inherit compiler-nix-name; }; + coverage = callTest ./coverage { inherit compiler-nix-name; }; unit = unitTests; } // lib.optionalAttrs (!stdenv.hostPlatform.isGhcjs && compiler-nix-name != "ghc8101" && compiler-nix-name != "ghc8102" ) { diff --git a/test/exe-only/default.nix b/test/exe-only/default.nix index d30a5cf9ad..086dc61df0 100644 --- a/test/exe-only/default.nix +++ b/test/exe-only/default.nix @@ -26,7 +26,7 @@ in recurseIntoAttrs { # fixme: run on target platform when cross-compiled printf "checking whether executable ran... " >& 2 - cat ${haskellLib.check packages.exe-only.components.exes.exe-only} + cat ${haskellLib.check packages.exe-only.components.exes.exe-only}/test-stdout '' + # Aarch are statically linked and does not have ldd for these tests. optionalString (!stdenv.hostPlatform.isAarch32 && !stdenv.hostPlatform.isAarch64) ( diff --git a/test/extra-hackage/default.nix b/test/extra-hackage/default.nix index 6665754f1e..ad44c3e223 100644 --- a/test/extra-hackage/default.nix +++ b/test/extra-hackage/default.nix @@ -41,7 +41,7 @@ in recurseIntoAttrs { printf "size of executable $exe is $size. \n" >& 2 # fixme: run on target platform when cross-compiled printf "checking whether executable runs... " >& 2 - cat ${haskellLib.check packages.external-package-user.components.exes.external-package-user} + cat ${haskellLib.check packages.external-package-user.components.exes.external-package-user}/test-stdout '' + (if stdenv.hostPlatform.isMusl then '' printf "checking that executable is statically linked... " >& 2 diff --git a/test/ghc-options/cabal.nix b/test/ghc-options/cabal.nix index 2f1463ae22..dec691d942 100644 --- a/test/ghc-options/cabal.nix +++ b/test/ghc-options/cabal.nix @@ -21,7 +21,7 @@ in recurseIntoAttrs { buildCommand = '' printf "checking whether executable runs... " >& 2 - cat ${haskellLib.check packages.test-ghc-options.components.exes.test-ghc-options-exe} + cat ${haskellLib.check packages.test-ghc-options.components.exes.test-ghc-options-exe}/test-stdout touch $out ''; diff --git a/test/ghc-options/stack.nix b/test/ghc-options/stack.nix index 9ff5381959..a451c2e481 100644 --- a/test/ghc-options/stack.nix +++ b/test/ghc-options/stack.nix @@ -22,7 +22,7 @@ in recurseIntoAttrs { buildCommand = '' printf "checking whether executable runs... " >& 2 - cat ${haskellLib.check packages.test-ghc-options.components.exes.test-ghc-options-exe} + cat ${haskellLib.check packages.test-ghc-options.components.exes.test-ghc-options-exe}/test-stdout echo '${concatStringsSep " " packageNames}' > $out ''; diff --git a/test/project-flags/cabal.nix b/test/project-flags/cabal.nix index 09cb224088..977017460d 100644 --- a/test/project-flags/cabal.nix +++ b/test/project-flags/cabal.nix @@ -21,7 +21,7 @@ in recurseIntoAttrs { exe="${packages.test-project-flags.components.exes.test-project-flags-exe}/bin/test-project-flags-exe${stdenv.hostPlatform.extensions.executable}" printf "checking whether executable runs... " >& 2 - cat ${haskellLib.check packages.test-project-flags.components.exes.test-project-flags-exe} + cat ${haskellLib.check packages.test-project-flags.components.exes.test-project-flags-exe}/test-stdout touch $out ''; diff --git a/test/project-flags/stack.nix b/test/project-flags/stack.nix index db0c58e408..42a4930cec 100644 --- a/test/project-flags/stack.nix +++ b/test/project-flags/stack.nix @@ -19,7 +19,7 @@ in recurseIntoAttrs { exe="${packages.test-project-flags.components.exes.test-project-flags-exe}/bin/test-project-flags-exe${stdenv.hostPlatform.extensions.executable}" printf "checking whether executable runs... " >& 2 - cat ${haskellLib.check packages.test-project-flags.components.exes.test-project-flags-exe} + cat ${haskellLib.check packages.test-project-flags.components.exes.test-project-flags-exe}/test-stdout touch $out ''; diff --git a/test/sublib-docs/default.nix b/test/sublib-docs/default.nix index 1d128ea6cf..30d50dfdb2 100644 --- a/test/sublib-docs/default.nix +++ b/test/sublib-docs/default.nix @@ -26,7 +26,7 @@ in recurseIntoAttrs { # fixme: run on target platform when cross-compiled printf "checking whether executable runs... " >& 2 - cat ${haskellLib.check packages.sublib-docs.components.exes.sublib-docs} + cat ${haskellLib.check packages.sublib-docs.components.exes.sublib-docs}/test-stdout '' + # Musl and Aarch are statically linked..