Skip to content

Commit 73a1ddc

Browse files
sevanspowellsevanspowell
committed
Add support for coverage
- Added the ability to generate coverage reports for packages and projects. - Outputs mix and tix information, as well as a HTML report. - Added the "doCoverage" module option that allows users to choose packages to enable coverage for. - Added a "doCoverage" flag to the component builder that outputs HPC information when coverage is enabled. - Added the "overrideModules" library function to make it more ergonomic fo users to enable coverage on existing projects. - Modified the "check" builder to also output ".tix" files (if they exist). This information is required to generate the coverage report. - Added a test for coverage.
1 parent 40d0528 commit 73a1ddc

File tree

40 files changed

+837
-38
lines changed

40 files changed

+837
-38
lines changed

‎builder/comp-builder.nix

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,9 @@ let self =
5252
, enableExecutableProfiling ? component.enableExecutableProfiling
5353
, profilingDetail ? component.profilingDetail
5454

55+
# Coverage
56+
, doCoverage ? component.doCoverage
57+
5558
# Data
5659
, enableSeparateDataOutput ? component.enableSeparateDataOutput
5760

@@ -117,6 +120,7 @@ let
117120
(enableFeature enableExecutableProfiling "executable-profiling")
118121
(enableFeature enableStatic "static")
119122
(enableFeature enableShared "shared")
123+
(enableFeature doCoverage "coverage")
120124
] ++ lib.optionals (stdenv.hostPlatform.isMusl && (haskellLib.isExecutableType componentId)) [
121125
# These flags will make sure the resulting executable is statically linked.
122126
# If it uses other libraries it may be necessary for to add more
@@ -357,6 +361,11 @@ let
357361
fi
358362
done
359363
'')
364+
+ (lib.optionalString doCoverage ''
365+
mkdir -p $out/share
366+
cp -r dist/hpc $out/share
367+
cp dist/setup-config $out/
368+
'')
360369
}
361370
runHook postInstall
362371
'' + (lib.optionalString keepSource ''

‎docs/dev/coverage.md

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
# Developer Coverage Overview
2+
3+
## Building
4+
5+
The implementation of coverage starts with the "doCoverage" flag on
6+
the builder in `comp-builder.nix`. The doCoverage flag enables and
7+
disables the Cabal coverage flag and copies any generated coverage
8+
data to "$out/share/hpc".
9+
10+
## Mix and tix files
11+
12+
The coverage information for any derivation consists of "mix" and
13+
"tix" files.
14+
15+
Mix files record static information about a source file and are
16+
generated at build time. They primarily contain a path to the source
17+
file and information about expressions and regions of the source file,
18+
which are later referenced by tix files.
19+
20+
Tix files contain dynamic information about a test run, recording when
21+
a portion of a source file is touched by a test. These are generated
22+
when the test is run.
23+
24+
## Coverage reports
25+
26+
### Package reports
27+
28+
The coverage information generated will look something like this:
29+
30+
```bash
31+
/nix/store/...-my-project-0.1.0.0-coverage-report/
32+
└── share
33+
└── hpc
34+
└── vanilla
35+
├── mix
36+
│   ├── my-library-0.1.0.0
37+
│   │   └── my-library-0.1.0.0-ERSaOroBZhe9awsoBkhmcV
38+
│   │   ├── My.Lib.Config.mix
39+
│   │   ├── My.Lib.Types.mix
40+
│   │   └── My.Lib.Util.mix
41+
│   └── my-test-1
42+
│   ├── Spec.mix
43+
│   └── Main.mix
44+
├── tix
45+
│ ├── my-library-0.1.0.0
46+
│ │   └── my-library-0.1.0.0.tix
47+
│ └── my-test-1
48+
│ └── my-test-1.tix
49+
└── html
50+
└── my-library-0.1.0.0
51+
├── my-library-0.1.0.0-ERSaOroBZhe9awsoBkhmcV
52+
│ ├── My.Lib.Config.hs.html
53+
│ ├── My.Lib.Types.hs.html
54+
│ └── My.Lib.Util.hs.html
55+
├── hpc_index_alt.html
56+
├── hpc_index_exp.html
57+
├── hpc_index_fun.html
58+
└── hpc_index.html
59+
```
60+
61+
- The mix files are copied verbatim from the builds with coverage.
62+
- The tix files for each test are copied from the check run verbatim.
63+
- The tix files for each library are generated by summing the tix
64+
files for each test, but excluding any test modules.
65+
- Test modules are determined by inspecting the plan for the project
66+
(i.e. for the project "my-project" and test-suite "my-test-1", the
67+
test modules are read from:
68+
`my-project.project.pkg-set.config.packages.my-project.components.tests.my-test-1.modules`)
69+
- The hpc reports for each component are generated from their
70+
respective tix files.
71+
- `html/my-library-0.1.0.0/hpc_index.html` contains coverage
72+
information for the library, excluding any test modules.
73+
74+
### Project-wide reports
75+
76+
The coverage information for an entire project will look something
77+
like this:
78+
79+
```bash
80+
/nix/store/...-coverage-report
81+
└── share
82+
└── hpc
83+
└── vanilla
84+
├── mix
85+
│   ├── my-library-0.1.0.0
86+
│   │   └── my-library-0.1.0.0-ERSaOroBZhe9awsoBkhmcV
87+
│   │   ├── My.Lib.Config.mix
88+
│   │   ├── My.Lib.Types.mix
89+
│   │   └── My.Lib.Util.mix
90+
│   ├── my-test-1
91+
│   │   ├── Spec.mix
92+
│   │   └── Main.mix
93+
│   ├── other-library-0.1.0.0
94+
│   │   └── other-library-0.1.0.0-48EVZBwW9Kj29VTaRMhBDf
95+
│   │   ├── Other.Lib.A.mix
96+
│   │   └── Other.Lib.B.mix
97+
│   └── other-test-1
98+
│   ├── Spec.mix
99+
│   └── Main.mix
100+
├── tix
101+
│ ├── all
102+
│ │   └── all.tix
103+
│ ├── my-library-0.1.0.0
104+
│ │   └── my-library-0.1.0.0.tix
105+
│ ├── my-test-1
106+
│ │   └── my-test-1.tix
107+
│ ├── other-library-0.1.0.0
108+
│ │   └── other-library-0.1.0.0.tix
109+
│ └── other-test-1
110+
│ └── other-test-1.tix
111+
└── html
112+
├── my-library-0.1.0.0
113+
│   ├── my-library-0.1.0.0-ERSaOroBZhe9awsoBkhmcV
114+
│   │ ├── My.Lib.Config.hs.html
115+
│   │ ├── My.Lib.Types.hs.html
116+
│   │ └── My.Lib.Util.hs.html
117+
│   ├── hpc_index_alt.html
118+
│   ├── hpc_index_exp.html
119+
│   ├── hpc_index_fun.html
120+
│   └── hpc_index.html
121+
└── other-libray-0.1.0.0
122+
├── other-library-0.1.0.0-48EVZBwW9Kj29VTaRMhBDf
123+
│   ├── Other.Lib.A.hs.html
124+
│   └── Other.Lib.B.hs.html
125+
├── hpc_index_alt.html
126+
├── hpc_index_exp.html
127+
├── hpc_index_fun.html
128+
└── hpc_index.html
129+
```
130+
131+
All of the coverage information is copied verbatim from the coverage
132+
reports for each of the constituent packages. A few additions are
133+
made:
134+
- `tix/all/all.tix` is generated from the union of all the library
135+
tix files.
136+
- We use this file when generating coverage reports for
137+
"coveralls.io".
138+
- An index page (`html/index.html`) is generated which links to the
139+
HTML coverage reports of the constituent packages.

‎docs/tutorials/coverage.md

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
# Coverage
2+
3+
haskell.nix can generate coverage information for your package or
4+
project using Cabal's inbuilt hpc support.
5+
6+
## Prerequisites
7+
8+
To get a sensible coverage report, you need to enable coverage on each
9+
of the components of your project. We recommend you use the
10+
`overrideModules` function to do this:
11+
12+
```nix
13+
let
14+
inherit (pkgs.haskell-nix) haskellLib;
15+
16+
project = pkgs.haskell-nix.project {
17+
src = pkgs.haskell-nix.haskellLib.cleanGit {
18+
name = "haskell-nix-project";
19+
src = ./.;
20+
};
21+
# For `cabal.project` based projects specify the GHC version to use.
22+
compiler-nix-name = "ghc884"; # Not used for `stack.yaml` based projects.
23+
};
24+
25+
projectWithCoverage = project.overrideModules(oldModules: oldModules ++ [{
26+
packages.$pkg.components.library.doCoverage = true;
27+
packages.$pkg.components.tests.a-test.doCoverage = true;
28+
}]);
29+
30+
in {
31+
inherit project projectWithCoverage;
32+
}
33+
34+
```
35+
36+
## Per-package
37+
38+
```bash
39+
nix-build default.nix -A "projectWithCoverage.$pkg.coverageReport"
40+
```
41+
42+
This will generate a coverage report for the package you requested.
43+
All tests that are enabled (configured with `doCheck == true`) are
44+
included in the coverage report.
45+
46+
See the [developer coverage docs](../dev/coverage.md#package-reports) for more information.
47+
48+
## Project-wide
49+
50+
```bash
51+
nix-build default.nix -A "projectWithCoverage.projectCoverageReport"
52+
```
53+
54+
This will generate a coverage report for all the local packages in
55+
your project.
56+
57+
See the [developer coverage docs](../dev/coverage.md#project-wide-reports) for more information.
58+
59+
## Custom
60+
61+
By default, `projectCoverageReport` generates a coverage report
62+
including all the packages in your project, and `coverageReport`
63+
generates a report for the library and all enabled tests in the
64+
requested package. You can modify what is included in each report by
65+
using the `coverageReport'` and `projectCoverageReport'` functions.
66+
These are found in the haskell.nix library:
67+
68+
```nix
69+
let
70+
inherit (pkgs.haskell-nix) haskellLib;
71+
72+
project = pkgs.haskell-nix.project {
73+
src = pkgs.haskell-nix.haskellLib.cleanGit {
74+
name = "haskell-nix-project";
75+
src = ./.;
76+
};
77+
# For `cabal.project` based projects specify the GHC version to use.
78+
compiler-nix-name = "ghc884"; # Not used for `stack.yaml` based projects.
79+
};
80+
81+
projectWithCoverage = project.overrideModules(oldModules: oldModules ++ [{
82+
packages.$pkg.components.library.doCoverage = true;
83+
packages.$pkg.components.tests.a-test.doCoverage = true;
84+
}]);
85+
86+
# Choose the library and tests you want included in the coverage
87+
# report for a package.
88+
custom$pkgCoverageReport = haskellLib.coverageReport' {
89+
inherit (projectWithCoverage.$pkg.identifier) name version;
90+
inherit (projectWithCoverage.$pkg.components) library tests;
91+
};
92+
93+
# Override the coverage report for a package, and also choose which
94+
# packages you want included in the coverage report.
95+
customProjectCoverageReport = haskellLib.projectCoverageReport' {
96+
packages = haskellLib.selectProjectPackages projectWithCoverage;
97+
coverageReportOverrides = { "${projectWithCoverage.$pkg.identifier.name}" = custom$pkgCoverageReport; };
98+
};
99+
in {
100+
inherit project projectWithCoverage custom$pkgCoverageReport customProjectCoverageReport;
101+
}
102+
103+
```

‎lib/check.nix

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ in stdenv.mkDerivation ({
1515
src = drv.source or (srcOnly drv);
1616

1717
passthru = {
18-
inherit (drv) identifier config configFiles executableToolDepends cleanSrc env;
18+
inherit (drv) identifier config configFiles executableToolDepends cleanSrc env exeName;
1919
};
2020

2121
inherit (drv) meta LANG LC_ALL buildInputs nativeBuildInputs;
@@ -27,11 +27,14 @@ in stdenv.mkDerivation ({
2727
# If doCheck or doCrossCheck are false we may still build this
2828
# component and we want it to quietly succeed.
2929
buildPhase = ''
30-
touch $out
30+
mkdir $out
3131
3232
runHook preCheck
3333
34-
${toString component.testWrapper} ${drv}/bin/${drv.exeName} ${lib.concatStringsSep " " component.testFlags} | tee $out
34+
${toString component.testWrapper} ${drv}/bin/${drv.exeName} ${lib.concatStringsSep " " component.testFlags} | tee $out/test-stdout
35+
36+
# Copy over tix files, if they exist
37+
find . -iname '*.tix' -exec mkdir -p $out/share/hpc/vanilla/tix/${drv.exeName} \; -exec cp {} $out/share/hpc/vanilla/tix/${drv.exeName}/ \;
3538
3639
runHook postCheck
3740
'';

‎lib/cover-project.nix

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
{ pkgs, lib, haskellLib }:
2+
3+
# List of project packages to generate a coverage report for
4+
{ packages
5+
, coverageReportOverrides ? {}
6+
}:
7+
8+
let
9+
getPackageCoverageReport = packageName: (coverageReportOverrides."${packageName}" or packages."${packageName}".coverageReport);
10+
11+
# Create table rows for an project coverage index page that look something like:
12+
#
13+
# | Package |
14+
# |------------------|
15+
# | cardano-shell |
16+
# | cardano-launcher |
17+
packageTableRows = package: with lib;
18+
let
19+
testsOnly = filterAttrs (n: d: isDerivation d) package.components.tests;
20+
testNames = mapAttrsToList (testName: _: testName) testsOnly;
21+
in
22+
concatStringsSep "\n" (map (testName:
23+
''
24+
<tr>
25+
<td>
26+
<a href="${package.identifier.name}-${package.identifier.version}/hpc_index.html">${package.identifier.name}</href>
27+
</td>
28+
</tr>
29+
'') testNames);
30+
31+
projectIndexHtml = pkgs.writeText "index.html" ''
32+
<html>
33+
<head>
34+
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
35+
</head>
36+
<body>
37+
<table border="1" width="100%">
38+
<tbody>
39+
<tr>
40+
<th>Package</th>
41+
</tr>
42+
43+
${with lib; concatStringsSep "\n" (mapAttrsToList (_ : packageTableRows) packages)}
44+
45+
</tbody>
46+
</table>
47+
</body>
48+
</html>
49+
'';
50+
in pkgs.runCommand "project-coverage-report"
51+
{ buildInputs = (with pkgs; [ghc]); }
52+
''
53+
mkdir -p $out/share/hpc/vanilla/tix/all
54+
mkdir -p $out/share/hpc/vanilla/mix/
55+
mkdir -p $out/share/hpc/vanilla/html/
56+
57+
# Find all tix files in each package
58+
tixFiles=()
59+
${with lib; concatStringsSep "\n" (mapAttrsToList (n: package: ''
60+
identifier="${package.identifier.name}-${package.identifier.version}"
61+
report=${getPackageCoverageReport n}
62+
tix="$report/share/hpc/vanilla/tix/$identifier/$identifier.tix"
63+
if test -f "$tix"; then
64+
tixFiles+=("$tix")
65+
fi
66+
67+
# Copy mix and tix information over from each report
68+
cp -R $report/share/hpc/vanilla/mix/* $out/share/hpc/vanilla/mix
69+
cp -R $report/share/hpc/vanilla/tix/* $out/share/hpc/vanilla/tix
70+
cp -R $report/share/hpc/vanilla/html/* $out/share/hpc/vanilla/html
71+
'') packages)}
72+
73+
if [ ''${#tixFiles[@]} -ne 0 ]; then
74+
# Create tix file with test run information for all packages
75+
tixFile="$out/share/hpc/vanilla/tix/all/all.tix"
76+
hpcSumCmd=("hpc" "sum" "--union" "--output=$tixFile")
77+
hpcSumCmd+=("''${tixFiles[@]}")
78+
echo "''${hpcSumCmd[@]}"
79+
eval "''${hpcSumCmd[@]}"
80+
81+
# Markup a HTML coverage report for the entire project
82+
cp ${projectIndexHtml} $out/share/hpc/vanilla/html/index.html
83+
fi
84+
''

0 commit comments

Comments
 (0)