Skip to content

Commit f8b473c

Browse files
ConorMacBrideConorMacBride
committed
Test options for generating
1 parent 7e9af62 commit f8b473c

9 files changed

+1014
-5
lines changed

‎pytest_mpl/plugin.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -689,7 +689,8 @@ def pytest_unconfigure(self, config):
689689
with open(hash_library_path, "w") as fp:
690690
json.dump(self._generated_hash_library, fp, indent=2)
691691
if self.results_always: # Make accessible in results directory
692-
result_hash_library.name = hash_library_path.name # use same name as generated
692+
# Use same name as generated
693+
result_hash_library = self.results_dir / hash_library_path.name
693694
shutil.copy(hash_library_path, result_hash_library)
694695
elif self.results_always and self.results_hash_library_name:
695696
result_hashes = {k: v['result_hash'] for k, v in self._test_results.items()

‎tests/subtests/helpers.py

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@ class MatchError(Exception):
99
pass
1010

1111

12-
def diff_summary(baseline, result, baseline_hash_library=None, result_hash_library=None):
12+
def diff_summary(baseline, result, baseline_hash_library=None, result_hash_library=None,
13+
generating_hashes=False):
1314
"""Diff a pytest-mpl summary dictionary.
1415
1516
Parameters
@@ -26,6 +27,9 @@ def diff_summary(baseline, result, baseline_hash_library=None, result_hash_libra
2627
Path to the "baseline" image hash library.
2728
Result hashes in the baseline summary are updated to these values
2829
to handle different Matplotlib versions.
30+
generating_hashes : bool, optional, default=False
31+
Whether `--mpl-generate-hash-library` was specified and
32+
both of `--mpl-hash-library` and `hash_library=` were not.
2933
"""
3034
if baseline_hash_library and baseline_hash_library.exists():
3135
# Load "correct" baseline hashes
@@ -57,10 +61,13 @@ def diff_summary(baseline, result, baseline_hash_library=None, result_hash_libra
5761

5862
# Swap the baseline and result hashes in the summary
5963
# for the corresponding hashes in each hash library
60-
if baseline_hash_library and test in baseline_hash_library:
64+
if baseline_hash_library and test in baseline_hash_library and not generating_hashes:
6165
baseline_summary = replace_hash(baseline_summary, 'baseline_hash',
6266
baseline_hash_library[test])
6367
if result_hash_library:
68+
if generating_hashes: # Newly generate result will appear as baseline_hash
69+
baseline_summary = replace_hash(baseline_summary, 'baseline_hash',
70+
result_hash_library[test])
6471
baseline_summary = replace_hash(baseline_summary, 'result_hash',
6572
result_hash_library[test])
6673

‎tests/subtests/summaries/test_generate.json

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
{
2+
"subtests.subtest.test_hmatch_imatch": {
3+
"status": "skipped",
4+
"status_msg": "Skipped test, since generating image.",
5+
"baseline_image": null,
6+
"diff_image": null,
7+
"rms": null,
8+
"tolerance": null,
9+
"result_image": null,
10+
"baseline_hash": "4a47c9b7920779cc83eabe2bbb64b9c40745d9d8abfa57857f93a5d8f12a5a03",
11+
"result_hash": null
12+
},
13+
"subtests.subtest.test_hmatch_idiff": {
14+
"status": "skipped",
15+
"status_msg": "Skipped test, since generating image.",
16+
"baseline_image": null,
17+
"diff_image": null,
18+
"rms": null,
19+
"tolerance": null,
20+
"result_image": null,
21+
"baseline_hash": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa",
22+
"result_hash": null
23+
},
24+
"subtests.subtest.test_hmatch_imissing": {
25+
"status": "skipped",
26+
"status_msg": "Skipped test, since generating image.",
27+
"baseline_image": null,
28+
"diff_image": null,
29+
"rms": null,
30+
"tolerance": null,
31+
"result_image": null,
32+
"baseline_hash": "e937fa1997d088c904ca35b1ab542e2285ea47b84df976490380f9c5f5b5f8ae",
33+
"result_hash": null
34+
},
35+
"subtests.subtest.test_hdiff_imatch": {
36+
"status": "skipped",
37+
"status_msg": "Skipped test, since generating image.",
38+
"baseline_image": null,
39+
"diff_image": null,
40+
"rms": null,
41+
"tolerance": null,
42+
"result_image": null,
43+
"baseline_hash": "2cae8f315d44b06de8f45d937af46a67bd1389edd6e4cde32f9feb4b7472284f",
44+
"result_hash": null
45+
},
46+
"subtests.subtest.test_hdiff_idiff": {
47+
"status": "skipped",
48+
"status_msg": "Skipped test, since generating image.",
49+
"baseline_image": null,
50+
"diff_image": null,
51+
"rms": null,
52+
"tolerance": null,
53+
"result_image": null,
54+
"baseline_hash": "927521206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde",
55+
"result_hash": null
56+
},
57+
"subtests.subtest.test_hdiff_imissing": {
58+
"status": "skipped",
59+
"status_msg": "Skipped test, since generating image.",
60+
"baseline_image": null,
61+
"diff_image": null,
62+
"rms": null,
63+
"tolerance": null,
64+
"result_image": null,
65+
"baseline_hash": "afc411cfa34db3a5819ac4127704e86acf27d24d1ea2410718853d3d7e1d6ae0",
66+
"result_hash": null
67+
},
68+
"subtests.subtest.test_hmissing_imatch": {
69+
"status": "skipped",
70+
"status_msg": "Skipped test, since generating image.",
71+
"baseline_image": null,
72+
"diff_image": null,
73+
"rms": null,
74+
"tolerance": null,
75+
"result_image": null,
76+
"baseline_hash": "7ee8370efdc4b767634d12355657ca4f2460176670c07b31f3fb72cea0e79856",
77+
"result_hash": null
78+
},
79+
"subtests.subtest.test_hmissing_idiff": {
80+
"status": "skipped",
81+
"status_msg": "Skipped test, since generating image.",
82+
"baseline_image": null,
83+
"diff_image": null,
84+
"rms": null,
85+
"tolerance": null,
86+
"result_image": null,
87+
"baseline_hash": "4eeda1d349f4b0f26df97df41ba5410dce2b1c7ed520062d58f3c5f0e3790ebd",
88+
"result_hash": null
89+
},
90+
"subtests.subtest.test_hmissing_imissing": {
91+
"status": "skipped",
92+
"status_msg": "Skipped test, since generating image.",
93+
"baseline_image": null,
94+
"diff_image": null,
95+
"rms": null,
96+
"tolerance": null,
97+
"result_image": null,
98+
"baseline_hash": "5101e60ac100cf2c2f418a0a6a382aae0060339e76718730344f539b61f7dc7e",
99+
"result_hash": null
100+
},
101+
"subtests.subtest.test_hdiff_imatch_tolerance": {
102+
"status": "skipped",
103+
"status_msg": "Skipped test, since generating image.",
104+
"baseline_image": null,
105+
"diff_image": null,
106+
"rms": null,
107+
"tolerance": null,
108+
"result_image": null,
109+
"baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6",
110+
"result_hash": null
111+
},
112+
"subtests.subtest.test_hdiff_idiff_tolerance": {
113+
"status": "skipped",
114+
"status_msg": "Skipped test, since generating image.",
115+
"baseline_image": null,
116+
"diff_image": null,
117+
"rms": null,
118+
"tolerance": null,
119+
"result_image": null,
120+
"baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6",
121+
"result_hash": null
122+
},
123+
"subtests.subtest.test_hdiff_imatch_savefig": {
124+
"status": "skipped",
125+
"status_msg": "Skipped test, since generating image.",
126+
"baseline_image": null,
127+
"diff_image": null,
128+
"rms": null,
129+
"tolerance": null,
130+
"result_image": null,
131+
"baseline_hash": "8864803a4b4026d8c6dc0ab950228793ea255cd9b6c629c39db9e6315a9af6bc",
132+
"result_hash": null
133+
},
134+
"subtests.subtest.test_hdiff_imatch_style": {
135+
"status": "skipped",
136+
"status_msg": "Skipped test, since generating image.",
137+
"baseline_image": null,
138+
"diff_image": null,
139+
"rms": null,
140+
"tolerance": null,
141+
"result_image": null,
142+
"baseline_hash": "e3c8de36c2bad7dca131e4cbbfe229f882b5beec62750fb7da29314fd6a1ff13",
143+
"result_hash": null
144+
},
145+
"subtests.subtest.test_hdiff_imatch_removetext": {
146+
"status": "skipped",
147+
"status_msg": "Skipped test, since generating image.",
148+
"baseline_image": null,
149+
"diff_image": null,
150+
"rms": null,
151+
"tolerance": null,
152+
"result_image": null,
153+
"baseline_hash": "e4c06cf613c6836c1b1202abaae69cf65bc2232a8e31ab1040454bedc8e31e7a",
154+
"result_hash": null
155+
}
156+
}

‎tests/subtests/summaries/test_generate_hashes_only.json

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
{
2+
"subtests.subtest.test_hmatch_imatch": {
3+
"status": "passed",
4+
"status_msg": "Image comparison passed.",
5+
"baseline_image": null,
6+
"diff_image": null,
7+
"rms": null,
8+
"tolerance": 2,
9+
"result_image": null,
10+
"baseline_hash": "4a47c9b7920779cc83eabe2bbb64b9c40745d9d8abfa57857f93a5d8f12a5a03",
11+
"result_hash": null
12+
},
13+
"subtests.subtest.test_hmatch_idiff": {
14+
"status": "failed",
15+
"status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2",
16+
"baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png",
17+
"diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png",
18+
"rms": 11.100353848213828,
19+
"tolerance": 2,
20+
"result_image": "subtests.subtest.test_hmatch_idiff/result.png",
21+
"baseline_hash": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa",
22+
"result_hash": null
23+
},
24+
"subtests.subtest.test_hmatch_imissing": {
25+
"status": "failed",
26+
"status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png",
27+
"baseline_image": null,
28+
"diff_image": null,
29+
"rms": null,
30+
"tolerance": null,
31+
"result_image": "subtests.subtest.test_hmatch_imissing/result.png",
32+
"baseline_hash": "e937fa1997d088c904ca35b1ab542e2285ea47b84df976490380f9c5f5b5f8ae",
33+
"result_hash": null
34+
},
35+
"subtests.subtest.test_hdiff_imatch": {
36+
"status": "passed",
37+
"status_msg": "Image comparison passed.",
38+
"baseline_image": null,
39+
"diff_image": null,
40+
"rms": null,
41+
"tolerance": 2,
42+
"result_image": null,
43+
"baseline_hash": "2cae8f315d44b06de8f45d937af46a67bd1389edd6e4cde32f9feb4b7472284f",
44+
"result_hash": null
45+
},
46+
"subtests.subtest.test_hdiff_idiff": {
47+
"status": "failed",
48+
"status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2",
49+
"baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png",
50+
"diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png",
51+
"rms": 11.182677079602481,
52+
"tolerance": 2,
53+
"result_image": "subtests.subtest.test_hdiff_idiff/result.png",
54+
"baseline_hash": "927521206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde",
55+
"result_hash": null
56+
},
57+
"subtests.subtest.test_hdiff_imissing": {
58+
"status": "failed",
59+
"status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png",
60+
"baseline_image": null,
61+
"diff_image": null,
62+
"rms": null,
63+
"tolerance": null,
64+
"result_image": "subtests.subtest.test_hdiff_imissing/result.png",
65+
"baseline_hash": "afc411cfa34db3a5819ac4127704e86acf27d24d1ea2410718853d3d7e1d6ae0",
66+
"result_hash": null
67+
},
68+
"subtests.subtest.test_hmissing_imatch": {
69+
"status": "passed",
70+
"status_msg": "Image comparison passed.",
71+
"baseline_image": null,
72+
"diff_image": null,
73+
"rms": null,
74+
"tolerance": 2,
75+
"result_image": null,
76+
"baseline_hash": "7ee8370efdc4b767634d12355657ca4f2460176670c07b31f3fb72cea0e79856",
77+
"result_hash": null
78+
},
79+
"subtests.subtest.test_hmissing_idiff": {
80+
"status": "failed",
81+
"status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 12\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2",
82+
"baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png",
83+
"diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png",
84+
"rms": 12.12938597648977,
85+
"tolerance": 2,
86+
"result_image": "subtests.subtest.test_hmissing_idiff/result.png",
87+
"baseline_hash": "4eeda1d349f4b0f26df97df41ba5410dce2b1c7ed520062d58f3c5f0e3790ebd",
88+
"result_hash": null
89+
},
90+
"subtests.subtest.test_hmissing_imissing": {
91+
"status": "failed",
92+
"status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png",
93+
"baseline_image": null,
94+
"diff_image": null,
95+
"rms": null,
96+
"tolerance": null,
97+
"result_image": "subtests.subtest.test_hmissing_imissing/result.png",
98+
"baseline_hash": "5101e60ac100cf2c2f418a0a6a382aae0060339e76718730344f539b61f7dc7e",
99+
"result_hash": null
100+
},
101+
"subtests.subtest.test_hdiff_imatch_tolerance": {
102+
"status": "passed",
103+
"status_msg": "Image comparison passed.",
104+
"baseline_image": null,
105+
"diff_image": null,
106+
"rms": null,
107+
"tolerance": 200,
108+
"result_image": null,
109+
"baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6",
110+
"result_hash": null
111+
},
112+
"subtests.subtest.test_hdiff_idiff_tolerance": {
113+
"status": "failed",
114+
"status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 29\\.2[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3",
115+
"baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png",
116+
"diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png",
117+
"rms": 29.260332173249314,
118+
"tolerance": 3,
119+
"result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png",
120+
"baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6",
121+
"result_hash": null
122+
},
123+
"subtests.subtest.test_hdiff_imatch_savefig": {
124+
"status": "passed",
125+
"status_msg": "Image comparison passed.",
126+
"baseline_image": null,
127+
"diff_image": null,
128+
"rms": null,
129+
"tolerance": 2,
130+
"result_image": null,
131+
"baseline_hash": "8864803a4b4026d8c6dc0ab950228793ea255cd9b6c629c39db9e6315a9af6bc",
132+
"result_hash": null
133+
},
134+
"subtests.subtest.test_hdiff_imatch_style": {
135+
"status": "passed",
136+
"status_msg": "Image comparison passed.",
137+
"baseline_image": null,
138+
"diff_image": null,
139+
"rms": null,
140+
"tolerance": 2,
141+
"result_image": null,
142+
"baseline_hash": "e3c8de36c2bad7dca131e4cbbfe229f882b5beec62750fb7da29314fd6a1ff13",
143+
"result_hash": null
144+
},
145+
"subtests.subtest.test_hdiff_imatch_removetext": {
146+
"status": "passed",
147+
"status_msg": "Image comparison passed.",
148+
"baseline_image": null,
149+
"diff_image": null,
150+
"rms": null,
151+
"tolerance": 2,
152+
"result_image": null,
153+
"baseline_hash": "e4c06cf613c6836c1b1202abaae69cf65bc2232a8e31ab1040454bedc8e31e7a",
154+
"result_hash": null
155+
}
156+
}

0 commit comments

Comments
 (0)