Skip to content

Downloadable hash library in HTML summary #138

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Feb 21, 2022
Merged

Downloadable hash library in HTML summary #138

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
7027739
Show baseline image in HTML when generating
ConorMacBride Feb 14, 2022
d4746bc
Clarify what happens when generating baseline hashes
ConorMacBride Feb 14, 2022
d19a995
Save a result hash library to results dir
ConorMacBride Feb 17, 2022
7e9af62
Remove incorrect warning
ConorMacBride Feb 17, 2022
f8b473c
Test options for generating
ConorMacBride Feb 17, 2022
6388f6f
Download hash library in HTML
ConorMacBride Feb 17, 2022
113acba
Refactor image_status and hash_status to plugin.py
ConorMacBride Feb 19, 2022
39b8d83
Improve hiding of irrelevant info
ConorMacBride Feb 19, 2022
0b63114
Add hashes download to navbar
ConorMacBride Feb 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGES.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
0.15 (unreleased)
-----------------

- An updated hash library will be saved to the results directory when
generating a HTML summary page or when the `--mpl-results-always` flag is
set. A button to download this file is included in the HTML summary.
Various bugfixes, test improvements and documentation updates. [#138]

0.14 (2022-02-09)
-----------------
Expand Down
10 changes: 10 additions & 0 deletions README.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,10 @@ can either be specified via the ``--mpl-hash-library=`` command line argument,
or via the ``hash_library=`` keyword argument to the
``@pytest.mark.mpl_image_compare`` decorator.

When generating a hash library, the tests will also be run as usual against the
existing hash library specified by ``--mpl-hash-library`` or the keyword argument.
However, generating baseline images will always result in the tests being skipped.


Hybrid Mode: Hashes and Images
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Expand Down Expand Up @@ -225,6 +229,12 @@ test (based on the hash library) also shown in the generated
summary. This option is applied automatically when generating
a HTML summary.

When the ``--mpl-results-always`` option is active, and some hash
comparison tests are performed, a hash library containing all the
result hashes will also be saved to the root of the results directory.
The filename will be extracted from ``--mpl-generate-hash-library``,
``--mpl-hash-library`` or ``hash_library=`` in that order.

Base style
^^^^^^^^^^

Expand Down
74 changes: 58 additions & 16 deletions pytest_mpl/plugin.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,8 +181,6 @@ def pytest_configure(config):
if generate_dir is not None:
if baseline_dir is not None:
warnings.warn("Ignoring --mpl-baseline-path since --mpl-generate-path is set")
if results_dir is not None and generate_dir is not None:
warnings.warn("Ignoring --mpl-result-path since --mpl-generate-path is set")

if baseline_dir is not None and not baseline_dir.startswith(("https", "http")):
baseline_dir = os.path.abspath(baseline_dir)
Expand Down Expand Up @@ -283,6 +281,12 @@ def __init__(self,
self.results_dir = Path(tempfile.mkdtemp(dir=self.results_dir))
self.results_dir.mkdir(parents=True, exist_ok=True)

# Decide what to call the downloadable results hash library
if self.hash_library is not None:
self.results_hash_library_name = self.hash_library.name
else: # Use the first filename encountered in a `hash_library=` kwarg
self.results_hash_library_name = None

# We need global state to store all the hashes generated over the run
self._generated_hash_library = {}
self._test_results = {}
Expand Down Expand Up @@ -390,11 +394,14 @@ def generate_baseline_image(self, item, fig):
if not os.path.exists(self.generate_dir):
os.makedirs(self.generate_dir)

fig.savefig(str((self.generate_dir / self.generate_filename(item)).absolute()),
**savefig_kwargs)
baseline_filename = self.generate_filename(item)
baseline_path = (self.generate_dir / baseline_filename).absolute()
fig.savefig(str(baseline_path), **savefig_kwargs)

close_mpl_figure(fig)

return baseline_path

def generate_image_hash(self, item, fig):
"""
For a `matplotlib.figure.Figure`, returns the SHA256 hash as a hexadecimal
Expand Down Expand Up @@ -435,6 +442,7 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None):

if not os.path.exists(baseline_image_ref):
summary['status'] = 'failed'
summary['image_status'] = 'missing'
error_message = ("Image file not found for comparison test in: \n\t"
f"{self.get_baseline_directory(item)}\n"
"(This is expected for new tests.)\n"
Expand All @@ -456,6 +464,7 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None):
actual_shape = imread(str(test_image)).shape[:2]
if expected_shape != actual_shape:
summary['status'] = 'failed'
summary['image_status'] = 'diff'
error_message = SHAPE_MISMATCH_ERROR.format(expected_path=baseline_image,
expected_shape=expected_shape,
actual_path=test_image,
Expand All @@ -467,10 +476,12 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None):
summary['tolerance'] = tolerance
if results is None:
summary['status'] = 'passed'
summary['image_status'] = 'match'
summary['status_msg'] = 'Image comparison passed.'
return None
else:
summary['status'] = 'failed'
summary['image_status'] = 'diff'
summary['rms'] = results['rms']
diff_image = (result_dir / 'result-failed-diff.png').absolute()
summary['diff_image'] = diff_image.relative_to(self.results_dir).as_posix()
Expand All @@ -496,6 +507,10 @@ def compare_image_to_hash_library(self, item, fig, result_dir, summary=None):
compare = self.get_compare(item)
savefig_kwargs = compare.kwargs.get('savefig_kwargs', {})

if not self.results_hash_library_name:
# Use hash library name of current test as results hash library name
self.results_hash_library_name = Path(compare.kwargs.get("hash_library", "")).name

hash_library_filename = self.hash_library or compare.kwargs.get('hash_library', None)
hash_library_filename = (Path(item.fspath).parent / hash_library_filename).absolute()

Expand All @@ -512,14 +527,17 @@ def compare_image_to_hash_library(self, item, fig, result_dir, summary=None):

if baseline_hash is None: # hash-missing
summary['status'] = 'failed'
summary['hash_status'] = 'missing'
summary['status_msg'] = (f"Hash for test '{hash_name}' not found in {hash_library_filename}. "
f"Generated hash is {test_hash}.")
elif test_hash == baseline_hash: # hash-match
hash_comparison_pass = True
summary['status'] = 'passed'
summary['hash_status'] = 'match'
summary['status_msg'] = 'Test hash matches baseline hash.'
else: # hash-diff
summary['status'] = 'failed'
summary['hash_status'] = 'diff'
summary['status_msg'] = (f"Hash {test_hash} doesn't match hash "
f"{baseline_hash} in library "
f"{hash_library_filename} for test {hash_name}.")
Expand All @@ -544,7 +562,8 @@ def compare_image_to_hash_library(self, item, fig, result_dir, summary=None):
except Exception as baseline_error: # Append to test error later
baseline_comparison = str(baseline_error)
else: # Update main summary
for k in ['baseline_image', 'diff_image', 'rms', 'tolerance', 'result_image']:
for k in ['image_status', 'baseline_image', 'diff_image',
'rms', 'tolerance', 'result_image']:
summary[k] = summary[k] or baseline_summary.get(k)

# Append the log from image comparison
Expand Down Expand Up @@ -597,9 +616,12 @@ def item_function_wrapper(*args, **kwargs):
remove_ticks_and_titles(fig)

test_name = self.generate_test_name(item)
result_dir = self.make_test_results_dir(item)

summary = {
'status': None,
'image_status': None,
'hash_status': None,
'status_msg': None,
'baseline_image': None,
'diff_image': None,
Expand All @@ -614,21 +636,23 @@ def item_function_wrapper(*args, **kwargs):
# reference images or simply running the test.
if self.generate_dir is not None:
summary['status'] = 'skipped'
summary['image_status'] = 'generated'
summary['status_msg'] = 'Skipped test, since generating image.'
self.generate_baseline_image(item, fig)
if self.generate_hash_library is None:
self._test_results[str(pathify(test_name))] = summary
pytest.skip("Skipping test, since generating image.")
generate_image = self.generate_baseline_image(item, fig)
if self.results_always: # Make baseline image available in HTML
result_image = (result_dir / "baseline.png").absolute()
shutil.copy(generate_image, result_image)
summary['baseline_image'] = \
result_image.relative_to(self.results_dir).as_posix()

if self.generate_hash_library is not None:
summary['hash_status'] = 'generated'
image_hash = self.generate_image_hash(item, fig)
self._generated_hash_library[test_name] = image_hash
summary['result_hash'] = image_hash
summary['baseline_hash'] = image_hash

# Only test figures if not generating images
if self.generate_dir is None:
result_dir = self.make_test_results_dir(item)

# Compare to hash library
if self.hash_library or compare.kwargs.get('hash_library', None):
msg = self.compare_image_to_hash_library(item, fig, result_dir, summary=summary)
Expand All @@ -645,12 +669,15 @@ def item_function_wrapper(*args, **kwargs):
for image_type in ['baseline_image', 'diff_image', 'result_image']:
summary[image_type] = None # image no longer exists
else:
self._test_results[str(pathify(test_name))] = summary
self._test_results[test_name] = summary
pytest.fail(msg, pytrace=False)

close_mpl_figure(fig)

self._test_results[str(pathify(test_name))] = summary
self._test_results[test_name] = summary

if summary['status'] == 'skipped':
pytest.skip(summary['status_msg'])

if item.cls is not None:
setattr(item.cls, item.function.__name__, item_function_wrapper)
Expand All @@ -667,21 +694,36 @@ def pytest_unconfigure(self, config):
"""
Save out the hash library at the end of the run.
"""
result_hash_library = self.results_dir / (self.results_hash_library_name or "temp.json")
if self.generate_hash_library is not None:
hash_library_path = Path(config.rootdir) / self.generate_hash_library
hash_library_path.parent.mkdir(parents=True, exist_ok=True)
with open(hash_library_path, "w") as fp:
json.dump(self._generated_hash_library, fp, indent=2)
if self.results_always: # Make accessible in results directory
# Use same name as generated
result_hash_library = self.results_dir / hash_library_path.name
shutil.copy(hash_library_path, result_hash_library)
elif self.results_always and self.results_hash_library_name:
result_hashes = {k: v['result_hash'] for k, v in self._test_results.items()
if v['result_hash']}
if len(result_hashes) > 0: # At least one hash comparison test
with open(result_hash_library, "w") as fp:
json.dump(result_hashes, fp, indent=2)

if self.generate_summary:
kwargs = {}
if 'json' in self.generate_summary:
summary = self.generate_summary_json()
print(f"A JSON report can be found at: {summary}")
if result_hash_library.exists(): # link to it in the HTML
kwargs["hash_library"] = result_hash_library.name
if 'html' in self.generate_summary:
summary = generate_summary_html(self._test_results, self.results_dir)
summary = generate_summary_html(self._test_results, self.results_dir, **kwargs)
print(f"A summary of the failed tests can be found at: {summary}")
if 'basic-html' in self.generate_summary:
summary = generate_summary_basic_html(self._test_results, self.results_dir)
summary = generate_summary_basic_html(self._test_results, self.results_dir,
**kwargs)
print(f"A summary of the failed tests can be found at: {summary}")


Expand Down
Loading