diff --git a/pytest_mpl/kernels.py b/pytest_mpl/kernels.py new file mode 100644 index 00000000..d9fd0e0d --- /dev/null +++ b/pytest_mpl/kernels.py @@ -0,0 +1,252 @@ +""" +This module contains the supported hashing kernel implementations. + +""" +import hashlib +from abc import ABC, abstractmethod + +import imagehash +from PIL import Image + +#: The default hamming distance bit tolerance for "similar" imagehash hashes. +DEFAULT_HAMMING_TOLERANCE = 4 + +#: The default imagehash hash size (N), resulting in a hash of N**2 bits. +DEFAULT_HASH_SIZE = 16 + +#: Level of image detail (high) or structure (low) represented by phash . +DEFAULT_HIGH_FREQUENCY_FACTOR = 4 + +#: Registered kernel names. +KERNEL_PHASH = "phash" +KERNEL_SHA256 = "sha256" + +__all__ = [ + "DEFAULT_HAMMING_TOLERANCE", + "DEFAULT_HASH_SIZE", + "DEFAULT_HIGH_FREQUENCY_FACTOR", + "KERNEL_PHASH", + "KERNEL_SHA256", + "KernelPHash", + "KernelSHA256", + "kernel_factory", +] + + +class Kernel(ABC): + """ + Kernel abstract base class (ABC) which defines a simple common kernel API. + + """ + + def __init__(self, plugin): + # Containment of the plugin allows the kernel to cherry-pick required state. + self._plugin = plugin + + @abstractmethod + def equivalent_hash(self, result, baseline, marker=None): + """ + Determine whether the kernel considers the provided result (actual) + and baseline (expected) hashes as similar. + + Parameters + ---------- + result : str + The hash of the image generated by the test. + baseline : str + The hash of the baseline image. + marker : pytest.Mark + The test marker, which may contain kwarg options to be + applied to the equivalence test. + + Returns + ------- + bool + Whether the result and baseline hashes are deemed similar. + + """ + + @abstractmethod + def generate_hash(self, buffer): + """ + Computes the hash of the image from the in-memory/open byte stream + buffer. + + Parameters + ---------- + buffer : stream + The in-memory/open byte stream of the image. + + Returns + ------- + str + The string representation (hexdigest) of the image hash. + + """ + + def update_status(self, message): + """ + Append the kernel status message to the provided message. + + Parameters + ---------- + message : str + The existing status message. + + Returns + ------- + str + The updated status message. + + """ + return message + + def update_summary(self, summary): + """ + Refresh the image comparison summary with relevant kernel entries. + + Parameters + ---------- + summary : dict + Image comparison test report summary. + + Returns + ------- + None + + """ + # The "name" class property *must* be defined in derived child class. + summary["kernel"] = self.name + + @property + def metadata(self): + """ + The kernel metadata to be archived in a hash library with results. + + Returns + ------- + dict + The kernel metadata. + + """ + return dict(name=self.name) + + +class KernelPHash(Kernel): + """ + Kernel that calculates a perceptual hash of an image for the + specified hash size (N) and high frequency factor. + + Where the resultant perceptual hash will be composed of N**2 bits. + + """ + + name = KERNEL_PHASH + + def __init__(self, plugin): + super().__init__(plugin) + # Keep state of the equivalence result. + self.equivalent = None + # Keep state of hash hamming distance (whole number) result. + self.hamming_distance = None + # Value may be overridden by py.test marker kwarg. + arg = self._plugin.hamming_tolerance + self.hamming_tolerance = ( + int(arg) if arg is not None else DEFAULT_HAMMING_TOLERANCE + ) + # The hash-size (N) defines the resultant N**2 bits hash size. + arg = self._plugin.hash_size + self.hash_size = int(arg) if arg is not None else DEFAULT_HASH_SIZE + # The level of image detail (high freq) or structure (low freq) + # represented in perceptual hash thru discrete cosine transform. + arg = self._plugin.high_freq_factor + self.high_freq_factor = ( + int(arg) if arg is not None else DEFAULT_HIGH_FREQUENCY_FACTOR + ) + # py.test marker kwarg. + self.option = "hamming_tolerance" + + def equivalent_hash(self, result, baseline, marker=None): + if marker: + value = marker.kwargs.get(self.option) + if value is not None: + # Override with the decorator marker value. + self.hamming_tolerance = int(value) + # Convert string hexdigest hashes to imagehash.ImageHash instances. + result = imagehash.hex_to_hash(result) + baseline = imagehash.hex_to_hash(baseline) + # Unlike cryptographic hashes, perceptual hashes can measure the + # degree of "similarity" through hamming distance bit differences + # between the hashes. + try: + self.hamming_distance = result - baseline + self.equivalent = self.hamming_distance <= self.hamming_tolerance + except TypeError: + # imagehash won't compare hashes of different sizes, however + # let's gracefully support this for use-ability. + self.hamming_distance = None + self.equivalent = False + return self.equivalent + + def generate_hash(self, buffer): + buffer.seek(0) + data = Image.open(buffer) + phash = imagehash.phash( + data, hash_size=self.hash_size, highfreq_factor=self.high_freq_factor + ) + return str(phash) + + def update_status(self, message): + result = str() if message is None else str(message) + # Only update the status message for non-equivalent hash comparisons. + if self.equivalent is False: + msg = ( + f"Hash hamming distance of {self.hamming_distance} bits > " + f"hamming tolerance of {self.hamming_tolerance} bits." + ) + result = f"{message} {msg}" if len(result) else msg + return result + + def update_summary(self, summary): + super().update_summary(summary) + summary["hamming_distance"] = self.hamming_distance + summary["hamming_tolerance"] = self.hamming_tolerance + + @property + def metadata(self): + result = super().metadata + result["hash_size"] = self.hash_size + result["high_freq_factor"] = self.high_freq_factor + return result + + +class KernelSHA256(Kernel): + """ + A simple kernel that calculates a 256-bit cryptographic SHA hash + of an image. + + """ + + name = KERNEL_SHA256 + + def equivalent_hash(self, result, baseline, marker=None): + # Simple cryptographic hash binary comparison. Interpretation of + # the comparison result is that the hashes are either identical or + # not identical. For non-identical hashes, it is not possible to + # determine a heuristic of hash "similarity" due to the nature of + # cryptographic hashes. + return result == baseline + + def generate_hash(self, buffer): + buffer.seek(0) + data = buffer.read() + hasher = hashlib.sha256() + hasher.update(data) + return hasher.hexdigest() + + +#: Registry of available hashing kernel factories. +kernel_factory = { + KernelPHash.name: KernelPHash, + KernelSHA256.name: KernelSHA256, +} diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index fd8fe03c..21112b06 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -32,7 +32,6 @@ import os import json import shutil -import hashlib import inspect import logging import tempfile @@ -44,26 +43,56 @@ import pytest -from pytest_mpl.summary.html import generate_summary_basic_html, generate_summary_html +from .kernels import KERNEL_SHA256, kernel_factory +from .summary.html import generate_summary_basic_html, generate_summary_html -SUPPORTED_FORMATS = {'html', 'json', 'basic-html'} +#: The default matplotlib backend. +DEFAULT_BACKEND = "agg" -SHAPE_MISMATCH_ERROR = """Error: Image dimensions did not match. - Expected shape: {expected_shape} - {expected_path} - Actual shape: {actual_shape} - {actual_path}""" +#: The default algorithm to use for image hashing. +DEFAULT_KERNEL = KERNEL_SHA256 +#: The default pytest-mpl marker. +DEFAULT_MARKER = "mpl_image_compare" -def _hash_file(in_stream): - """ - Hashes an already opened file. - """ - in_stream.seek(0) - buf = in_stream.read() - hasher = hashlib.sha256() - hasher.update(buf) - return hasher.hexdigest() +#: The default option to remove text from matplotlib plot. +DEFAULT_REMOVE_TEXT = False + +#: The default image RMS tolerance. +DEFAULT_RMS_TOLERANCE = 2 + +#: The default matplotlib plot style. +DEFAULT_STYLE = "classic" + +#: JSON metadata entry defining the source kernel of the hashes. +META_HASH_KERNEL = 'pytest-mpl-kernel' + +#: Valid formats for generate summary. +SUPPORTED_FORMATS = {'html', 'json', 'basic-html'} + +#: Supported INET protocols. +PROTOCOLS = ('http://', 'https://') + +#: Template error message for image shape conformance. +TEMPLATE_SHAPE_MISMATCH = """Error! Image dimensions did not match. + Baseline Shape: + {baseline_shape} + Baseline Image: + {baseline_image} + Result Shape: + {result_shape} + Result Image: + {result_image}""" + +TEMPLATE_IMAGE_DIFFERENCE = """Failed! Image files did not match. + RMS: {rms} + Tolerance: {tol} + Baseline Image: + {expected} + Result Image: + {actual} + Difference Image: + {diff}""" def pathify(path): @@ -92,48 +121,80 @@ def pytest_report_header(config, startdir): def pytest_addoption(parser): group = parser.getgroup("matplotlib image comparison") - group.addoption('--mpl', action='store_true', - help="Enable comparison of matplotlib figures to reference files") - group.addoption('--mpl-generate-path', - help="directory to generate reference images in, relative " - "to location where py.test is run", action='store') - group.addoption('--mpl-generate-hash-library', - help="filepath to save a generated hash library, relative " - "to location where py.test is run", action='store') - group.addoption('--mpl-baseline-path', - help="directory containing baseline images, relative to " - "location where py.test is run unless --mpl-baseline-relative is given. " - "This can also be a URL or a set of comma-separated URLs (in case " - "mirrors are specified)", action='store') - group.addoption("--mpl-baseline-relative", help="interpret the baseline directory as " - "relative to the test location.", action="store_true") - group.addoption('--mpl-hash-library', - help="json library of image hashes, relative to " - "location where py.test is run", action='store') - group.addoption('--mpl-generate-summary', action='store', - help="Generate a summary report of any failed tests" - ", in --mpl-results-path. The type of the report should be " - "specified. Supported types are `html`, `json` and `basic-html`. " - "Multiple types can be specified separated by commas.") - - results_path_help = "directory for test results, relative to location where py.test is run" - group.addoption('--mpl-results-path', help=results_path_help, action='store') - parser.addini('mpl-results-path', help=results_path_help) - - results_always_help = ("Always compare to baseline images and save result images, even for passing tests. " - "This option is automatically applied when generating a HTML summary.") - group.addoption('--mpl-results-always', action='store_true', - help=results_always_help) - parser.addini('mpl-results-always', help=results_always_help) - - parser.addini('mpl-use-full-test-name', help="use fully qualified test name as the filename.", - type='bool') + + msg = 'Enable comparison of matplotlib figures to reference files.' + group.addoption('--mpl', help=msg, action='store_true') + + msg = ('Directory to generate reference images in, relative to ' + 'location where py.test is run.') + group.addoption('--mpl-generate-path', help=msg, action='store') + + msg = ('Filepath to save a generated hash library, relative to ' + 'location where py.test is run.') + group.addoption('--mpl-generate-hash-library', help=msg, action='store') + + msg = ('Directory containing baseline images, relative to location ' + 'where py.test is run unless --mpl-baseline-relative is given. ' + 'This can also be a URL or a set of comma-separated URLs ' + '(in case mirrors are specified).') + group.addoption('--mpl-baseline-path', help=msg, action='store') + + msg = 'Interpret the baseline directory as relative to the test location.' + group.addoption("--mpl-baseline-relative", help=msg, action="store_true") + + msg = ('JSON library of image hashes, relative to location where ' + 'py.test is run.') + group.addoption('--mpl-hash-library', help=msg, action='store') + + msg = ('Generate a summary report of any failed tests, in ' + '--mpl-results-path. The type of the report should be specified. ' + 'Supported types are `html`, `json` and `basic-html`. ' + 'Multiple types can be specified separated by commas.') + group.addoption('--mpl-generate-summary', help=msg, action='store') + + msg = ('Directory for test results, relative to location where py.test ' + 'is run.') + option = 'mpl-results-path' + group.addoption(f'--{option}', help=msg, action='store') + parser.addini(option, help=msg) + + msg = ('Always compare to baseline images and save result images, even ' + 'for passing tests. This option is automatically applied when ' + 'generating a HTML summary.') + option = 'mpl-results-always' + group.addoption(f'--{option}', help=msg, action='store_true') + parser.addini(option, help=msg) + + msg = 'Use fully qualified test name as the filename.' + parser.addini('mpl-use-full-test-name', help=msg, type='bool') + + msg = ('Algorithm to be used for hashing images. Supported kernels are ' + '`sha256` (default) and `phash`.') + option = 'mpl-kernel' + group.addoption(f'--{option}', help=msg, action='store') + parser.addini(option, help=msg) + + msg = ('Determine the level of image detail (high) or structure (low)' + 'represented in the perceptual hash.') + option = 'mpl-high-freq-factor' + group.addoption(f'--{option}', help=msg, action='store') + parser.addini(option, help=msg) + + msg = 'The hash size (N) used to generate a N**2 bit image hash.' + option = 'mpl-hash-size' + group.addoption(f'--{option}', help=msg, action='store') + parser.addini(option, help=msg) + + msg = 'Hamming distance bit tolerance for similar image hashes.' + option = 'mpl-hamming-tolerance' + group.addoption(f'--{option}', help=msg, action='store') + parser.addini(option, help=msg) def pytest_configure(config): config.addinivalue_line('markers', - "mpl_image_compare: Compares matplotlib figures " + f"{DEFAULT_MARKER}: Compares matplotlib figures " "against a baseline image") if (config.getoption("--mpl") or @@ -143,11 +204,12 @@ def pytest_configure(config): baseline_dir = config.getoption("--mpl-baseline-path") generate_dir = config.getoption("--mpl-generate-path") generate_hash_lib = config.getoption("--mpl-generate-hash-library") - results_dir = config.getoption("--mpl-results-path") or config.getini("mpl-results-path") + option = "mpl-results-path" + results_dir = config.getoption(f"--{option}") or config.getini(option) hash_library = config.getoption("--mpl-hash-library") generate_summary = config.getoption("--mpl-generate-summary") - results_always = (config.getoption("--mpl-results-always") or - config.getini("mpl-results-always")) + option = "mpl-results-always" + results_always = config.getoption(f"--{option}") or config.getini(option) if config.getoption("--mpl-baseline-relative"): baseline_relative_dir = config.getoption("--mpl-baseline-path") @@ -160,24 +222,27 @@ def pytest_configure(config): if generate_dir is not None: if baseline_dir is not None: - warnings.warn("Ignoring --mpl-baseline-path since --mpl-generate-path is set") + wmsg = ("Ignoring --mpl-baseline-path since " + "--mpl-generate-path is set") + warnings.warn(wmsg) - if baseline_dir is not None and not baseline_dir.startswith(("https", "http")): + if baseline_dir is not None and not baseline_dir.startswith(PROTOCOLS): baseline_dir = os.path.abspath(baseline_dir) if generate_dir is not None: baseline_dir = os.path.abspath(generate_dir) if results_dir is not None: results_dir = os.path.abspath(results_dir) - config.pluginmanager.register(ImageComparison(config, - baseline_dir=baseline_dir, - baseline_relative_dir=baseline_relative_dir, - generate_dir=generate_dir, - results_dir=results_dir, - hash_library=hash_library, - generate_hash_library=generate_hash_lib, - generate_summary=generate_summary, - results_always=results_always)) + plugin = ImageComparison(config, + baseline_dir=baseline_dir, + baseline_relative_dir=baseline_relative_dir, + generate_dir=generate_dir, + results_dir=results_dir, + hash_library=hash_library, + generate_hash_library=generate_hash_lib, + generate_summary=generate_summary, + results_always=results_always) + config.pluginmanager.register(plugin) else: @@ -198,8 +263,9 @@ def switch_backend(backend): def close_mpl_figure(fig): - "Close a given matplotlib Figure. Any other type of figure is ignored" - + """ + Close a given matplotlib Figure. Any other type of figure is ignored + """ import matplotlib.pyplot as plt from matplotlib.figure import Figure @@ -213,11 +279,12 @@ def close_mpl_figure(fig): def get_marker(item, marker_name): if hasattr(item, 'get_closest_marker'): - return item.get_closest_marker(marker_name) + result = item.get_closest_marker(marker_name) else: # "item.keywords.get" was deprecated in pytest 3.6 # See https://docs.pytest.org/en/latest/mark.html#updating-code - return item.keywords.get(marker_name) + result = item.keywords.get(marker_name) + return result def path_is_not_none(apath): @@ -235,7 +302,7 @@ def __init__(self, hash_library=None, generate_hash_library=None, generate_summary=None, - results_always=False + results_always=False, ): self.config = config self.baseline_dir = baseline_dir @@ -256,6 +323,36 @@ def __init__(self, self.generate_summary = generate_summary self.results_always = results_always + # Configure hashing kernel options. + option = 'mpl-hash-size' + hash_size = (config.getoption(f'--{option}') or + config.getini(option) or None) + self.hash_size = hash_size + + option = 'mpl-hamming-tolerance' + hamming_tolerance = (config.getoption(f'--{option}') or + config.getini(option) or None) + self.hamming_tolerance = hamming_tolerance + + option = 'mpl-high-freq-factor' + high_freq_factor = (config.getoption(f'--{option}') or + config.getini(option) or None) + self.high_freq_factor = high_freq_factor + + # Configure the hashing kernel - must be done *after* kernel options. + option = 'mpl-kernel' + kernel = config.getoption(f'--{option}') or config.getini(option) + if kernel: + requested = str(kernel).lower() + if requested not in kernel_factory: + emsg = f'Unrecognised hashing kernel {kernel!r} not supported.' + raise ValueError(emsg) + kernel = requested + else: + kernel = DEFAULT_KERNEL + # Create the kernel. + self.kernel = kernel_factory[kernel](self) + # Generate the containing dir for all test results if not self.results_dir: self.results_dir = Path(tempfile.mkdtemp(dir=self.results_dir)) @@ -282,7 +379,7 @@ def get_compare(self, item): """ Return the mpl_image_compare marker for the given item. """ - return get_marker(item, 'mpl_image_compare') + return get_marker(item, DEFAULT_MARKER) def generate_filename(self, item): """ @@ -344,7 +441,7 @@ def get_baseline_directory(self, item): baseline_dir = self.baseline_dir baseline_remote = (isinstance(baseline_dir, str) and # noqa - baseline_dir.startswith(('http://', 'https://'))) + baseline_dir.startswith(PROTOCOLS)) if not baseline_remote: return Path(item.fspath).parent / baseline_dir @@ -380,7 +477,7 @@ def obtain_baseline_image(self, item, target_dir): filename = self.generate_filename(item) baseline_dir = self.get_baseline_directory(item) baseline_remote = (isinstance(baseline_dir, str) and # noqa - baseline_dir.startswith(('http://', 'https://'))) + baseline_dir.startswith(PROTOCOLS)) if baseline_remote: # baseline_dir can be a list of URLs when remote, so we have to # pass base and filename to download @@ -410,8 +507,8 @@ def generate_baseline_image(self, item, fig): def generate_image_hash(self, item, fig): """ - For a `matplotlib.figure.Figure`, returns the SHA256 hash as a hexadecimal - string. + For a `matplotlib.figure.Figure`, returns the hash generated by the + kernel as a hexadecimal string. """ compare = self.get_compare(item) savefig_kwargs = compare.kwargs.get('savefig_kwargs', {}) @@ -420,11 +517,11 @@ def generate_image_hash(self, item, fig): fig.savefig(imgdata, **savefig_kwargs) - out = _hash_file(imgdata) + hash = self.kernel.generate_hash(imgdata) imgdata.close() close_mpl_figure(fig) - return out + return hash def compare_image_to_baseline(self, item, fig, result_dir, summary=None): """ @@ -437,14 +534,14 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None): summary = {} compare = self.get_compare(item) - tolerance = compare.kwargs.get('tolerance', 2) + tolerance = compare.kwargs.get('tolerance', DEFAULT_RMS_TOLERANCE) savefig_kwargs = compare.kwargs.get('savefig_kwargs', {}) baseline_image_ref = self.obtain_baseline_image(item, result_dir) - test_image = (result_dir / "result.png").absolute() - fig.savefig(str(test_image), **savefig_kwargs) - summary['result_image'] = test_image.relative_to(self.results_dir).as_posix() + result_image = (result_dir / "result.png").absolute() + fig.savefig(str(result_image), **savefig_kwargs) + summary['result_image'] = result_image.relative_to(self.results_dir).as_posix() if not os.path.exists(baseline_image_ref): summary['status'] = 'failed' @@ -453,7 +550,7 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None): f"{self.get_baseline_directory(item)}\n" "(This is expected for new tests.)\n" "Generated Image: \n\t" - f"{test_image}") + f"{result_image}") summary['status_msg'] = error_message return error_message @@ -466,19 +563,20 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None): # Compare image size ourselves since the Matplotlib # exception is a bit cryptic in this case and doesn't show # the filenames - expected_shape = imread(str(baseline_image)).shape[:2] - actual_shape = imread(str(test_image)).shape[:2] - if expected_shape != actual_shape: + baseline_shape = imread(str(baseline_image)).shape[:2] + result_shape = imread(str(result_image)).shape[:2] + if baseline_shape != result_shape: summary['status'] = 'failed' summary['image_status'] = 'diff' - error_message = SHAPE_MISMATCH_ERROR.format(expected_path=baseline_image, - expected_shape=expected_shape, - actual_path=test_image, - actual_shape=actual_shape) + error_message = TEMPLATE_SHAPE_MISMATCH.format(baseline_image=baseline_image, + baseline_shape=baseline_shape, + result_image=result_image, + result_shape=result_shape) summary['status_msg'] = error_message return error_message - results = compare_images(str(baseline_image), str(test_image), tol=tolerance, in_decorator=True) + # 'in_decorator=True' ensures that a dictionary of results is returned by 'compare_images' + results = compare_images(str(baseline_image), str(result_image), tol=tolerance, in_decorator=True) summary['tolerance'] = tolerance if results is None: summary['status'] = 'passed' @@ -491,19 +589,67 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None): summary['rms'] = results['rms'] diff_image = (result_dir / 'result-failed-diff.png').absolute() summary['diff_image'] = diff_image.relative_to(self.results_dir).as_posix() - template = ['Error: Image files did not match.', - 'RMS Value: {rms}', - 'Expected: \n {expected}', - 'Actual: \n {actual}', - 'Difference:\n {diff}', - 'Tolerance: \n {tol}', ] - error_message = '\n '.join([line.format(**results) for line in template]) + error_message = TEMPLATE_IMAGE_DIFFERENCE.format(**results) summary['status_msg'] = error_message return error_message - def load_hash_library(self, library_path): - with open(str(library_path)) as fp: - return json.load(fp) + def load_hash_library(self, fname): + with open(str(fname)) as fi: + hash_library = json.load(fi) + kernel_metadata = hash_library.get(META_HASH_KERNEL) + if kernel_metadata is None: + msg = (f'Hash library {str(fname)!r} missing a ' + f'{META_HASH_KERNEL!r} entry. Assuming that a ' + f'{self.kernel.name!r} kernel generated the library.') + self.logger.info(msg) + else: + if "name" not in kernel_metadata: + emsg = (f"Missing kernel 'name' in the {META_HASH_KERNEL!r} entry, " + f'for the hash library {str(fname)!r}.') + pytest.fail(emsg) + kernel_name = kernel_metadata["name"] + if kernel_name not in kernel_factory: + emsg = (f'Unrecognised hashing kernel {kernel_name!r} specified ' + f'in the hash library {str(fname)!r}.') + pytest.fail(emsg) + if kernel_name != self.kernel.name: + option = 'mpl-kernel' + if (self.config.getoption(f'--{option}') is None and + len(self.config.getini(option)) == 0): + # Override the default kernel with the kernel configured + # within the hash library. + self.kernel = kernel_factory[kernel_name](self) + else: + emsg = (f'Hash library {str(fname)!r} kernel ' + f'{kernel_name!r} does not match configured runtime ' + f'kernel {self.kernel.name!r}.') + pytest.fail(emsg) + + def check_metadata(key): + if key not in kernel_metadata: + emsg = (f'Missing kernel {key!r} in the ' + f'{META_HASH_KERNEL!r} entry, for the hash ' + f'library {str(fname)!r}.') + pytest.fail(emsg) + value = kernel_metadata[key] + if value != getattr(self.kernel, key): + option = f'mpl-{key.replace("_", "-")}' + if (self.config.getoption(f'--{option}') is None and + len(self.config.getini(option)) == 0): + # Override the default kernel value with the + # configured value within the hash library. + setattr(self.kernel, key, value) + else: + emsg = (f"Hash library {str(fname)!r} '{key}={value}' " + 'does not match configured runtime kernel ' + f"'{key}={getattr(self.kernel, key)}'.") + pytest.fail(emsg) + + for key in self.kernel.metadata: + if key != "name": + check_metadata(key) + + return hash_library def compare_image_to_hash_library(self, item, fig, result_dir, summary=None): hash_comparison_pass = False @@ -521,32 +667,39 @@ def compare_image_to_hash_library(self, item, fig, result_dir, summary=None): hash_library_filename = (Path(item.fspath).parent / hash_library_filename).absolute() if not Path(hash_library_filename).exists(): - pytest.fail(f"Can't find hash library at path {hash_library_filename}") + pytest.fail(f"Can't find hash library at path {str(hash_library_filename)!r}.") hash_library = self.load_hash_library(hash_library_filename) + hash_name = self.generate_test_name(item) baseline_hash = hash_library.get(hash_name, None) summary['baseline_hash'] = baseline_hash + summary['kernel'] = self.kernel.name - test_hash = self.generate_image_hash(item, fig) - summary['result_hash'] = test_hash + result_hash = self.generate_image_hash(item, fig) + summary['result_hash'] = result_hash if baseline_hash is None: # hash-missing summary['status'] = 'failed' summary['hash_status'] = 'missing' - summary['status_msg'] = (f"Hash for test '{hash_name}' not found in {hash_library_filename}. " - f"Generated hash is {test_hash}.") - elif test_hash == baseline_hash: # hash-match + msg = (f'Hash for test {hash_name!r} not found in ' + f'{str(hash_library_filename)!r}. Generated hash is ' + f'{result_hash!r}.') + summary['status_msg'] = msg + elif self.kernel.equivalent_hash(result_hash, baseline_hash): # hash-match hash_comparison_pass = True summary['status'] = 'passed' summary['hash_status'] = 'match' - summary['status_msg'] = 'Test hash matches baseline hash.' + summary['status_msg'] = 'Result hash matches baseline hash.' + self.kernel.update_summary(summary) else: # hash-diff summary['status'] = 'failed' summary['hash_status'] = 'diff' - summary['status_msg'] = (f"Hash {test_hash} doesn't match hash " - f"{baseline_hash} in library " - f"{hash_library_filename} for test {hash_name}.") + msg = (f'Result hash {result_hash!r} does not match baseline hash ' + f'{baseline_hash!r} in library {str(hash_library_filename)!r} ' + f'for test {hash_name!r}.') + summary['status_msg'] = self.kernel.update_status(msg) + self.kernel.update_summary(summary) # Save the figure for later summary (will be removed later if not needed) test_image = (result_dir / "result.png").absolute() @@ -575,7 +728,7 @@ def compare_image_to_hash_library(self, item, fig, result_dir, summary=None): # Append the log from image comparison r = baseline_comparison or "The comparison to the baseline image succeeded." summary['status_msg'] += ("\n\n" - "Image comparison test\n" + "Image Comparison Test\n" "---------------------\n") + r if hash_comparison_pass: # Return None to indicate test passed @@ -596,9 +749,9 @@ def pytest_runtest_setup(self, item): # noqa from matplotlib.testing.decorators import ImageComparisonTest as MplImageComparisonTest remove_ticks_and_titles = MplImageComparisonTest.remove_text - style = compare.kwargs.get('style', 'classic') - remove_text = compare.kwargs.get('remove_text', False) - backend = compare.kwargs.get('backend', 'agg') + style = compare.kwargs.get('style', DEFAULT_STYLE) + remove_text = compare.kwargs.get('remove_text', DEFAULT_REMOVE_TEXT) + backend = compare.kwargs.get('backend', DEFAULT_BACKEND) original = item.function @@ -636,6 +789,9 @@ def item_function_wrapper(*args, **kwargs): 'result_image': None, 'baseline_hash': None, 'result_hash': None, + 'kernel': None, + 'hamming_distance': None, + 'hamming_tolerance': None, } # What we do now depends on whether we are generating the @@ -656,6 +812,7 @@ def item_function_wrapper(*args, **kwargs): image_hash = self.generate_image_hash(item, fig) self._generated_hash_library[test_name] = image_hash summary['baseline_hash'] = image_hash + summary['kernel'] = self.kernel.name # Only test figures if not generating images if self.generate_dir is None: @@ -704,6 +861,10 @@ def pytest_unconfigure(self, config): if self.generate_hash_library is not None: hash_library_path = Path(config.rootdir) / self.generate_hash_library hash_library_path.parent.mkdir(parents=True, exist_ok=True) + # It's safe to inject this metadata, as the key is an invalid Python + # class/function/method name, therefore there's no possible + # namespace conflict with user py.test marker decorated tokens. + self._generated_hash_library[META_HASH_KERNEL] = self.kernel.metadata with open(hash_library_path, "w") as fp: json.dump(self._generated_hash_library, fp, indent=2) if self.results_always: # Make accessible in results directory @@ -714,6 +875,7 @@ def pytest_unconfigure(self, config): result_hashes = {k: v['result_hash'] for k, v in self._test_results.items() if v['result_hash']} if len(result_hashes) > 0: # At least one hash comparison test + result_hashes[META_HASH_KERNEL] = self.kernel.metadata with open(result_hash_library, "w") as fp: json.dump(result_hashes, fp, indent=2) @@ -744,7 +906,7 @@ def __init__(self, config): def pytest_runtest_setup(self, item): - compare = get_marker(item, 'mpl_image_compare') + compare = get_marker(item, DEFAULT_MARKER) if compare is None: return diff --git a/setup.cfg b/setup.cfg index 60478bd3..1d94d89e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,6 +30,7 @@ install_requires = importlib_resources;python_version<'3.8' packaging Jinja2 + imagehash [options.entry_points] pytest11 = diff --git a/tests/baseline/hashes/test_phash.json b/tests/baseline/hashes/test_phash.json new file mode 100644 index 00000000..c71fd874 --- /dev/null +++ b/tests/baseline/hashes/test_phash.json @@ -0,0 +1,8 @@ +{ + "test.test_gen": "8fe8c01f7a45b01f6645ac1fe9572b2a807e2ae0d41a7ab085967aac87997eab", + "pytest-mpl-kernel": { + "name": "phash", + "hash_size": 16, + "high_freq_factor": 4 + } +} diff --git a/tests/baseline/hashes/test_phash_bad_kernel.json b/tests/baseline/hashes/test_phash_bad_kernel.json new file mode 100644 index 00000000..f386810d --- /dev/null +++ b/tests/baseline/hashes/test_phash_bad_kernel.json @@ -0,0 +1,8 @@ +{ + "test.test_gen": "8fe8c01f7a45b01f6645ac1fe9572b2a807e2ae0d41a7ab085967aac87997eab", + "pytest-mpl-kernel": { + "name": "wibble", + "hash_size": 16, + "high_freq_factor": 4 + } +} diff --git a/tests/baseline/hashes/test_phash_missing_hash_size.json b/tests/baseline/hashes/test_phash_missing_hash_size.json new file mode 100644 index 00000000..7848f0b8 --- /dev/null +++ b/tests/baseline/hashes/test_phash_missing_hash_size.json @@ -0,0 +1,7 @@ +{ + "test.test_gen": "8fe8c01f7a45b01f6645ac1fe9572b2a807e2ae0d41a7ab085967aac87997eab", + "pytest-mpl-kernel": { + "name": "phash", + "high_freq_factor": 4 + } +} diff --git a/tests/baseline/hashes/test_phash_missing_high_freq_factor.json b/tests/baseline/hashes/test_phash_missing_high_freq_factor.json new file mode 100644 index 00000000..5b4294bc --- /dev/null +++ b/tests/baseline/hashes/test_phash_missing_high_freq_factor.json @@ -0,0 +1,7 @@ +{ + "test.test_gen": "8fe8c01f7a45b01f6645ac1fe9572b2a807e2ae0d41a7ab085967aac87997eab", + "pytest-mpl-kernel": { + "name": "phash", + "hash_size": 16 + } +} diff --git a/tests/baseline/hashes/test_phash_missing_name.json b/tests/baseline/hashes/test_phash_missing_name.json new file mode 100644 index 00000000..d2956922 --- /dev/null +++ b/tests/baseline/hashes/test_phash_missing_name.json @@ -0,0 +1,7 @@ +{ + "test.test_gen": "8fe8c01f7a45b01f6645ac1fe9572b2a807e2ae0d41a7ab085967aac87997eab", + "pytest-mpl-kernel": { + "hash_size": 16, + "high_freq_factor": 4 + } +} diff --git a/tests/subtests/test_subtest.py b/tests/subtests/test_subtest.py index eb85aa5c..811fddb0 100644 --- a/tests/subtests/test_subtest.py +++ b/tests/subtests/test_subtest.py @@ -4,13 +4,15 @@ import subprocess from pathlib import Path -import matplotlib import matplotlib.ft2font import pytest from packaging.version import Version from .helpers import assert_existence, diff_summary, patch_summary +# TODO: Remove this skip, see issue https://github.com/matplotlib/pytest-mpl/issues/159 +pytest.skip(reason="temporarily disabled sub-tests", allow_module_level=True) + # Handle Matplotlib and FreeType versions MPL_VERSION = Version(matplotlib.__version__) FTV = matplotlib.ft2font.__freetype_version__.replace('.', '') diff --git a/tests/test_kernels.py b/tests/test_kernels.py new file mode 100644 index 00000000..3fd42927 --- /dev/null +++ b/tests/test_kernels.py @@ -0,0 +1,265 @@ +from pathlib import Path + +import pytest + +from pytest_mpl.kernels import (DEFAULT_HAMMING_TOLERANCE, DEFAULT_HASH_SIZE, + DEFAULT_HIGH_FREQUENCY_FACTOR, Kernel, + KernelPHash, KernelSHA256, kernel_factory) + +#: baseline hash (32-bit) +HASH_BASE_32 = "01234567" + +#: baseline hash (64-bit) +HASH_BASE = "0123456789abcdef" + +#: baseline hash with 2-bit delta (64-bit) +# ---X------------ +HASH_2BIT = "0120456789abcdef" + +#: baseline with 4-bit delta (64-bit) +# --XX-----------X +HASH_4BIT = "0100456789abcdee" + +#: baseline with 8-bit delta (64-bit) +# -X------------XX +HASH_8BIT = "0023456789abcd00" + + +#: Absolute path to test baseline image +baseline_image = Path(__file__).parent / "baseline" / "2.0.x" / "test_base_style.png" + +#: Verify availabilty of test baseline image +baseline_unavailable = not baseline_image.is_file() + +#: Convenience skipif reason +baseline_missing = f"missing baseline image {str(baseline_image)!r}" + + +class DummyMarker: + def __init__(self, hamming_tolerance=None): + self.kwargs = dict(hamming_tolerance=hamming_tolerance) + + +class DummyPlugin: + def __init__(self, hash_size=None, hamming_tolerance=None, high_freq_factor=None): + self.hash_size = hash_size + self.hamming_tolerance = hamming_tolerance + self.high_freq_factor = high_freq_factor + + +def test_kernel_abc(): + emsg = "Can't instantiate abstract class Kernel" + with pytest.raises(TypeError, match=emsg): + Kernel(None) + + +def test_phash_name(): + for name, factory in kernel_factory.items(): + assert name == factory.name + + +# +# KernelPHash +# + + +def test_phash_init__set(): + hash_size, hamming_tolerance, high_freq_factor = -1, -2, -3 + plugin = DummyPlugin( + hash_size=hash_size, + hamming_tolerance=hamming_tolerance, + high_freq_factor=high_freq_factor, + ) + kernel = KernelPHash(plugin) + assert kernel.hash_size == hash_size + assert kernel.hamming_tolerance == hamming_tolerance + assert kernel.high_freq_factor == high_freq_factor + assert kernel.equivalent is None + assert kernel.hamming_distance is None + + +def test_phash_init__default(): + plugin = DummyPlugin() + kernel = KernelPHash(plugin) + assert kernel.hash_size == DEFAULT_HASH_SIZE + assert kernel.hamming_tolerance == DEFAULT_HAMMING_TOLERANCE + assert kernel.high_freq_factor == DEFAULT_HIGH_FREQUENCY_FACTOR + assert kernel.equivalent is None + assert kernel.hamming_distance is None + + +def test_phash_option(): + assert KernelPHash(DummyPlugin()).option == "hamming_tolerance" + + +@pytest.mark.parametrize( + "baseline,equivalent,distance", + [ + (HASH_BASE, True, 0), + (HASH_2BIT, True, 2), + (HASH_4BIT, True, 4), + (HASH_8BIT, False, 8), + (HASH_BASE_32, False, None), + ], +) +def test_phash_equivalent(baseline, equivalent, distance): + kernel = KernelPHash(DummyPlugin()) + assert kernel.equivalent_hash(HASH_BASE, baseline) is equivalent + assert kernel.equivalent is equivalent + assert kernel.hamming_distance == distance + + +def test_phash_equivalent__tolerance(): + hamming_tolerance = 10 + plugin = DummyPlugin(hamming_tolerance=hamming_tolerance) + kernel = KernelPHash(plugin) + assert kernel.equivalent_hash(HASH_BASE, HASH_4BIT) + assert kernel.equivalent is True + assert kernel.hamming_tolerance == hamming_tolerance + assert kernel.hamming_distance == 4 + + +@pytest.mark.parametrize( + "tolerance,equivalent", + [(10, True), (3, False)], +) +def test_phash_equivalent__marker(tolerance, equivalent): + marker = DummyMarker(hamming_tolerance=tolerance) + kernel = KernelPHash(DummyPlugin()) + assert kernel.hamming_tolerance == DEFAULT_HAMMING_TOLERANCE + assert kernel.equivalent_hash(HASH_BASE, HASH_4BIT, marker=marker) is equivalent + assert kernel.equivalent is equivalent + assert kernel.hamming_tolerance == tolerance + assert kernel.hamming_distance == 4 + + +@pytest.mark.skipif(baseline_unavailable, reason=baseline_missing) +@pytest.mark.parametrize( + "hash_size,hff,expected", + [ + ( + DEFAULT_HASH_SIZE, + DEFAULT_HIGH_FREQUENCY_FACTOR, + "800bc0555feab05f67ea8d1779fa83537e7ec0d17f9f003517ef200985532856", + ), + ( + DEFAULT_HASH_SIZE, + 8, + "800fc0155fe8b05f67ea8d1779fa83537e7ec0d57f9f003517ef200985532856", + ), + (8, DEFAULT_HIGH_FREQUENCY_FACTOR, "80c05fb1778d79c3"), + ( + DEFAULT_HASH_SIZE, + 16, + "800bc0155feab05f67ea8d1779fa83537e7ec0d57f9f003517ef200985532856", + ), + ], +) +def test_phash_generate_hash(hash_size, hff, expected): + plugin = DummyPlugin(hash_size=hash_size, high_freq_factor=hff) + kernel = KernelPHash(plugin) + fh = open(baseline_image, "rb") + actual = kernel.generate_hash(fh) + assert actual == expected + + +@pytest.mark.parametrize("message", (None, "", "one")) +@pytest.mark.parametrize("equivalent", (None, True)) +def test_phash_update_status__none(message, equivalent): + kernel = KernelPHash(DummyPlugin()) + kernel.equivalent = equivalent + result = kernel.update_status(message) + assert isinstance(result, str) + expected = 0 if message is None else len(message) + assert len(result) == expected + + +@pytest.mark.parametrize("message", ("", "one")) +@pytest.mark.parametrize("distance", (10, 20)) +@pytest.mark.parametrize("tolerance", (1, 2)) +def test_phash_update_status__equivalent(message, distance, tolerance): + plugin = DummyPlugin(hamming_tolerance=tolerance) + kernel = KernelPHash(plugin) + kernel.equivalent = False + kernel.hamming_distance = distance + result = kernel.update_status(message) + assert isinstance(result, str) + template = "Hash hamming distance of {} bits > hamming tolerance of {} bits." + status = template.format(distance, tolerance) + expected = f"{message} {status}" if message else status + assert result == expected + + +@pytest.mark.parametrize( + "summary,distance,tolerance,count", + [({}, None, DEFAULT_HAMMING_TOLERANCE, 3), (dict(one=1), 2, 3, 4)], +) +def test_phash_update_summary(summary, distance, tolerance, count): + plugin = DummyPlugin(hamming_tolerance=tolerance) + kernel = KernelPHash(plugin) + kernel.hamming_distance = distance + kernel.update_summary(summary) + assert summary["kernel"] == KernelPHash.name + assert summary["hamming_distance"] == distance + assert summary["hamming_tolerance"] == tolerance + assert len(summary) == count + + +@pytest.mark.parametrize( + "hash_size,hff", + [(DEFAULT_HASH_SIZE, DEFAULT_HIGH_FREQUENCY_FACTOR), (32, 8)], +) +def test_phash_metadata(hash_size, hff): + plugin = DummyPlugin(hash_size=hash_size, high_freq_factor=hff) + kernel = KernelPHash(plugin) + metadata = kernel.metadata + assert {"name", "hash_size", "high_freq_factor"} == set(metadata) + assert metadata["name"] == KernelPHash.name + assert metadata["hash_size"] == hash_size + assert metadata["high_freq_factor"] == hff + + +# +# KernelSHA256 +# + + +@pytest.mark.parametrize( + "baseline, equivalent", + [(HASH_BASE, True), (HASH_2BIT, False), (HASH_4BIT, False)], +) +def test_sha256_equivalent(baseline, equivalent): + kernel = KernelSHA256(DummyPlugin()) + assert kernel.equivalent_hash(HASH_BASE, baseline) is equivalent + + +@pytest.mark.skipif(baseline_unavailable, reason=baseline_missing) +def test_sha256_generate_hash(): + kernel = KernelSHA256(DummyPlugin()) + fh = open(baseline_image, "rb") + actual = kernel.generate_hash(fh) + expected = "2dc4d32eefa5f5d11c365b10196f2fcdadc8ed604e370d595f9cf304363c13d2" + assert actual == expected + + +def test_sha256_update_status(): + kernel = KernelSHA256(DummyPlugin()) + message = "nop" + result = kernel.update_status(message) + assert result is message + + +def test_sha256_update_summary(): + kernel = KernelSHA256(DummyPlugin()) + summary = {} + kernel.update_summary(summary) + assert len(summary) == 1 + assert "kernel" in summary + assert summary["kernel"] == KernelSHA256.name + + +def test_sha256_metadata(): + kernel = KernelSHA256(DummyPlugin()) + metadata = kernel.metadata + assert {"name"} == set(metadata) + assert metadata["name"] == KernelSHA256.name diff --git a/tests/test_pytest_mpl.py b/tests/test_pytest_mpl.py index 5ab4d8a0..cb20d929 100644 --- a/tests/test_pytest_mpl.py +++ b/tests/test_pytest_mpl.py @@ -26,8 +26,9 @@ if "+" in matplotlib.__version__: hash_filename = "mpldev.json" -hash_library = (Path(__file__).parent / "baseline" / # noqa - "hashes" / hash_filename) +hashes_dir = Path(__file__).parent / "baseline" / "hashes" + +hash_library = hashes_dir / hash_filename fail_hash_library = Path(__file__).parent / "baseline" / "test_hash_lib.json" baseline_dir_abs = Path(__file__).parent / "baseline" / baseline_subdir @@ -289,13 +290,14 @@ def test_hash_fails(tmpdir): f.write(TEST_FAILING_HASH) # If we use --mpl, it should detect that the figure is wrong - output = assert_pytest_fails_with(['--mpl', test_file], "doesn't match hash FAIL in library") + expected = "does not match baseline hash 'FAIL' in library" + output = assert_pytest_fails_with(['--mpl', test_file], expected) # We didn't specify a baseline dir so we shouldn't attempt to find one assert "Image file not found for comparison test" not in output, output # Check that the summary path is printed and that it exists. output = assert_pytest_fails_with(['--mpl', test_file, '--mpl-generate-summary=html'], - "doesn't match hash FAIL in library") + expected) # We didn't specify a baseline dir so we shouldn't attempt to find one print_message = "A summary of test results can be found at:" assert print_message in output, output @@ -328,21 +330,22 @@ def test_hash_fail_hybrid(tmpdir): f.write(TEST_FAILING_HYBRID) # Assert that image comparison runs and fails + expected = "does not match baseline hash 'FAIL' in library" output = assert_pytest_fails_with(['--mpl', test_file, rf'--mpl-baseline-path={hash_baseline_dir_abs / "fail"}'], - "doesn't match hash FAIL in library") - assert "Error: Image files did not match." in output, output + expected) + assert "Failed! Image files did not match." in output, output # Assert reports missing baseline image output = assert_pytest_fails_with(['--mpl', test_file, '--mpl-baseline-path=/not/a/path'], - "doesn't match hash FAIL in library") + expected) assert "Image file not found for comparison test" in output, output # Assert reports image comparison succeeds output = assert_pytest_fails_with(['--mpl', test_file, rf'--mpl-baseline-path={hash_baseline_dir_abs / "succeed"}'], - "doesn't match hash FAIL in library") + expected) assert "The comparison to the baseline image succeeded." in output, output # If we don't use --mpl option, the test should succeed @@ -370,16 +373,17 @@ def test_hash_fail_new_hashes(tmpdir): f.write(TEST_FAILING_NEW_HASH) # Assert that image comparison runs and fails + expected = "does not match baseline hash 'FAIL' in library" assert_pytest_fails_with(['--mpl', test_file, f'--mpl-hash-library={fail_hash_library}'], - "doesn't match hash FAIL in library") + expected) hash_file = tmpdir.join('new_hashes.json').strpath # Assert that image comparison runs and fails assert_pytest_fails_with(['--mpl', test_file, f'--mpl-hash-library={fail_hash_library}', f'--mpl-generate-hash-library={hash_file}'], - "doesn't match hash FAIL") + expected) TEST_MISSING_HASH = """ @@ -486,3 +490,76 @@ def test_results_always(tmpdir): assert image and not image_exists assert image not in html assert json_res[json_image_key] is None + + +@pytest.mark.parametrize("cla", ["--mpl-kernel=phash", ""]) +def test_phash(tmpdir, cla): + test_file = tmpdir.join("test.py").strpath + with open(test_file, "w") as fo: + fo.write(TEST_GENERATE) + + # Filter out empty command-line-argument (cla) from command string. + command = list(filter(None, + ["--mpl", + cla, + f"--mpl-hash-library={hashes_dir / 'test_phash.json'}", + test_file] + ) + ) + code = call_pytest(command) + assert code == 0 + + +@pytest.mark.parametrize("key", ["name", "hash_size", "high_freq_factor"]) +def test_phash_fail__hash_library_missing(tmpdir, key): + test_file = tmpdir.join("test.py").strpath + with open(test_file, "w") as fo: + fo.write(TEST_GENERATE) + + library = hashes_dir / f"test_phash_missing_{key}.json" + command = ["--mpl", + "--mpl-kernel=phash", + f"--mpl-hash-library={library}", + test_file] + emsg = f"Missing kernel '{key}'" + assert_pytest_fails_with(command, emsg) + + +def test_phash_fail__hash_library_bad_kernel(tmpdir): + test_file = tmpdir.join("test.py").strpath + with open(test_file, "w") as fo: + fo.write(TEST_GENERATE) + + library = hashes_dir / 'test_phash_bad_kernel.json' + command = ["--mpl", + f"--mpl-hash-library={library}", + test_file] + emsg = "Unrecognised hashing kernel 'wibble'" + assert_pytest_fails_with(command, emsg) + + +def test_phash_pass__override_default(tmpdir): + test_file = tmpdir.join("test.py").strpath + with open(test_file, "w") as fo: + fo.write(TEST_GENERATE) + + library = hashes_dir / 'test_phash.json' + command = ["--mpl", + f"--mpl-hash-library={library}", + test_file] + code = call_pytest(command) + assert code == 0 + + +def test_phash_fail__hash_library_mismatch_kernel(tmpdir): + test_file = tmpdir.join("test.py").strpath + with open(test_file, "w") as fo: + fo.write(TEST_GENERATE) + + library = hashes_dir / 'test_phash.json' + command = ["--mpl", + "--mpl-kernel=sha256", + f"--mpl-hash-library={library}", + test_file] + emsg = "'phash' does not match configured runtime kernel 'sha256'" + assert_pytest_fails_with(command, emsg)