diff --git a/.github/workflows/test_and_publish.yml b/.github/workflows/test_and_publish.yml index f8fb97b7..ab3dc3a1 100644 --- a/.github/workflows/test_and_publish.yml +++ b/.github/workflows/test_and_publish.yml @@ -17,6 +17,10 @@ jobs: test: uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1 with: + libraries: | + apt: + - ghostscript + - inkscape envs: | # Test the oldest and newest configuration on Mac and Windows - macos: py36-test-mpl20 diff --git a/README.rst b/README.rst index 21d02a62..13e413e0 100644 --- a/README.rst +++ b/README.rst @@ -90,7 +90,6 @@ When generating a hash library, the tests will also be run as usual against the existing hash library specified by ``--mpl-hash-library`` or the keyword argument. However, generating baseline images will always result in the tests being skipped. - Hybrid Mode: Hashes and Images ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -278,6 +277,81 @@ decorator: This will make the test insensitive to changes in e.g. the freetype library. +Supported formats and deterministic output +------------------------------------------ + +By default, pytest-mpl will save and compare figures in PNG format. However, +it is possible to set the format to use by setting e.g. ``savefig_kwargs={'format': 'pdf'}`` +in ``mpl_image_compare``. Supported formats are ``'eps'``, ``'pdf'``, ``'png'``, and ``'svg'``. +Note that Ghostscript is required to be installed for comparing PDF and EPS figures, while +Inkscape is required for SVG comparison. + +By default, Matplotlib does not produce deterministic output that will have a +consistent hash every time it is run, or over different Matplotlib versions. In +order to enforce that the output is deterministic, you will need to set metadata +as described in the following subsections. + +PNG +^^^ + +For PNG files, the output can be made deterministic by setting: + +.. code:: python + + @pytest.mark.mpl_image_compare(savefig_kwargs={'metadata': {"Software": None}}) + +PDF +^^^ + +For PDF files, the output can be made deterministic by setting: + +.. code:: python + + @pytest.mark.mpl_image_compare(savefig_kwargs={'format': 'pdf', + 'metadata': {"Creator": None, + "Producer": None, + "CreationDate": None}}) + +Note that deterministic PDF output can only be achieved with Matplotlib 2.1 and above + +EPS +^^^ + +For PDF files, the output can be made deterministic by setting: + +.. code:: python + + @pytest.mark.mpl_image_compare(savefig_kwargs={'format': 'pdf', + 'metadata': {"Creator": "test"}) + +and in addition you will need to set the SOURCE_DATE_EPOCH environment variable to +a constant value (this is a unit timestamp): + +.. code:: python + + os.environ['SOURCE_DATE_EPOCH'] = '1680254601' + +You could do this inside the test. + +Note that deterministic PDF output can only be achieved with Matplotlib 2.1 and above + +SVG +^^^ + +For SVG files, the output can be made deterministic by setting: + +.. code:: python + + @pytest.mark.mpl_image_compare(savefig_kwargs={'metadata': '{"Date": None}}) + +and in addition, you should make sure the following rcParam is set to a constant string: + +.. code:: python + + plt.rcParams['svg.hashsalt'] = 'test' + +Note that SVG files can only be used in pytest-mpl with Matplotlib 3.3 and above. + Test failure example -------------------- diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index 13cced37..4128fd16 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -52,6 +52,12 @@ Actual shape: {actual_shape} {actual_path}""" +# The following are the subsets of formats supported by the Matplotlib image +# comparison machinery +RASTER_IMAGE_FORMATS = ['png'] +VECTOR_IMAGE_FORMATS = ['eps', 'pdf', 'svg'] +ALL_IMAGE_FORMATS = RASTER_IMAGE_FORMATS + VECTOR_IMAGE_FORMATS + def _hash_file(in_stream): """ @@ -70,8 +76,8 @@ def pathify(path): """ path = Path(path) ext = '' - if path.suffixes[-1] == '.png': - ext = '.png' + if path.suffixes[-1][1:] in ALL_IMAGE_FORMATS: + ext = path.suffixes[-1] path = str(path).split(ext)[0] path = str(path) path = path.replace('[', '_').replace(']', '_') @@ -315,18 +321,24 @@ def __init__(self, self.logger.setLevel(level) self.logger.addHandler(handler) + def _file_extension(self, item): + compare = get_compare(item) + savefig_kwargs = compare.kwargs.get('savefig_kwargs', {}) + return savefig_kwargs.get('format', 'png') + def generate_filename(self, item): """ Given a pytest item, generate the figure filename. """ + ext = self._file_extension(item) if self.config.getini('mpl-use-full-test-name'): - filename = generate_test_name(item) + '.png' + filename = generate_test_name(item) + f'.{ext}' else: compare = get_compare(item) # Find test name to use as plot name filename = compare.kwargs.get('filename', None) if filename is None: - filename = item.name + '.png' + filename = item.name + f'.{ext}' filename = str(pathify(filename)) return filename @@ -441,10 +453,10 @@ def generate_image_hash(self, item, fig): compare = get_compare(item) savefig_kwargs = compare.kwargs.get('savefig_kwargs', {}) - imgdata = io.BytesIO() + ext = self._file_extension(item) + imgdata = io.BytesIO() fig.savefig(imgdata, **savefig_kwargs) - out = _hash_file(imgdata) imgdata.close() @@ -465,11 +477,17 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None): tolerance = compare.kwargs.get('tolerance', 2) savefig_kwargs = compare.kwargs.get('savefig_kwargs', {}) + ext = self._file_extension(item) + baseline_image_ref = self.obtain_baseline_image(item, result_dir) - test_image = (result_dir / "result.png").absolute() + test_image = (result_dir / f"result.{ext}").absolute() fig.savefig(str(test_image), **savefig_kwargs) - summary['result_image'] = test_image.relative_to(self.results_dir).as_posix() + + if ext in ['png', 'svg']: # Use original file + summary['result_image'] = test_image.relative_to(self.results_dir).as_posix() + else: + summary['result_image'] = (result_dir / f"result_{ext}.png").relative_to(self.results_dir).as_posix() if not os.path.exists(baseline_image_ref): summary['status'] = 'failed' @@ -484,26 +502,33 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None): # setuptools may put the baseline images in non-accessible places, # copy to our tmpdir to be sure to keep them in case of failure - baseline_image = (result_dir / "baseline.png").absolute() + baseline_image = (result_dir / f"baseline.{ext}").absolute() shutil.copyfile(baseline_image_ref, baseline_image) - summary['baseline_image'] = baseline_image.relative_to(self.results_dir).as_posix() + + if ext in ['png', 'svg']: # Use original file + summary['baseline_image'] = baseline_image.relative_to(self.results_dir).as_posix() + else: + summary['baseline_image'] = (result_dir / f"baseline_{ext}.png").relative_to(self.results_dir).as_posix() # Compare image size ourselves since the Matplotlib # exception is a bit cryptic in this case and doesn't show - # the filenames - expected_shape = imread(str(baseline_image)).shape[:2] - actual_shape = imread(str(test_image)).shape[:2] - if expected_shape != actual_shape: - summary['status'] = 'failed' - summary['image_status'] = 'diff' - error_message = SHAPE_MISMATCH_ERROR.format(expected_path=baseline_image, - expected_shape=expected_shape, - actual_path=test_image, - actual_shape=actual_shape) - summary['status_msg'] = error_message - return error_message + # the filenames. However imread won't work for vector graphics so we + # only do this for raster files. + if ext in RASTER_IMAGE_FORMATS: + expected_shape = imread(str(baseline_image)).shape[:2] + actual_shape = imread(str(test_image)).shape[:2] + if expected_shape != actual_shape: + summary['status'] = 'failed' + summary['image_status'] = 'diff' + error_message = SHAPE_MISMATCH_ERROR.format(expected_path=baseline_image, + expected_shape=expected_shape, + actual_path=test_image, + actual_shape=actual_shape) + summary['status_msg'] = error_message + return error_message results = compare_images(str(baseline_image), str(test_image), tol=tolerance, in_decorator=True) + summary['tolerance'] = tolerance if results is None: summary['status'] = 'passed' @@ -514,8 +539,7 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None): summary['status'] = 'failed' summary['image_status'] = 'diff' summary['rms'] = results['rms'] - diff_image = (result_dir / 'result-failed-diff.png').absolute() - summary['diff_image'] = diff_image.relative_to(self.results_dir).as_posix() + summary['diff_image'] = Path(results['diff']).relative_to(self.results_dir).as_posix() template = ['Error: Image files did not match.', 'RMS Value: {rms}', 'Expected: \n {expected}', @@ -538,6 +562,8 @@ def compare_image_to_hash_library(self, item, fig, result_dir, summary=None): compare = get_compare(item) savefig_kwargs = compare.kwargs.get('savefig_kwargs', {}) + ext = self._file_extension(item) + if not self.results_hash_library_name: # Use hash library name of current test as results hash library name self.results_hash_library_name = Path(compare.kwargs.get("hash_library", "")).name @@ -574,7 +600,7 @@ def compare_image_to_hash_library(self, item, fig, result_dir, summary=None): f"{hash_library_filename} for test {hash_name}.") # Save the figure for later summary (will be removed later if not needed) - test_image = (result_dir / "result.png").absolute() + test_image = (result_dir / f"result.{ext}").absolute() fig.savefig(str(test_image), **savefig_kwargs) summary['result_image'] = test_image.relative_to(self.results_dir).as_posix() @@ -627,6 +653,8 @@ def pytest_runtest_call(self, item): # noqa remove_text = compare.kwargs.get('remove_text', False) backend = compare.kwargs.get('backend', 'agg') + ext = self._file_extension(item) + with plt.style.context(style, after_reset=True), switch_backend(backend): # Run test and get figure object @@ -665,7 +693,7 @@ def pytest_runtest_call(self, item): # noqa summary['status_msg'] = 'Skipped test, since generating image.' generate_image = self.generate_baseline_image(item, fig) if self.results_always: # Make baseline image available in HTML - result_image = (result_dir / "baseline.png").absolute() + result_image = (result_dir / f"baseline.{ext}").absolute() shutil.copy(generate_image, result_image) summary['baseline_image'] = \ result_image.relative_to(self.results_dir).as_posix() diff --git a/tests/baseline/2.0.x/test_format_eps.eps b/tests/baseline/2.0.x/test_format_eps.eps new file mode 100644 index 00000000..d8438114 --- /dev/null +++ b/tests/baseline/2.0.x/test_format_eps.eps @@ -0,0 +1,780 @@ +%!PS-Adobe-3.0 EPSF-3.0 +%%Title: test_format_eps.eps +%%Creator: Matplotlib v3.6.3, https://matplotlib.org/ +%%CreationDate: Thu Mar 30 10:58:11 2023 +%%Orientation: portrait +%%BoundingBox: 18 180 594 612 +%%HiResBoundingBox: 18.000000 180.000000 594.000000 612.000000 +%%EndComments +%%BeginProlog +/mpldict 11 dict def +mpldict begin +/_d { bind def } bind def +/m { moveto } _d +/l { lineto } _d +/r { rlineto } _d +/c { curveto } _d +/cl { closepath } _d +/ce { closepath eofill } _d +/box { + m + 1 index 0 r + 0 exch r + neg 0 r + cl + } _d +/clipbox { + box + clip + newpath + } _d +/sc { setcachedevice } _d +%!PS-Adobe-3.0 Resource-Font +%%Creator: Converted from TrueType to Type 3 by Matplotlib. +10 dict begin +/FontName /DejaVuSans def +/PaintType 0 def +/FontMatrix [0.00048828125 0 0 0.00048828125 0 0] def +/FontBBox [-2090 -948 3673 2524] def +/FontType 3 def +/Encoding [/period /zero /one /two /three /five] def +/CharStrings 7 dict dup begin +/.notdef 0 def +/period{651 0 219 0 430 254 sc +219 254 m +430 254 l +430 0 l +219 0 l +219 254 l + +ce} _d +/zero{1303 0 135 -29 1167 1520 sc +651 1360 m +547 1360 469 1309 416 1206 c +364 1104 338 950 338 745 c +338 540 364 387 416 284 c +469 182 547 131 651 131 c +756 131 834 182 886 284 c +939 387 965 540 965 745 c +965 950 939 1104 886 1206 c +834 1309 756 1360 651 1360 c + +651 1520 m +818 1520 946 1454 1034 1321 c +1123 1189 1167 997 1167 745 c +1167 494 1123 302 1034 169 c +946 37 818 -29 651 -29 c +484 -29 356 37 267 169 c +179 302 135 494 135 745 c +135 997 179 1189 267 1321 c +356 1454 484 1520 651 1520 c + +ce} _d +/one{1303 0 225 0 1114 1493 sc +254 170 m +584 170 l +584 1309 l +225 1237 l +225 1421 l +582 1493 l +784 1493 l +784 170 l +1114 170 l +1114 0 l +254 0 l +254 170 l + +ce} _d +/two{1303 0 150 0 1098 1520 sc +393 170 m +1098 170 l +1098 0 l +150 0 l +150 170 l +227 249 331 356 463 489 c +596 623 679 709 713 748 c +778 821 823 882 848 932 c +874 983 887 1032 887 1081 c +887 1160 859 1225 803 1275 c +748 1325 675 1350 586 1350 c +523 1350 456 1339 385 1317 c +315 1295 240 1262 160 1217 c +160 1421 l +241 1454 317 1478 388 1495 c +459 1512 523 1520 582 1520 c +737 1520 860 1481 952 1404 c +1044 1327 1090 1223 1090 1094 c +1090 1033 1078 974 1055 919 c +1032 864 991 800 930 725 c +913 706 860 650 771 557 c +682 465 556 336 393 170 c + +ce} _d +/three{1303 0 156 -29 1139 1520 sc +831 805 m +928 784 1003 741 1057 676 c +1112 611 1139 530 1139 434 c +1139 287 1088 173 987 92 c +886 11 742 -29 555 -29 c +492 -29 428 -23 361 -10 c +295 2 227 20 156 45 c +156 240 l +212 207 273 183 340 166 c +407 149 476 141 549 141 c +676 141 772 166 838 216 c +905 266 938 339 938 434 c +938 522 907 591 845 640 c +784 690 698 715 588 715 c +414 715 l +414 881 l +596 881 l +695 881 771 901 824 940 c +877 980 903 1037 903 1112 c +903 1189 876 1247 821 1288 c +767 1329 689 1350 588 1350 c +533 1350 473 1344 410 1332 c +347 1320 277 1301 201 1276 c +201 1456 l +278 1477 349 1493 416 1504 c +483 1515 547 1520 606 1520 c +759 1520 881 1485 970 1415 c +1059 1346 1104 1252 1104 1133 c +1104 1050 1080 980 1033 923 c +986 866 918 827 831 805 c + +ce} _d +/five{1303 0 158 -29 1124 1493 sc +221 1493 m +1014 1493 l +1014 1323 l +406 1323 l +406 957 l +435 967 465 974 494 979 c +523 984 553 987 582 987 c +749 987 881 941 978 850 c +1075 759 1124 635 1124 479 c +1124 318 1074 193 974 104 c +874 15 733 -29 551 -29 c +488 -29 424 -24 359 -13 c +294 -2 227 14 158 35 c +158 238 l +218 205 280 181 344 165 c +408 149 476 141 547 141 c +662 141 754 171 821 232 c +888 293 922 375 922 479 c +922 583 888 665 821 726 c +754 787 662 817 547 817 c +493 817 439 811 385 799 c +332 787 277 768 221 743 c +221 1493 l + +ce} _d +end readonly def + +/BuildGlyph { + exch begin + CharStrings exch + 2 copy known not {pop /.notdef} if + true 3 1 roll get exec + end +} _d + +/BuildChar { + 1 index /Encoding get exch get + 1 index /BuildGlyph get exec +} _d + +FontName currentdict end definefont pop +end +%%EndProlog +mpldict begin +18 180 translate +576 432 0 0 clipbox +gsave +0 0 m +576 0 l +576 432 l +0 432 l +cl +1.000 setgray +fill +grestore +gsave +72 43.2 m +518.4 43.2 l +518.4 388.8 l +72 388.8 l +cl +1.000 setgray +fill +grestore +1.000 setlinewidth +1 setlinejoin +2 setlinecap +[] 0 setdash +0.000 0.000 1.000 setrgbcolor +gsave +446.4 345.6 72 43.2 clipbox +72 43.2 m +295.2 216 l +518.4 388.8 l +stroke +grestore +0 setlinejoin +0.000 setgray +gsave +72 43.2 m +72 388.8 l +stroke +grestore +gsave +518.4 43.2 m +518.4 388.8 l +stroke +grestore +gsave +72 43.2 m +518.4 43.2 l +stroke +grestore +gsave +72 388.8 m +518.4 388.8 l +stroke +grestore +0.500 setlinewidth +1 setlinejoin +0 setlinecap +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +0 0 m +0 4 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +72 43.2 o +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +0 0 m +0 -4 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +72 388.8 o +grestore +/DejaVuSans 12.000 selectfont +gsave + +62.4531 30.075 translate +0 rotate +0 0 m /zero glyphshow +7.63477 0 m /period glyphshow +11.4492 0 m /zero glyphshow +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +0 0 m +0 4 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +183.6 43.2 o +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +0 0 m +0 -4 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +183.6 388.8 o +grestore +/DejaVuSans 12.000 selectfont +gsave + +174.053 30.075 translate +0 rotate +0 0 m /zero glyphshow +7.63477 0 m /period glyphshow +11.4492 0 m /five glyphshow +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +0 0 m +0 4 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +295.2 43.2 o +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +0 0 m +0 -4 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +295.2 388.8 o +grestore +/DejaVuSans 12.000 selectfont +gsave + +285.653 30.075 translate +0 rotate +0 0 m /one glyphshow +7.63477 0 m /period glyphshow +11.4492 0 m /zero glyphshow +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +0 0 m +0 4 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +406.8 43.2 o +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +0 0 m +0 -4 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +406.8 388.8 o +grestore +/DejaVuSans 12.000 selectfont +gsave + +397.253 30.075 translate +0 rotate +0 0 m /one glyphshow +7.63477 0 m /period glyphshow +11.4492 0 m /five glyphshow +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +0 0 m +0 4 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +518.4 43.2 o +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +0 0 m +0 -4 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +518.4 388.8 o +grestore +/DejaVuSans 12.000 selectfont +gsave + +508.853 30.075 translate +0 rotate +0 0 m /two glyphshow +7.63477 0 m /period glyphshow +11.4492 0 m /zero glyphshow +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +0 0 m +4 0 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +72 43.2 o +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +-0 0 m +-4 0 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +518.4 43.2 o +grestore +/DejaVuSans 12.000 selectfont +gsave + +48.9062 39.8875 translate +0 rotate +0 0 m /one glyphshow +7.63477 0 m /period glyphshow +11.4492 0 m /zero glyphshow +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +0 0 m +4 0 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +72 129.6 o +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +-0 0 m +-4 0 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +518.4 129.6 o +grestore +/DejaVuSans 12.000 selectfont +gsave + +48.9062 126.288 translate +0 rotate +0 0 m /one glyphshow +7.63477 0 m /period glyphshow +11.4492 0 m /five glyphshow +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +0 0 m +4 0 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +72 216 o +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +-0 0 m +-4 0 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +518.4 216 o +grestore +/DejaVuSans 12.000 selectfont +gsave + +48.9062 212.688 translate +0 rotate +0 0 m /two glyphshow +7.63477 0 m /period glyphshow +11.4492 0 m /zero glyphshow +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +0 0 m +4 0 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +72 302.4 o +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +-0 0 m +-4 0 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +518.4 302.4 o +grestore +/DejaVuSans 12.000 selectfont +gsave + +48.9062 299.087 translate +0 rotate +0 0 m /two glyphshow +7.63477 0 m /period glyphshow +11.4492 0 m /five glyphshow +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +0 0 m +4 0 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +72 388.8 o +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +-0 0 m +-4 0 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +518.4 388.8 o +grestore +/DejaVuSans 12.000 selectfont +gsave + +48.9062 385.488 translate +0 rotate +0 0 m /three glyphshow +7.63477 0 m /period glyphshow +11.4492 0 m /zero glyphshow +grestore + +end +showpage diff --git a/tests/baseline/2.0.x/test_format_pdf.pdf b/tests/baseline/2.0.x/test_format_pdf.pdf new file mode 100644 index 00000000..04d2a9f1 Binary files /dev/null and b/tests/baseline/2.0.x/test_format_pdf.pdf differ diff --git a/tests/baseline/2.0.x/test_format_png.png b/tests/baseline/2.0.x/test_format_png.png new file mode 100644 index 00000000..68ac04da Binary files /dev/null and b/tests/baseline/2.0.x/test_format_png.png differ diff --git a/tests/baseline/2.0.x/test_format_svg.svg b/tests/baseline/2.0.x/test_format_svg.svg new file mode 100644 index 00000000..4ea32f65 --- /dev/null +++ b/tests/baseline/2.0.x/test_format_svg.svg @@ -0,0 +1,430 @@ + + + + + + + + 2023-03-30T10:58:11.674243 + image/svg+xml + + + Matplotlib v3.6.3, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/baseline/hashes/mpl20_ft261.json b/tests/baseline/hashes/mpl20_ft261.json index 76322063..7fe1a4e8 100644 --- a/tests/baseline/hashes/mpl20_ft261.json +++ b/tests/baseline/hashes/mpl20_ft261.json @@ -12,5 +12,6 @@ "test_pytest_mpl.test_hash_succeeds": "480062c2239ed9d70e361d1a5b578dc2aa756971161ac6e7287b492ae6118c59", "test.test_modified": "54f6cf83d5b06fa2ecb7fa23d6e87898679178ef5d0dfdd2551a139f1932127b", "test.test_new": "54f6cf83d5b06fa2ecb7fa23d6e87898679178ef5d0dfdd2551a139f1932127b", - "test.test_unmodified": "54f6cf83d5b06fa2ecb7fa23d6e87898679178ef5d0dfdd2551a139f1932127b" + "test.test_unmodified": "54f6cf83d5b06fa2ecb7fa23d6e87898679178ef5d0dfdd2551a139f1932127b", + "test_formats.test_format_png": "480062c2239ed9d70e361d1a5b578dc2aa756971161ac6e7287b492ae6118c59" } diff --git a/tests/baseline/hashes/mpl21_ft261.json b/tests/baseline/hashes/mpl21_ft261.json index 8b2beb5a..caa4f644 100644 --- a/tests/baseline/hashes/mpl21_ft261.json +++ b/tests/baseline/hashes/mpl21_ft261.json @@ -12,5 +12,8 @@ "test_pytest_mpl.test_hash_succeeds": "17b65dd0247b0dfd8c1b4b079352414ae0fe03c0a3e79d63c8b8670d84d4098f", "test.test_modified": "14d326881467bc613e6504b87bd7d556a5e58668ff16b896fa3c15745cfb6336", "test.test_new": "14d326881467bc613e6504b87bd7d556a5e58668ff16b896fa3c15745cfb6336", - "test.test_unmodified": "14d326881467bc613e6504b87bd7d556a5e58668ff16b896fa3c15745cfb6336" + "test.test_unmodified": "14d326881467bc613e6504b87bd7d556a5e58668ff16b896fa3c15745cfb6336", + "test_formats.test_format_eps": "f8a0fbb33dcd473ef5cfdd69317db6eb58d64a7f5f3b5072e0de69aa9e201224", + "test_formats.test_format_pdf": "82b2b58fc3a74591c85cdf2f06b2f72dfc154820fca98e7cfd5cb6904ed60b23", + "test_formats.test_format_png": "d577a3c8c7550413d8d50bc26a68f3e8d9c35d4763c52cbcc15df4f61c8406b2" } diff --git a/tests/baseline/hashes/mpl22_ft261.json b/tests/baseline/hashes/mpl22_ft261.json index 08c26f4d..92b28d02 100644 --- a/tests/baseline/hashes/mpl22_ft261.json +++ b/tests/baseline/hashes/mpl22_ft261.json @@ -12,5 +12,8 @@ "test_pytest_mpl.test_hash_succeeds": "e80557c8784fb920fb79b03b26dc072649a98811f00a8c212df8761e4351acde", "test.test_modified": "80e0ee6df7cf7d9d9407395a25af30beb8763e98820a7be972764899246d2cd7", "test.test_new": "80e0ee6df7cf7d9d9407395a25af30beb8763e98820a7be972764899246d2cd7", - "test.test_unmodified": "80e0ee6df7cf7d9d9407395a25af30beb8763e98820a7be972764899246d2cd7" + "test.test_unmodified": "80e0ee6df7cf7d9d9407395a25af30beb8763e98820a7be972764899246d2cd7", + "test_formats.test_format_eps": "f8a0fbb33dcd473ef5cfdd69317db6eb58d64a7f5f3b5072e0de69aa9e201224", + "test_formats.test_format_pdf": "8963ba9209080091c0961553bdf195cdcd0f2ba29081a122f9aad8e94c444aff", + "test_formats.test_format_png": "d577a3c8c7550413d8d50bc26a68f3e8d9c35d4763c52cbcc15df4f61c8406b2" } diff --git a/tests/baseline/hashes/mpl30_ft261.json b/tests/baseline/hashes/mpl30_ft261.json index 313078ea..198ed755 100644 --- a/tests/baseline/hashes/mpl30_ft261.json +++ b/tests/baseline/hashes/mpl30_ft261.json @@ -12,5 +12,8 @@ "test_pytest_mpl.test_hash_succeeds": "4e1157a93733cdb327f1741afdb0525f4d0e3f12e60b54f72c93db9f9c9ae27f", "test.test_modified": "6e2e4ba7b77caf62df24f6b92d6fc51ab1b837bf98039750334f65c0a6c5d898", "test.test_new": "6e2e4ba7b77caf62df24f6b92d6fc51ab1b837bf98039750334f65c0a6c5d898", - "test.test_unmodified": "6e2e4ba7b77caf62df24f6b92d6fc51ab1b837bf98039750334f65c0a6c5d898" + "test.test_unmodified": "6e2e4ba7b77caf62df24f6b92d6fc51ab1b837bf98039750334f65c0a6c5d898", + "test_formats.test_format_eps": "f8a0fbb33dcd473ef5cfdd69317db6eb58d64a7f5f3b5072e0de69aa9e201224", + "test_formats.test_format_pdf": "8963ba9209080091c0961553bdf195cdcd0f2ba29081a122f9aad8e94c444aff", + "test_formats.test_format_png": "d577a3c8c7550413d8d50bc26a68f3e8d9c35d4763c52cbcc15df4f61c8406b2" } diff --git a/tests/baseline/hashes/mpl31_ft261.json b/tests/baseline/hashes/mpl31_ft261.json index dbde4d54..18a1fabf 100644 --- a/tests/baseline/hashes/mpl31_ft261.json +++ b/tests/baseline/hashes/mpl31_ft261.json @@ -12,5 +12,8 @@ "test_pytest_mpl.test_hash_succeeds": "2a4da3a36b384df539f3f47d476f67a918f5eee1df360dbab9469b96260df78f", "test.test_modified": "3675e5a48388e8cc341580e9b41115d3cf63d2465cf11eeed3faa23e84030fc2", "test.test_new": "3675e5a48388e8cc341580e9b41115d3cf63d2465cf11eeed3faa23e84030fc2", - "test.test_unmodified": "3675e5a48388e8cc341580e9b41115d3cf63d2465cf11eeed3faa23e84030fc2" + "test.test_unmodified": "3675e5a48388e8cc341580e9b41115d3cf63d2465cf11eeed3faa23e84030fc2", + "test_formats.test_format_eps": "108a341ce450cb5adef9f41e27175da1809fcdeb64f17b13f58d3eb0efc08006", + "test_formats.test_format_pdf": "8963ba9209080091c0961553bdf195cdcd0f2ba29081a122f9aad8e94c444aff", + "test_formats.test_format_png": "d577a3c8c7550413d8d50bc26a68f3e8d9c35d4763c52cbcc15df4f61c8406b2" } diff --git a/tests/baseline/hashes/mpl32_ft261.json b/tests/baseline/hashes/mpl32_ft261.json index c39964b0..3cb208ef 100644 --- a/tests/baseline/hashes/mpl32_ft261.json +++ b/tests/baseline/hashes/mpl32_ft261.json @@ -12,5 +12,8 @@ "test_pytest_mpl.test_hash_succeeds": "8b8ff9ce044bc9075876278781667a708414460209bba25a39d8262ed73d0f04", "test.test_modified": "3b7db65812fd59403d17a2fba3ebe1fd0abdfde8633df06636e4e1daea259da0", "test.test_new": "3b7db65812fd59403d17a2fba3ebe1fd0abdfde8633df06636e4e1daea259da0", - "test.test_unmodified": "3b7db65812fd59403d17a2fba3ebe1fd0abdfde8633df06636e4e1daea259da0" + "test.test_unmodified": "3b7db65812fd59403d17a2fba3ebe1fd0abdfde8633df06636e4e1daea259da0", + "test_formats.test_format_eps": "76d0a064029518d1e72c1ff5f159c5e9c3ca77ecb2c7fbab82229aacf61aa4fc", + "test_formats.test_format_pdf": "8963ba9209080091c0961553bdf195cdcd0f2ba29081a122f9aad8e94c444aff", + "test_formats.test_format_png": "d577a3c8c7550413d8d50bc26a68f3e8d9c35d4763c52cbcc15df4f61c8406b2" } diff --git a/tests/baseline/hashes/mpl33_ft261.json b/tests/baseline/hashes/mpl33_ft261.json index 77c5f03f..b8568f2c 100644 --- a/tests/baseline/hashes/mpl33_ft261.json +++ b/tests/baseline/hashes/mpl33_ft261.json @@ -12,5 +12,9 @@ "test_pytest_mpl.test_hash_succeeds": "55ad74a44c99606f1df1e79f67a59a4986bddc2b48ea2b2d7ea8365db91dc7ca", "test.test_modified": "ce07de6b726c3b01afb03aa7c9e939d584bc71a54b9737d69853a0d915cd6181", "test.test_new": "ce07de6b726c3b01afb03aa7c9e939d584bc71a54b9737d69853a0d915cd6181", - "test.test_unmodified": "ce07de6b726c3b01afb03aa7c9e939d584bc71a54b9737d69853a0d915cd6181" + "test.test_unmodified": "ce07de6b726c3b01afb03aa7c9e939d584bc71a54b9737d69853a0d915cd6181", + "test_formats.test_format_eps": "4a605a2cd24101b9292151f5ab6d6846ba1b9c856cfda2bee6a142380e257b04", + "test_formats.test_format_pdf": "34a9eb10372b35c0bd26472e8571a91031c055ab47cc3682ebc0c5e47c2b6cbd", + "test_formats.test_format_png": "e73d228183ddfdced366191399cdecef9685d1248b852162f179750fc7b8b904", + "test_formats.test_format_svg": "b7f85f4b44e0c5871f5cc230b5a9042f2e73aa70384ab584d6cd8cde29344cd2" } diff --git a/tests/test_pytest_mpl.py b/tests/test_pytest_mpl.py index a9d2cba1..23cfe783 100644 --- a/tests/test_pytest_mpl.py +++ b/tests/test_pytest_mpl.py @@ -9,6 +9,7 @@ import matplotlib.ft2font import matplotlib.pyplot as plt import pytest +from matplotlib.testing.compare import converter from packaging.version import Version MPL_VERSION = Version(matplotlib.__version__) @@ -670,3 +671,74 @@ def test_raises(): result = pytester.runpytest(*runpytest_args) result.assert_outcomes(failed=1) result.stdout.fnmatch_lines("FAILED*ValueError*User code*") + + +@pytest.mark.parametrize('use_hash_library', (False, True)) +@pytest.mark.parametrize('passes', (False, True)) +@pytest.mark.parametrize("file_format", ['eps', 'pdf', 'png', 'svg']) +def test_formats(pytester, use_hash_library, passes, file_format): + """ + Note that we don't test all possible formats as some do not compress well + and would bloat the baseline directory. + """ + + if file_format == 'svg' and MPL_VERSION < Version('3.3'): + pytest.skip('SVG comparison is only supported in Matplotlib 3.3 and above') + + if use_hash_library: + + if file_format == 'pdf' and MPL_VERSION < Version('2.1'): + pytest.skip('PDF hashes are only deterministic in Matplotlib 2.1 and above') + elif file_format == 'eps' and MPL_VERSION < Version('2.1'): + pytest.skip('EPS hashes are only deterministic in Matplotlib 2.1 and above') + + if MPL_VERSION >= Version('3.4'): + pytest.skip('No hash library in test suite for Matplotlib >= 3.4') + + if use_hash_library and not sys.platform.startswith('linux'): + pytest.skip('Hashes for vector graphics are only provided in the hash library for Linux') + + if file_format != 'png' and file_format not in converter: + if file_format == 'svg': + pytest.skip('Comparing SVG files requires inkscape to be installed') + else: + pytest.skip('Comparing EPS and PDF files requires ghostscript to be installed') + + if file_format == 'png': + metadata = '{"Software": None}' + elif file_format == 'pdf': + metadata = '{"Creator": None, "Producer": None, "CreationDate": None}' + elif file_format == 'eps': + metadata = '{"Creator": "test"}' + elif file_format == 'svg': + metadata = '{"Date": None}' + + pytester.makepyfile( + f""" + import os + import pytest + import matplotlib.pyplot as plt + @pytest.mark.mpl_image_compare(baseline_dir=r"{baseline_dir_abs}", + {f'hash_library=r"{hash_library}",' if use_hash_library else ''} + tolerance={DEFAULT_TOLERANCE}, + savefig_kwargs={{'format': '{file_format}', + 'metadata': {metadata}}}) + def test_format_{file_format}(): + + # For reproducible EPS output + os.environ['SOURCE_DATE_EPOCH'] = '1680254601' + + # For reproducible SVG output + plt.rcParams['svg.hashsalt'] = 'test' + + fig = plt.figure() + ax = fig.add_subplot(1, 1, 1) + ax.plot([{1 if passes else 3}, 2, 3]) + return fig + """ + ) + result = pytester.runpytest('--mpl', '-rs') + if passes: + result.assert_outcomes(passed=1) + else: + result.assert_outcomes(failed=1)