diff --git a/.travis.yml b/.travis.yml index 6ca23723..0c1a40e5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,46 +1,52 @@ -language: c +language: python -# Setting sudo to false opts in to Travis-CI container-based builds. sudo: false -os: - - linux - - osx - env: global: - SETUP_XVFB=True - - CONDA_DEPENDENCIES="pytest matplotlib nose coverage freetype=2.5.5" - - PIP_DEPENDENCIES="pytest-cov coveralls" - matrix: - - PYTHON_VERSION=3.4 MATPLOTLIB_VERSION=1.5 - - PYTHON_VERSION=2.7 MATPLOTLIB_VERSION=1.5 - - PYTHON_VERSION=3.5 MATPLOTLIB_VERSION=1.5 - - PYTHON_VERSION=2.7 MATPLOTLIB_VERSION=2.0 - - PYTHON_VERSION=3.5 MATPLOTLIB_VERSION=2.0 - - PYTHON_VERSION=3.6 MATPLOTLIB_VERSION=2.0 - - # The following build is meant to check that dependencies get set up correctly, but currently - # the image tests fail, which should be investigated. - # - PYTHON_VERSION=3.5 CONDA_DEPENDENCIES="coverage freetype libpng" + - TOXENV='test' + - TOXARGS='-v' + - TOXPOSARGS='' + +matrix: + include: + # Test the oldest and newest configuration on Mac and Windows + - os: osx + language: c + env: PYTHON_VERSION=3.6 TOXENV=py36-test-mpl20 + - os: osx + language: c + env: PYTHON_VERSION=3.8 TOXENV=py38-test-mpl31 + - os: windows + language: c + env: PYTHON_VERSION=3.6 TOXENV=py36-test-mpl20 + - os: windows + language: c + env: PYTHON_VERSION=3.8 TOXENV=py38-test-mpl31 + # Test all configurations on Linux + - python: 3.6 + env: TOXENV=py36-test-mpl20 + - python: 3.6 + env: TOXENV=py36-test-mpl21 + - python: 3.6 + env: TOXENV=py36-test-mpl22 + - python: 3.7 + env: TOXENV=py37-test-mpl30 + - python: 3.8 + env: TOXENV=py38-test-mpl31 + - python: 3.8 + env: TOXENV=codestyle install: - - # We use the ci-helpers package from the Astropy project to set up conda - # with any requested dependencies above. - - git clone git://github.com/astropy/ci-helpers.git - - source ci-helpers/travis/setup_conda_$TRAVIS_OS_NAME.sh - - # Need to use develop instead of install to make sure coverage works - - python setup.py develop + - if [[ $TRAVIS_OS_NAME != linux ]]; then + git clone git://github.com/astropy/ci-helpers.git; + source ci-helpers/travis/setup_conda.sh; + fi script: - - python -c 'import pytest_mpl.plugin' - - pytest -vv --mpl --cov pytest_mpl tests - # Make sure that the tests run ok even without the --mpl option (we close - # figures anyway in this case) - - pytest -vv --cov pytest_mpl --cov-append tests - - python setup.py check --restructuredtext + - pip install tox + - tox $TOXARGS -- $TOXPOSARGS after_success: - - coveralls + - coveralls diff --git a/CHANGES.md b/CHANGES.md index 260e68d3..8bb042ae 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,8 @@ - Properly register mpl_image_compare marker with pytest. [#83] +- Drop support for Python 3.5 and earlier, and Matplotlib 1.5. [#87] + 0.10 (2018-09-25) ----------------- diff --git a/README.rst b/README.rst index d9ce575d..bde85ebb 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -|Travis Build Status| |AppVeyor Build status| |Coveralls coverage| +|Travis Build Status| |Coveralls coverage| About ----- @@ -6,11 +6,6 @@ About This is a plugin to facilitate image comparison for `Matplotlib `__ figures in pytest. -Matplotlib includes a number of test utilities and decorators, but these -are geared towards the `nose `__ testing -framework. Pytest-mpl makes it easy to compare figures produced by tests -to reference images when using `pytest `__. - For each figure to test, the reference image is subtracted from the generated image, and the RMS of the residual is compared to a user-specified tolerance. If the residual is too large, the test will @@ -23,23 +18,17 @@ section below. Installing ---------- -This plugin is compatible with Python 2.6, 2.7, and 3.3 and later, and -requires `pytest `__, -`matplotlib `__ and -`nose `__ to be installed (nose is -required by Matplotlib). - -To install, you can do: +This plugin is compatible with Python 2.7, and 3.5 and later, and +requires `pytest `__ and +`matplotlib ` to be installed. -:: +To install, you can do:: pip install pytest-mpl -You can check that the plugin is registered with pytest by doing: - -:: +You can check that the plugin is registered with pytest by doing:: - py.test --version + pytest --version which will show a list of plugins: @@ -71,24 +60,20 @@ function returns a Matplotlib figure (or any figure object that has a To generate the baseline images, run the tests with the ``--mpl-generate-path`` option with the name of the directory where the -generated images should be placed: - -:: +generated images should be placed:: - py.test --mpl-generate-path=baseline + pytest --mpl-generate-path=baseline If the directory does not exist, it will be created. The directory will -be interpreted as being relative to where you are running ``py.test``. +be interpreted as being relative to where you are running ``pytest``. Once you are happy with the generated images, you should move them to a sub-directory called ``baseline`` relative to the test files (this name is configurable, see below). You can also generate the baseline images directly in the right directory. -You can then run the tests simply with: +You can then run the tests simply with:: -:: - - py.test --mpl + pytest --mpl and the tests will pass if the images are the same. If you omit the ``--mpl`` option, the tests will run but will only check that the code @@ -145,11 +130,9 @@ a comma-separated list of URLs (real commas in the URL should be encoded as ``%2C``). Finally, you can also set a custom baseline directory globally when -running tests by running ``py.test`` with: - -:: +running tests by running ``pytest`` with:: - py.test --mpl --mpl-baseline-path=baseline_images + pytest --mpl --mpl-baseline-path=baseline_images This directory will be interpreted as being relative to where the tests are run. In addition, if both this option and the ``baseline_dir`` @@ -190,9 +173,7 @@ Test failure example If the images produced by the tests are correct, then the test will pass, but if they are not, the test will fail with a message similar to -the following: - -:: +the following:: E Exception: Error: Image files did not match. E RMS Value: 142.2287807767823 @@ -226,7 +207,7 @@ By default, the expected, actual and difference files are written to a temporary directory with a non-deterministic path. If you want to instead write them to a specific directory, you can use:: - py.test --mpl --mpl-results-path=results + pytest --mpl --mpl-results-path=results The ``results`` directory will then contain one sub-directory per test, and each sub-directory will contain the three files mentioned above. If you are using a @@ -242,20 +223,16 @@ Running the tests for pytest-mpl -------------------------------- If you are contributing some changes and want to run the tests, first -install the latest version of the plugin then do: - -:: +install the latest version of the plugin then do:: cd tests - py.test --mpl + pytest --mpl The reason for having to install the plugin first is to ensure that the plugin is correctly loaded as part of the test suite. .. |Travis Build Status| image:: https://travis-ci.org/matplotlib/pytest-mpl.svg?branch=master :target: https://travis-ci.org/matplotlib/pytest-mpl -.. |AppVeyor Build status| image:: https://ci.appveyor.com/api/projects/status/mf7hs44scg5mvcyo?svg=true - :target: https://ci.appveyor.com/project/astrofrog/pytest-mpl .. |Coveralls coverage| image:: https://coveralls.io/repos/matplotlib/pytest-mpl/badge.svg :target: https://coveralls.io/r/matplotlib/pytest-mpl .. |expected| image:: images/baseline-coords_overlay_auto_coord_meta.png diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 442325bf..00000000 --- a/appveyor.yml +++ /dev/null @@ -1,37 +0,0 @@ -# AppVeyor.com is a Continuous Integration service to build and run tests under -# Windows - -environment: - - global: - PYTHON: "C:\\conda" - MINICONDA_VERSION: "latest" - CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\ci-helpers\\appveyor\\windows_sdk.cmd" - PYTHON_ARCH: "64" # needs to be set for CMD_IN_ENV to succeed. If a mix - # of 32 bit and 64 bit builds are needed, move this - # to the matrix section. - - matrix: - - PYTHON_VERSION: "2.7" - CONDA_DEPENDENCIES: "matplotlib=1.5 nose" - - - PYTHON_VERSION: "3.5" - CONDA_DEPENDENCIES: "matplotlib=1.5 nose" - - - PYTHON_VERSION: "3.6" - CONDA_DEPENDENCIES: "matplotlib=2.0.1 nose" - -platform: - -x64 - -install: - - "git clone git://github.com/astropy/ci-helpers.git" - - "powershell ci-helpers/appveyor/install-miniconda.ps1" - - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - - "activate test" - - "%CMD_IN_ENV% python setup.py install" - -build: false - -test_script: - - "%CMD_IN_ENV% py.test tests --mpl -v" diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index 8096f93e..0736350c 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -63,7 +63,7 @@ def _download_file(baseline, filename): try: u = urlopen(base_url + filename) content = u.read() - except Exception as exc: + except Exception: warnings.warn('Downloading {0} failed'.format(base_url + filename)) else: break @@ -104,7 +104,9 @@ def pytest_addoption(parser): def pytest_configure(config): - config.addinivalue_line('markers', "mpl_image_compare: Compares matplotlib figures against a baseline image") + config.addinivalue_line('markers', + "mpl_image_compare: Compares matplotlib figures " + "against a baseline image") if config.getoption("--mpl") or config.getoption("--mpl-generate-path") is not None: @@ -192,8 +194,8 @@ def pytest_runtest_setup(self, item): if compare is None: return - from PIL import Image import matplotlib + from matplotlib.image import imread import matplotlib.pyplot as plt from matplotlib.testing.compare import compare_images try: @@ -269,23 +271,28 @@ def item_function_wrapper(*args, **kwargs): if baseline_remote: baseline_image_ref = _download_file(baseline_dir, filename) else: - baseline_image_ref = os.path.abspath(os.path.join(os.path.dirname(item.fspath.strpath), baseline_dir, filename)) + baseline_image_ref = os.path.abspath(os.path.join( + os.path.dirname(item.fspath.strpath), baseline_dir, filename)) if not os.path.exists(baseline_image_ref): pytest.fail("Image file not found for comparison test in: " "\n\t{baseline_dir}" "\n(This is expected for new tests.)\nGenerated Image: " - "\n\t{test}".format(baseline_dir=baseline_dir, test=test_image), pytrace=False) + "\n\t{test}".format(baseline_dir=baseline_dir, + test=test_image), + pytrace=False) # distutils may put the baseline images in non-accessible places, # copy to our tmpdir to be sure to keep them in case of failure - baseline_image = os.path.abspath(os.path.join(result_dir, 'baseline-' + filename)) + baseline_image = os.path.abspath(os.path.join(result_dir, + 'baseline-' + filename)) shutil.copyfile(baseline_image_ref, baseline_image) - # Compare image size ourselves since the Matplotlib exception is a bit cryptic in this case - # and doesn't show the filenames - expected_shape = Image.open(baseline_image).size - actual_shape = Image.open(test_image).size + # Compare image size ourselves since the Matplotlib + # exception is a bit cryptic in this case and doesn't show + # the filenames + expected_shape = imread(baseline_image).shape[:2] + actual_shape = imread(test_image).shape[:2] if expected_shape != actual_shape: error = SHAPE_MISMATCH_ERROR.format(expected_path=baseline_image, expected_shape=expected_shape, @@ -305,7 +312,8 @@ def item_function_wrapper(*args, **kwargs): if not os.path.exists(self.generate_dir): os.makedirs(self.generate_dir) - fig.savefig(os.path.abspath(os.path.join(self.generate_dir, filename)), **savefig_kwargs) + fig.savefig(os.path.abspath(os.path.join(self.generate_dir, filename)), + **savefig_kwargs) close_mpl_figure(fig) pytest.skip("Skipping test, since generating data") @@ -331,8 +339,6 @@ def pytest_runtest_setup(self, item): if compare is None: return - import matplotlib.pyplot as plt - original = item.function @wraps(item.function) diff --git a/setup.cfg b/setup.cfg index 0c9e0fc1..a7d968d7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,39 @@ [metadata] license_file = LICENSE +name = pytest-mpl +version = 0.11 +url = https://github.com/matplotlib/pytest-mpl +author = Thomas Robitaille +author_email = thomas.robitaille@gmail.com +classifiers = + Development Status :: 4 - Beta + Framework :: Pytest + Intended Audience :: Developers + Topic :: Software Development :: Testing + Topic :: Scientific/Engineering :: Visualization + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 3 + Operating System :: OS Independent + License :: OSI Approved :: BSD License +license = BSD +description = pytest plugin to help with testing figures output from Matplotlib +long_description = file: README.rst + +[options] +zip_safe = True +packages = find: +install_requires = + pytest + matplotlib + +[options.entry_points] +pytest11 = + pytest_mpl = pytest_mpl.plugin + +[options.extras_require] +test = + pytest-cov + +[tool:pytest] +testpaths = "tests" diff --git a/setup.py b/setup.py index eaead300..6008dbf0 100644 --- a/setup.py +++ b/setup.py @@ -1,38 +1,15 @@ -from setuptools import setup +#!/usr/bin/env python -from pytest_mpl import __version__ +import sys +from distutils.version import LooseVersion -# IMPORTANT: we deliberately use rst here instead of markdown because long_description -# needs to be in rst, and requiring pandoc to be installed to convert markdown to rst -# on-the-fly is over-complicated and sometimes the generated rst has warnings that -# cause PyPI to not display it correctly. +try: + import setuptools + assert LooseVersion(setuptools.__version__) >= LooseVersion('30.3') +except (ImportError, AssertionError): + sys.stderr.write("ERROR: setuptools 30.3 or later is required\n") + sys.exit(1) -with open('README.rst') as infile: - long_description = infile.read() +from setuptools import setup -setup( - version=__version__, - url="https://github.com/matplotlib/pytest-mpl", - name="pytest-mpl", - description='pytest plugin to help with testing figures output from Matplotlib', - long_description=long_description, - packages=['pytest_mpl'], - package_data={'pytest_mpl': ['classic.mplstyle']}, - install_requires=['pytest', 'matplotlib', 'pillow', 'nose'], - license='BSD', - author='Thomas Robitaille', - author_email='thomas.robitaille@gmail.com', - entry_points={'pytest11': ['pytest_mpl = pytest_mpl.plugin']}, - classifiers=[ - 'Development Status :: 4 - Beta', - 'Framework :: Pytest', - 'Intended Audience :: Developers', - 'Topic :: Software Development :: Testing', - 'Topic :: Scientific/Engineering :: Visualization', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 3', - 'Operating System :: OS Independent', - 'License :: OSI Approved :: BSD License', - ], -) +setup(use_scm_version=True) diff --git a/tests/baseline/2.0.x/test_base_style.png b/tests/baseline/2.0.x/test_base_style.png index 3502b495..6b8ca882 100644 Binary files a/tests/baseline/2.0.x/test_base_style.png and b/tests/baseline/2.0.x/test_base_style.png differ diff --git a/tests/test_pytest_mpl.py b/tests/test_pytest_mpl.py index 1fa0379c..ebf4bf88 100644 --- a/tests/test_pytest_mpl.py +++ b/tests/test_pytest_mpl.py @@ -22,8 +22,12 @@ WIN = sys.platform.startswith('win') +# In some cases, the fonts on Windows can be quite different +DEFAULT_TOLERANCE = 10 if WIN else 2 -@pytest.mark.mpl_image_compare(baseline_dir=baseline_dir_local) + +@pytest.mark.mpl_image_compare(baseline_dir=baseline_dir_local, + tolerance=DEFAULT_TOLERANCE) def test_succeeds(): fig = plt.figure() ax = fig.add_subplot(1, 1, 1) @@ -31,7 +35,8 @@ def test_succeeds(): return fig -@pytest.mark.mpl_image_compare(baseline_dir=baseline_dir_remote) +@pytest.mark.mpl_image_compare(baseline_dir=baseline_dir_remote, + tolerance=DEFAULT_TOLERANCE) def test_succeeds_remote(): fig = plt.figure() ax = fig.add_subplot(1, 1, 1) @@ -42,7 +47,8 @@ def test_succeeds_remote(): # The following tries an invalid URL first (or at least a URL where the baseline # image won't exist), but should succeed with the second mirror. @pytest.mark.mpl_image_compare(baseline_dir='http://www.python.org,' + baseline_dir_remote, - filename='test_succeeds_remote.png') + filename='test_succeeds_remote.png', + tolerance=DEFAULT_TOLERANCE) def test_succeeds_faulty_mirror(): fig = plt.figure() ax = fig.add_subplot(1, 1, 1) @@ -52,7 +58,8 @@ def test_succeeds_faulty_mirror(): class TestClass(object): - @pytest.mark.mpl_image_compare(baseline_dir=baseline_dir_local) + @pytest.mark.mpl_image_compare(baseline_dir=baseline_dir_local, + tolerance=DEFAULT_TOLERANCE) def test_succeeds(self): fig = plt.figure() ax = fig.add_subplot(1, 1, 1) @@ -60,13 +67,16 @@ def test_succeeds(self): return fig -@pytest.mark.mpl_image_compare(baseline_dir=baseline_dir_local, savefig_kwargs={'dpi': 30}) +@pytest.mark.mpl_image_compare(baseline_dir=baseline_dir_local, + savefig_kwargs={'dpi': 30}, + tolerance=DEFAULT_TOLERANCE) def test_dpi(): fig = plt.figure() ax = fig.add_subplot(1, 1, 1) ax.plot([1, 2, 3]) return fig + TEST_FAILING = """ import pytest import matplotlib.pyplot as plt @@ -86,11 +96,11 @@ def test_fails(tmpdir): f.write(TEST_FAILING) # If we use --mpl, it should detect that the figure is wrong - code = subprocess.call('{0} -m pytest --mpl {1}'.format(sys.executable, test_file), shell=True) + code = subprocess.call([sys.executable, '-m', 'pytest', '--mpl', test_file]) assert code != 0 # If we don't use --mpl option, the test should succeed - code = subprocess.call('{0} -m pytest {1}'.format(sys.executable, test_file), shell=True) + code = subprocess.call([sys.executable, '-m', 'pytest', test_file]) assert code == 0 @@ -113,14 +123,16 @@ def test_output_dir(tmpdir): # When we run the test, we should get output images where we specify output_dir = tmpdir.join('test_output_dir').strpath - code = subprocess.call('{0} -m pytest --mpl-results-path={1} --mpl {2}'.format(sys.executable, output_dir, test_file), - shell=True) + code = subprocess.call([sys.executable, '-m', 'pytest', + '--mpl-results-path={0}'.format(output_dir), + '--mpl', test_file]) assert code != 0 assert os.path.exists(output_dir) # Listdir() is to get the random name that the output for the one test is written into - assert os.path.exists(os.path.join(output_dir, os.listdir(output_dir)[0], 'test_output_dir.png')) + assert os.path.exists(os.path.join(output_dir, os.listdir(output_dir)[0], + 'test_output_dir.png')) TEST_GENERATE = """ @@ -135,12 +147,6 @@ def test_gen(): """ -# TODO: We skip the following test on Windows since the first subprocess calls. -# This should be fixed in the long term, but is not critical since we already -# test this on Linux. - - -@pytest.mark.skipif("WIN") def test_generate(tmpdir): test_file = tmpdir.join('test.py').strpath @@ -150,13 +156,15 @@ def test_generate(tmpdir): gen_dir = tmpdir.mkdir('spam').mkdir('egg').strpath # If we don't generate, the test will fail - p = subprocess.Popen('{0} -m pytest --mpl {1}'.format(sys.executable, test_file), shell=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - p.wait() - assert b'Image file not found for comparison test' in p.stdout.read() + try: + subprocess.check_output([sys.executable, '-m', 'pytest', '--mpl', test_file]) + except subprocess.CalledProcessError as exc: + assert b'Image file not found for comparison test' in exc.output # If we do generate, the test should succeed and a new file will appear - code = subprocess.call('{0} -m pytest --mpl-generate-path={1} {2}'.format(sys.executable, gen_dir, test_file), shell=True) + code = subprocess.call([sys.executable, '-m', 'pytest', + '--mpl-generate-path={0}'.format(gen_dir), + test_file]) assert code == 0 assert os.path.exists(os.path.join(gen_dir, 'test_gen.png')) @@ -173,9 +181,11 @@ def test_nofigure(): pass -@pytest.mark.skipif(MPL_LT_2, reason="the fivethirtyeight style is only available in Matplotlib 2.0 and later") +@pytest.mark.skipif(MPL_LT_2, reason="the fivethirtyeight style is only available " + "in Matplotlib 2.0 and later") @pytest.mark.mpl_image_compare(baseline_dir=baseline_dir_local, - style='fivethirtyeight') + style='fivethirtyeight', + tolerance=DEFAULT_TOLERANCE) def test_base_style(): fig = plt.figure() ax = fig.add_subplot(1, 1, 1) @@ -198,7 +208,7 @@ def test_remove_text(): def test_parametrized(s): fig = plt.figure() ax = fig.add_subplot(1, 1, 1) - ax.scatter([1,3,4,3,2],[1,4,3,3,1], s=s) + ax.scatter([1, 3, 4, 3, 2], [1, 4, 3, 3, 1], s=s) return fig @@ -210,7 +220,8 @@ def setup_method(self, method): self.x = [1, 2, 3] @pytest.mark.mpl_image_compare(baseline_dir=baseline_dir_local, - filename='test_succeeds.png') + filename='test_succeeds.png', + tolerance=DEFAULT_TOLERANCE) def test_succeeds(self): fig = plt.figure() ax = fig.add_subplot(1, 1, 1) diff --git a/tox.ini b/tox.ini index aeeb1958..63d5ed1c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,40 @@ [tox] -envlist = py27,py33 +envlist = + py{36,37,38}-test + codestyle +requires = + setuptools >= 30.3.0 + pip >= 19.3.1 + [testenv] -deps= - pytest - matplotlib - nose -commands=py.test --mpl tests +passenv = DISPLAY WINDIR +setenv = + MPLBACKEND = Agg +changedir = .tmp/{envname} +description = run tests +deps = + mpl15: matplotlib==1.5.* + mpl15: nose + mpl20: matplotlib==2.0.* + mpl20: nose + mpl21: matplotlib==2.1.* + mpl22: matplotlib==2.2.* + mpl30: matplotlib==3.0.* + mpl31: matplotlib==3.1.* + mpldev: git+https://github.com/matplotlib/matplotlib.git#egg=matplotlib +extras = + test +commands = + pip freeze + # Make sure the tests pass with and without --mpl + pytest '{toxinidir}' {posargs} + pytest '{toxinidir}' --mpl --cov pytest_mpl {posargs} + +[testenv:codestyle] +skip_install = true +changedir = . +description = check code style, e.g. with flake8 +deps = flake8 +commands = + flake8 pytest_mpl tests --count --max-line-length=100 + python setup.py check --restructuredtext