From 76db7895e48589a99389efecb080b8d5e44dbe36 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Wed, 29 Aug 2018 17:15:59 -0700 Subject: [PATCH 1/3] add tooling to freeze upstream requirements --- test/freeze-upstream-requirements.sh | 10 ++++++++++ tox.ini | 27 +++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100755 test/freeze-upstream-requirements.sh diff --git a/test/freeze-upstream-requirements.sh b/test/freeze-upstream-requirements.sh new file mode 100755 index 00000000..293ae16d --- /dev/null +++ b/test/freeze-upstream-requirements.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Minimal wrapper script for upstream requirements install and freeze. +# We do this here rather than as tox commands because tox does not support output redirection. +if [ -z ${1} ]; then + exit 1 +fi + +pip install -r requirements.txt +pip install -r test/requirements.txt +pip freeze > ${1} diff --git a/tox.ini b/tox.ini index 43d3c72f..7ccba842 100644 --- a/tox.ini +++ b/tox.ini @@ -70,6 +70,33 @@ sitepackages = False deps = -rtest/requirements.txt commands = {[testenv:base-command]commands} -m "local and not slow and not veryslow and not nope" +# Collect requirements for use in upstream tests +[testenv:freeze-upstream-requirements-base] +sitepackages = False +skip_install = True +recreate = True +deps = +commands = + {toxinidir}/test/freeze-upstream-requirements.sh + +# Freeze for Python 2.7 +[testenv:freeze-upstream-requirements-py27] +basepython = python2.7 +sitepackages = {[testenv:freeze-upstream-requirements-base]sitepackages} +skip_install = {[testenv:freeze-upstream-requirements-base]skip_install} +recreate = {[testenv:freeze-upstream-requirements-base]recreate} +deps = {[testenv:freeze-upstream-requirements-base]deps} +commands = {[testenv:freeze-upstream-requirements-base]commands} test/upstream-requirements-py27.txt + +# Freeze for Python 3.7 +[testenv:freeze-upstream-requirements-py37] +basepython = python3.7 +sitepackages = {[testenv:freeze-upstream-requirements-base]sitepackages} +skip_install = {[testenv:freeze-upstream-requirements-base]skip_install} +recreate = {[testenv:freeze-upstream-requirements-base]recreate} +deps = {[testenv:freeze-upstream-requirements-base]deps} +commands = {[testenv:freeze-upstream-requirements-base]commands} test/upstream-requirements-py37.txt + # Verify that tests can be successfully run from the source build. [testenv:sourcebuildcheck] basepython = python3 From 4b3a03b6365097eba2aa8625f273dbb2a8269dba Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Wed, 29 Aug 2018 17:34:00 -0700 Subject: [PATCH 2/3] add frozen upstream requirements --- test/upstream-requirements-py27.txt | 63 +++++++++++++++++++++++++++++ test/upstream-requirements-py37.txt | 54 +++++++++++++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 test/upstream-requirements-py27.txt create mode 100644 test/upstream-requirements-py37.txt diff --git a/test/upstream-requirements-py27.txt b/test/upstream-requirements-py27.txt new file mode 100644 index 00000000..8045e37e --- /dev/null +++ b/test/upstream-requirements-py27.txt @@ -0,0 +1,63 @@ +apipkg==1.5 +asn1crypto==0.24.0 +atomicwrites==1.2.0 +attrs==18.1.0 +aws-xray-sdk==0.95 +backports.ssl-match-hostname==3.5.0.1 +backports.tempfile==1.0 +backports.weakref==1.0.post1 +boto==2.49.0 +boto3==1.7.84 +botocore==1.10.84 +certifi==2018.8.24 +cffi==1.11.5 +chardet==3.0.4 +cookies==2.2.1 +coverage==4.5.1 +cryptography==2.3.1 +docker==3.5.0 +docker-pycreds==0.3.0 +docutils==0.14 +ecdsa==0.13 +enum34==1.1.6 +execnet==1.5.0 +funcsigs==1.0.2 +future==0.16.0 +futures==3.2.0 +hypothesis==3.63.0 +idna==2.7 +ipaddress==1.0.22 +Jinja2==2.10 +jmespath==0.9.3 +jsondiff==1.1.1 +jsonpickle==0.9.6 +MarkupSafe==1.0 +mock==2.0.0 +more-itertools==4.3.0 +moto==1.3.5 +pathlib2==2.3.2 +pbr==4.2.0 +pluggy==0.7.1 +py==1.6.0 +pyaml==17.12.1 +pycparser==2.18 +pycryptodome==3.6.6 +pytest==3.7.4 +pytest-cov==2.5.1 +pytest-forked==0.2 +pytest-mock==1.10.0 +pytest-xdist==1.23.0 +python-dateutil==2.7.3 +python-jose==2.0.2 +pytz==2018.5 +PyYAML==3.13 +requests==2.19.1 +responses==0.9.0 +s3transfer==0.1.13 +scandir==1.9.0 +six==1.11.0 +urllib3==1.23 +websocket-client==0.51.0 +Werkzeug==0.14.1 +wrapt==1.10.11 +xmltodict==0.11.0 diff --git a/test/upstream-requirements-py37.txt b/test/upstream-requirements-py37.txt new file mode 100644 index 00000000..c607f207 --- /dev/null +++ b/test/upstream-requirements-py37.txt @@ -0,0 +1,54 @@ +apipkg==1.5 +asn1crypto==0.24.0 +atomicwrites==1.2.0 +attrs==18.1.0 +aws-xray-sdk==0.95 +boto==2.49.0 +boto3==1.7.84 +botocore==1.10.84 +certifi==2018.8.24 +cffi==1.11.5 +chardet==3.0.4 +cookies==2.2.1 +coverage==4.5.1 +cryptography==2.3.1 +docker==3.5.0 +docker-pycreds==0.3.0 +docutils==0.14 +ecdsa==0.13 +execnet==1.5.0 +future==0.16.0 +hypothesis==3.63.0 +idna==2.7 +Jinja2==2.10 +jmespath==0.9.3 +jsondiff==1.1.1 +jsonpickle==0.9.6 +MarkupSafe==1.0 +mock==2.0.0 +more-itertools==4.3.0 +moto==1.3.5 +pbr==4.2.0 +pluggy==0.7.1 +py==1.6.0 +pyaml==17.12.1 +pycparser==2.18 +pycryptodome==3.6.6 +pytest==3.7.4 +pytest-cov==2.5.1 +pytest-forked==0.2 +pytest-mock==1.10.0 +pytest-xdist==1.23.0 +python-dateutil==2.7.3 +python-jose==2.0.2 +pytz==2018.5 +PyYAML==3.13 +requests==2.19.1 +responses==0.9.0 +s3transfer==0.1.13 +six==1.11.0 +urllib3==1.23 +websocket-client==0.51.0 +Werkzeug==0.14.1 +wrapt==1.10.11 +xmltodict==0.11.0 From 46f679719e4cbb6c9ad881b2f0f75ff16c862a46 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Thu, 30 Aug 2018 12:13:43 -0700 Subject: [PATCH 3/3] add testing of upstream requirements --- .travis.yml | 18 ++++++++++++++---- tox.ini | 30 ++++++++++++++++++++++++++---- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index f008e694..0429d18f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,38 +2,48 @@ sudo: false language: python matrix: include: + # CPython 2.7 - python: 2.7 env: TOXENV=py27-local-slow - python: 2.7 env: TOXENV=py27-integ-slow + # CPython 3.4 - python: 3.4 env: TOXENV=py34-local-slow - python: 3.4 env: TOXENV=py34-integ-slow + # CPython 3.5 - python: 3.5 env: TOXENV=py35-local-slow - python: 3.5 env: TOXENV=py35-integ-slow + # CPython 3.6 - python: 3.6 env: TOXENV=py36-local-slow - python: 3.6 env: TOXENV=py36-integ-slow -# Disabled pending completion of integration -# https://github.com/aws/aws-dynamodb-encryption-python/issues/66 + # Upstream tests + - python: 2.7 + env: TOXENV=test-upstream-requirements-py27 + # MyPy + # Disabled pending completion of integration + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 # - python: 3.6 # env: TOXENV=mypy-py2 # - python: 3.6 # env: TOXENV=mypy-py3 + # Security - python: 3.6 env: TOXENV=bandit + # Linting - python: 3.6 env: TOXENV=doc8 - python: 3.6 env: TOXENV=readme - python: 3.6 env: TOXENV=docs -# Disabled pending completion of cleanup -# https://github.com/aws/aws-dynamodb-encryption-python/issues/67 + # Disabled pending completion of cleanup + # https://github.com/aws/aws-dynamodb-encryption-python/issues/67 # - python: 3.6 # env: TOXENV=flake8 # - python: 3.6 diff --git a/tox.ini b/tox.ini index 7ccba842..0e9ac0dc 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,8 @@ envlist = flake8, pylint, flake8-tests, pylint-tests, flake8-examples, pylint-examples, - vulture + vulture, + test-upstream-requirements-py{2,3}7 # Additional environments: # vulture :: Runs vulture. Prone to false-positives. @@ -21,7 +22,7 @@ envlist = # release :: Builds dist files and uploads to pypi pypirc profile. [testenv:base-command] -commands = pytest --basetemp={envtmpdir} -l --cov dynamodb_encryption_sdk {posargs} +commands = pytest --basetemp={envtmpdir} -l --cov dynamodb_encryption_sdk test/ {posargs} [testenv] passenv = @@ -76,8 +77,7 @@ sitepackages = False skip_install = True recreate = True deps = -commands = - {toxinidir}/test/freeze-upstream-requirements.sh +commands = {toxinidir}/test/freeze-upstream-requirements.sh # Freeze for Python 2.7 [testenv:freeze-upstream-requirements-py27] @@ -97,6 +97,28 @@ recreate = {[testenv:freeze-upstream-requirements-base]recreate} deps = {[testenv:freeze-upstream-requirements-base]deps} commands = {[testenv:freeze-upstream-requirements-base]commands} test/upstream-requirements-py37.txt +# Test frozen upstream requirements +[testenv:test-upstream-requirements-base] +sitepackages = False +recreate = True +commands = {[testenv:base-command]commands} -m "local and not slow and not veryslow and not nope" + +# Test frozen upstream requirements for Python 2.7 +[testenv:test-upstream-requirements-py27] +basepython = python2.7 +deps = -rtest/upstream-requirements-py27.txt +sitepackages = {[testenv:test-upstream-requirements-base]sitepackages} +recreate = {[testenv:test-upstream-requirements-base]recreate} +commands = {[testenv:test-upstream-requirements-base]commands} + +# Test frozen upstream requirements for Python 3.7 +[testenv:test-upstream-requirements-py37] +basepython = python3.7 +deps = -rtest/upstream-requirements-py37.txt +sitepackages = {[testenv:test-upstream-requirements-base]sitepackages} +recreate = {[testenv:test-upstream-requirements-base]recreate} +commands = {[testenv:test-upstream-requirements-base]commands} + # Verify that tests can be successfully run from the source build. [testenv:sourcebuildcheck] basepython = python3