From 2ea8081e025acca1353e382ceac315fd515a5e40 Mon Sep 17 00:00:00 2001 From: Jose Corella Date: Sun, 20 Feb 2022 20:15:09 -0800 Subject: [PATCH 1/7] chore: pin linter dependencies --- dev_requirements/linter-requirements.txt | 16 +++++++++ test/pylintrc | 3 ++ test/unit/material_providers/test_aws_kms.py | 2 +- tox.ini | 34 ++++++++------------ 4 files changed, 34 insertions(+), 21 deletions(-) create mode 100644 dev_requirements/linter-requirements.txt diff --git a/dev_requirements/linter-requirements.txt b/dev_requirements/linter-requirements.txt new file mode 100644 index 00000000..ee0eab4d --- /dev/null +++ b/dev_requirements/linter-requirements.txt @@ -0,0 +1,16 @@ +bandit==1.7.0 +black==21.12b0 +doc8==0.10.1 +flake8==4.0.1 +flake8-bugbear==21.11.29 +flake8-docstrings==1.6.0 +flake8-isort==4.1.1 +flake8-print==4.0.0 +isort==5.10.1 +pydocstyle==3.0.0 +pyflakes==2.4.0 +pylint==2.12.2 +readme_renderer==32.0 +seed-isort-config==2.2.0 +vulture==2.3 +sphinx==4.4.0 diff --git a/test/pylintrc b/test/pylintrc index ce2bba60..7ffab6c2 100644 --- a/test/pylintrc +++ b/test/pylintrc @@ -10,10 +10,13 @@ disable = protected-access, # raised when calling _ methods redefined-outer-name, # raised when using pytest-mock unused-argument, # raised when patches and fixtures are needed but not called + unspecified-encoding, # All below are disabled because we need to support Python 2 useless-object-inheritance, raise-missing-from, super-with-arguments, + consider-using-f-string, + redundant-u-string-prefix, [DESIGN] max-args = 10 diff --git a/test/unit/material_providers/test_aws_kms.py b/test/unit/material_providers/test_aws_kms.py index edcd301d..2d14b8e4 100644 --- a/test/unit/material_providers/test_aws_kms.py +++ b/test/unit/material_providers/test_aws_kms.py @@ -225,7 +225,7 @@ def test_loaded_key_infos(): [ pytest.param(val, id=str(val)) for val in all_possible_combinations_kwargs( - dict(), + {}, dict(botocore_session=botocore.session.Session()), dict(grant_tokens=("sdvoaweih", "auwshefiouawh")), dict(material_description={"asoiufeoia": "soajfijewi"}), diff --git a/tox.ini b/tox.ini index 9f752505..05ae8db0 100644 --- a/tox.ini +++ b/tox.ini @@ -216,13 +216,7 @@ commands = [testenv:flake8] basepython = python3 deps = - flake8 - flake8-docstrings - flake8-isort - # https://github.com/PyCQA/pydocstyle/issues/375 - pydocstyle<4.0.0 - # https://github.com/JBKahn/flake8-print/pull/30 - flake8-print>=3.1.0 + -rdev_requirements/linter-requirements.txt commands = flake8 \ src/dynamodb_encryption_sdk/ \ @@ -238,7 +232,7 @@ commands = # Ignore D101-107 docstring requirements for tests # E203 is not PEP8 compliant https://github.com/ambv/black#slices # W503 is not PEP8 compliant https://github.com/ambv/black#line-breaks--binary-operators - --ignore F811,D101,D102,D103,D107,E203,W503 \ + --ignore F811,D101,D102,D103,D107,E203,W503,B011 \ test/ [testenv:flake8-examples] @@ -261,8 +255,7 @@ commands = basepython = python3 deps = {[testenv]deps} - pyflakes - pylint + -rdev_requirements/linter-requirements.txt commands = pylint \ --rcfile=src/pylintrc \ @@ -291,7 +284,7 @@ commands = [testenv:blacken-src] basepython = python3 deps = - black + -rdev_requirements/linter-requirements.txt commands = black --line-length 120 \ src/dynamodb_encryption_sdk/ \ @@ -324,9 +317,8 @@ commands = seed-isort-config [testenv:isort] basepython = python3 -# We need >=5.0.0 because -# several configuration settings changed with 5.0.0 -deps = isort>=5.0.0 +deps = + -rdev_requirements/linter-requirements.txt commands = isort \ src \ test \ @@ -366,27 +358,29 @@ commands = basepython = python3 whitelist_externals = {[testenv:resetdocs]whitelist_externals} deps = - sphinx - doc8 -commands = + -rdev_requirements/linter-requirements.txt +commands = {[testenv:resetdocs]commands} doc8 doc/index.rst doc/lib/ README.rst CHANGELOG.rst [testenv:readme] basepython = python3 -deps = readme_renderer +deps = + -rdev_requirements/linter-requirements.txt commands = python setup.py check -r -s [testenv:bandit] basepython = python3 -deps = bandit +deps = + -rdev_requirements/linter-requirements.txt commands = bandit -r src/dynamodb_encryption_sdk/ # Prone to false positives: only run independently [testenv:vulture] basepython = python3 -deps = vulture +deps = + -rdev_requirements/linter-requirements.txt commands = vulture src/dynamodb_encryption_sdk/ [testenv:linters] From 3de5897f8d7f530d9ed93249f5a448ffa7706f5b Mon Sep 17 00:00:00 2001 From: Jose Corella Date: Sun, 20 Feb 2022 21:11:01 -0800 Subject: [PATCH 2/7] chore: add correct version of setup-python --- .github/workflows/ci_tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index b8e72507..cf9f95cc 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -38,7 +38,7 @@ jobs: # - examples steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v2 with: python-version: 2.7 architecture: ${{ matrix.platform.architecture }} @@ -79,7 +79,7 @@ jobs: # - examples steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v2 with: python-version: ${{ matrix.python }} architecture: ${{ matrix.platform.architecture }} From 806e964d7511a47b07645db52944b43c3cfaab13 Mon Sep 17 00:00:00 2001 From: Jose Corella Date: Sun, 20 Feb 2022 22:05:38 -0800 Subject: [PATCH 3/7] chore: fix pylint --- doc/conf.py | 4 ++-- examples/test/pylintrc | 1 + src/pylintrc | 1 + test/requirements.txt | 3 ++- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 9c0b817c..443cf72e 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -29,7 +29,7 @@ def get_version(): return _release -project = u"dynamodb-encryption-sdk-python" +project = u"dynamodb-encryption-sdk-python" # pylint: disable=redundant-u-string-prefix version = get_version() release = get_release() @@ -53,7 +53,7 @@ def get_version(): source_suffix = ".rst" # The suffix of source filenames. master_doc = "index" # The master toctree document. -copyright = u"%s, Amazon" % datetime.now().year # pylint: disable=redefined-builtin +copyright = u"%s, Amazon" % datetime.now().year # pylint: disable=redefined-builtin,redundant-u-string-prefix # List of directories, relative to source directory, that shouldn't be searched # for source files. diff --git a/examples/test/pylintrc b/examples/test/pylintrc index f4dfcfe6..6f533a47 100644 --- a/examples/test/pylintrc +++ b/examples/test/pylintrc @@ -10,6 +10,7 @@ disable = # pylint does not recognize this duplicate-code, # tests for similar things tend to be similar redefined-outer-name, # raises false positives with fixtures + consider-using-f-string, [DESIGN] max-args = 10 diff --git a/src/pylintrc b/src/pylintrc index bc0406f6..bb705f85 100644 --- a/src/pylintrc +++ b/src/pylintrc @@ -8,6 +8,7 @@ disable = useless-object-inheritance, raise-missing-from, super-with-arguments, + consider-using-f-string [BASIC] # Allow function names up to 50 characters diff --git a/test/requirements.txt b/test/requirements.txt index 24ace5ac..3a3ddc1d 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,7 +1,8 @@ hypothesis>=5.0.0,<6.0.0;python_version>='3' hypothesis==4.57.1;python_version=='2.7' mock -moto>=1.3.8 +moto>=1.3.8;python_version>='3' +moto==1.3.13;python_version=='2.7' pytest>=3.4.0 pytest-cov pytest-mock From 28730d974cb9c15e7005ba255ae9e44781eefafd Mon Sep 17 00:00:00 2001 From: Jose Corella Date: Sun, 20 Feb 2022 22:13:01 -0800 Subject: [PATCH 4/7] chore: python2 support... --- doc/conf.py | 2 +- test/requirements.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/conf.py b/doc/conf.py index 443cf72e..3248bb35 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -29,7 +29,7 @@ def get_version(): return _release -project = u"dynamodb-encryption-sdk-python" # pylint: disable=redundant-u-string-prefix +project = u"dynamodb-encryption-sdk-python" # pylint: disable=redundant-u-string-prefix version = get_version() release = get_release() diff --git a/test/requirements.txt b/test/requirements.txt index 3a3ddc1d..5ceafbca 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,3 +1,4 @@ +rsa==4.5;python_version=='2.7' hypothesis>=5.0.0,<6.0.0;python_version>='3' hypothesis==4.57.1;python_version=='2.7' mock From 984044f5900e9b39026a631f7040797e2499cb6d Mon Sep 17 00:00:00 2001 From: Jose Corella Date: Sun, 20 Feb 2022 23:12:43 -0800 Subject: [PATCH 5/7] chore: more python2 fun --- .github/workflows/ci_tests.yaml | 2 +- test/requirements.txt | 2 -- tox.ini | 9 +++++++++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index cf9f95cc..07f27c7e 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -31,7 +31,7 @@ jobs: - os: macos-latest architecture: x64 category: - - local-fast + - py2-local-fast # These require credentials. # Enable them once we sort how to provide them. # - integ-fast diff --git a/test/requirements.txt b/test/requirements.txt index 5ceafbca..b52945bb 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,9 +1,7 @@ -rsa==4.5;python_version=='2.7' hypothesis>=5.0.0,<6.0.0;python_version>='3' hypothesis==4.57.1;python_version=='2.7' mock moto>=1.3.8;python_version>='3' -moto==1.3.13;python_version=='2.7' pytest>=3.4.0 pytest-cov pytest-mock diff --git a/tox.ini b/tox.ini index 05ae8db0..71099da7 100644 --- a/tox.ini +++ b/tox.ini @@ -212,6 +212,15 @@ commands = {posargs} {[testenv:mypy-coverage]commands} +[testenv:py2-local-fast] +basepython = python2.7 +sitepackages = False +deps = + -rtest/upstream-requirements-py27.txt +commands = + {[testenv:base-command]commands} test/ -m "local and not slow and not veryslow and not nope" + + # Linters [testenv:flake8] basepython = python3 From d9d4bdf27ff092309e29c189c600c85b0e8073fd Mon Sep 17 00:00:00 2001 From: Jose Corella Date: Mon, 21 Feb 2022 09:22:05 -0800 Subject: [PATCH 6/7] chore: fix codebuild --- codebuild/python2.7.yml | 6 ++++-- codebuild/python3.6.yml | 4 +++- codebuild/python3.8.yml | 4 +++- tox.ini | 7 +++++++ 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/codebuild/python2.7.yml b/codebuild/python2.7.yml index fd688d77..6d320370 100644 --- a/codebuild/python2.7.yml +++ b/codebuild/python2.7.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - TOXENV: "py27-integ-slow" + TOXENV: "py2-integ-slow" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- @@ -14,5 +14,7 @@ phases: python: latest build: commands: - - pip install tox + - pyenv install 2.7.17 + - pyenv local 2.7.17 + - pip install tox tox-pyenv - tox diff --git a/codebuild/python3.6.yml b/codebuild/python3.6.yml index 602dc113..864ea3b1 100644 --- a/codebuild/python3.6.yml +++ b/codebuild/python3.6.yml @@ -14,5 +14,7 @@ phases: python: latest build: commands: - - pip install tox + - pyenv install 3.6.15 + - pyenv local 3.6.15 + - pip install tox tox-pyenv - tox diff --git a/codebuild/python3.8.yml b/codebuild/python3.8.yml index 1c1524c8..c0170f2a 100644 --- a/codebuild/python3.8.yml +++ b/codebuild/python3.8.yml @@ -14,5 +14,7 @@ phases: python: latest build: commands: - - pip install tox + - pyenv install 3.8.12 + - pyenv local 3.8.12 + - pip install tox tox-pyenv - tox diff --git a/tox.ini b/tox.ini index 71099da7..a6b5d54e 100644 --- a/tox.ini +++ b/tox.ini @@ -220,6 +220,13 @@ deps = commands = {[testenv:base-command]commands} test/ -m "local and not slow and not veryslow and not nope" +[testenv:py2-integ-slow] +basepython = python2.7 +sitepackages = False +deps = + -rtest/upstream-requirements-py27.txt +commands = + {[testenv:base-command]commands} test/ -m "integ and not ddb_integ and not veryslow and not nope" # Linters [testenv:flake8] From 95b89904525b2a4705cf0b6f50a8a3010e41b484 Mon Sep 17 00:00:00 2001 From: Jose Corella Date: Mon, 21 Feb 2022 11:40:08 -0800 Subject: [PATCH 7/7] chore: fix python2 reqs --- .github/workflows/ci_tests.yaml | 2 +- codebuild/python2.7.yml | 2 +- test/requirements.txt | 3 +++ tox.ini | 17 ----------------- 4 files changed, 5 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 07f27c7e..cf9f95cc 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -31,7 +31,7 @@ jobs: - os: macos-latest architecture: x64 category: - - py2-local-fast + - local-fast # These require credentials. # Enable them once we sort how to provide them. # - integ-fast diff --git a/codebuild/python2.7.yml b/codebuild/python2.7.yml index 6d320370..de92e344 100644 --- a/codebuild/python2.7.yml +++ b/codebuild/python2.7.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - TOXENV: "py2-integ-slow" + TOXENV: "py27-integ-slow" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/test/requirements.txt b/test/requirements.txt index b52945bb..4956ab98 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,6 +2,9 @@ hypothesis>=5.0.0,<6.0.0;python_version>='3' hypothesis==4.57.1;python_version=='2.7' mock moto>=1.3.8;python_version>='3' +# must be less than 2.2.x as Moto removed support for python2 as of moto 2.2.x. +# https://github.com/spulec/moto/issues/3612 +moto>=1.3.8,<2.2.0;python_version=='2.7' pytest>=3.4.0 pytest-cov pytest-mock diff --git a/tox.ini b/tox.ini index a6b5d54e..4cdd4157 100644 --- a/tox.ini +++ b/tox.ini @@ -212,22 +212,6 @@ commands = {posargs} {[testenv:mypy-coverage]commands} -[testenv:py2-local-fast] -basepython = python2.7 -sitepackages = False -deps = - -rtest/upstream-requirements-py27.txt -commands = - {[testenv:base-command]commands} test/ -m "local and not slow and not veryslow and not nope" - -[testenv:py2-integ-slow] -basepython = python2.7 -sitepackages = False -deps = - -rtest/upstream-requirements-py27.txt -commands = - {[testenv:base-command]commands} test/ -m "integ and not ddb_integ and not veryslow and not nope" - # Linters [testenv:flake8] basepython = python3 @@ -378,7 +362,6 @@ deps = commands = {[testenv:resetdocs]commands} doc8 doc/index.rst doc/lib/ README.rst CHANGELOG.rst - [testenv:readme] basepython = python3