From cef1a0ce23d9b9e260a211e040c13c87e5526f1c Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Fri, 12 Jun 2020 16:48:20 -0700 Subject: [PATCH 01/23] chore: force tox to update pip --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index 0305eb5f..89a51fb1 100644 --- a/tox.ini +++ b/tox.ini @@ -52,6 +52,8 @@ passenv = AWS_DEFAULT_REGION sitepackages = False deps = -rtest/requirements.txt +# 'download' forces tox to always upgrade pip to the latest +download = true commands = # Only run small test scenario sets local-fast: {[testenv:base-command]commands} test/ -m "local and not slow and not veryslow and not nope" From 308dbe325af59b02b05bf91df9e28f05dcebb5eb Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Fri, 21 Aug 2020 12:19:37 -0700 Subject: [PATCH 02/23] chore: update isort configuration to 5.0.0 --- setup.cfg | 1 - tox.ini | 15 ++++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/setup.cfg b/setup.cfg index 5949c4dd..fdf72b58 100644 --- a/setup.cfg +++ b/setup.cfg @@ -61,6 +61,5 @@ multi_line_output = 3 include_trailing_comma = True force_grid_wrap = 0 combine_as_imports = True -not_skip = __init__.py known_first_party = dynamodb_encryption_sdk known_third_party =attr,aws_kms_encrypted_client,aws_kms_encrypted_item,aws_kms_encrypted_resource,aws_kms_encrypted_table,boto3,botocore,cryptography,dynamodb_encryption_sdk,functional_test_utils,functional_test_vector_generators,hypothesis,hypothesis_strategies,integration_test_utils,mock,most_recent_provider_encrypted_table,moto,mypy_extensions,pytest,pytest_mock,setuptools,six,wrapped_rsa_encrypted_table,wrapped_symmetric_encrypted_table diff --git a/tox.ini b/tox.ini index 89a51fb1..2b825184 100644 --- a/tox.ini +++ b/tox.ini @@ -303,14 +303,15 @@ commands = seed-isort-config [testenv:isort] basepython = python3 -deps = isort +# We need >=5.0.0 because +# several configuration settings changed with 5.0.0 +deps = isort>=5.0.0 commands = isort \ - -rc \ - src \ - test \ - examples/ \ - doc \ - setup.py \ + src \ + test \ + examples/ \ + doc \ + setup.py \ --skip examples/test/examples_test_utils.py \ {posargs} From 3b74a1192c2b14ff6100d6662eecef67f18b4c43 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Fri, 21 Aug 2020 12:20:09 -0700 Subject: [PATCH 03/23] chore: autoformat --- .../internal/crypto/authentication.py | 1 + src/dynamodb_encryption_sdk/internal/crypto/encryption.py | 1 + .../internal/formatting/deserialize/attribute.py | 1 + .../internal/formatting/material_description.py | 1 + .../internal/formatting/serialize/attribute.py | 1 + src/dynamodb_encryption_sdk/material_providers/aws_kms.py | 3 ++- .../material_providers/store/__init__.py | 2 +- src/dynamodb_encryption_sdk/materials/__init__.py | 1 + src/dynamodb_encryption_sdk/materials/wrapped.py | 7 ++++--- 9 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/dynamodb_encryption_sdk/internal/crypto/authentication.py b/src/dynamodb_encryption_sdk/internal/crypto/authentication.py index c45ead11..24fdc2a1 100644 --- a/src/dynamodb_encryption_sdk/internal/crypto/authentication.py +++ b/src/dynamodb_encryption_sdk/internal/crypto/authentication.py @@ -28,6 +28,7 @@ try: # Python 3.5.0 and 3.5.1 have incompatible typing modules from typing import Text # noqa pylint: disable=unused-import + from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks diff --git a/src/dynamodb_encryption_sdk/internal/crypto/encryption.py b/src/dynamodb_encryption_sdk/internal/crypto/encryption.py index 8283c82a..863ab142 100644 --- a/src/dynamodb_encryption_sdk/internal/crypto/encryption.py +++ b/src/dynamodb_encryption_sdk/internal/crypto/encryption.py @@ -18,6 +18,7 @@ """ try: # Python 3.5.0 and 3.5.1 have incompatible typing modules from typing import Text # noqa pylint: disable=unused-import + from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks diff --git a/src/dynamodb_encryption_sdk/internal/formatting/deserialize/attribute.py b/src/dynamodb_encryption_sdk/internal/formatting/deserialize/attribute.py index 13fe0d7b..fa2eceb2 100644 --- a/src/dynamodb_encryption_sdk/internal/formatting/deserialize/attribute.py +++ b/src/dynamodb_encryption_sdk/internal/formatting/deserialize/attribute.py @@ -32,6 +32,7 @@ try: # Python 3.5.0 and 3.5.1 have incompatible typing modules from typing import Callable, Dict, List, Text, Union # noqa pylint: disable=unused-import + from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import,ungrouped-imports except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks diff --git a/src/dynamodb_encryption_sdk/internal/formatting/material_description.py b/src/dynamodb_encryption_sdk/internal/formatting/material_description.py index aa0114c8..0b7ca10e 100644 --- a/src/dynamodb_encryption_sdk/internal/formatting/material_description.py +++ b/src/dynamodb_encryption_sdk/internal/formatting/material_description.py @@ -30,6 +30,7 @@ try: # Python 3.5.0 and 3.5.1 have incompatible typing modules from typing import Dict, Text # noqa pylint: disable=unused-import + from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks diff --git a/src/dynamodb_encryption_sdk/internal/formatting/serialize/attribute.py b/src/dynamodb_encryption_sdk/internal/formatting/serialize/attribute.py index cf2cca83..67651d3d 100644 --- a/src/dynamodb_encryption_sdk/internal/formatting/serialize/attribute.py +++ b/src/dynamodb_encryption_sdk/internal/formatting/serialize/attribute.py @@ -29,6 +29,7 @@ try: # Python 3.5.0 and 3.5.1 have incompatible typing modules from typing import Callable # noqa pylint: disable=unused-import + from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import,ungrouped-imports except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks diff --git a/src/dynamodb_encryption_sdk/material_providers/aws_kms.py b/src/dynamodb_encryption_sdk/material_providers/aws_kms.py index 7fa47854..7bcad4c4 100644 --- a/src/dynamodb_encryption_sdk/material_providers/aws_kms.py +++ b/src/dynamodb_encryption_sdk/material_providers/aws_kms.py @@ -37,8 +37,9 @@ from . import CryptographicMaterialsProvider try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import from typing import Dict, Optional, Text, Tuple # noqa pylint: disable=unused-import + + from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass diff --git a/src/dynamodb_encryption_sdk/material_providers/store/__init__.py b/src/dynamodb_encryption_sdk/material_providers/store/__init__.py index d817976c..e03b57b9 100644 --- a/src/dynamodb_encryption_sdk/material_providers/store/__init__.py +++ b/src/dynamodb_encryption_sdk/material_providers/store/__init__.py @@ -21,7 +21,7 @@ ) try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Dict, Text, Optional # noqa pylint: disable=unused-import + from typing import Dict, Optional, Text # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass diff --git a/src/dynamodb_encryption_sdk/materials/__init__.py b/src/dynamodb_encryption_sdk/materials/__init__.py index ba07d0c8..66797ec6 100644 --- a/src/dynamodb_encryption_sdk/materials/__init__.py +++ b/src/dynamodb_encryption_sdk/materials/__init__.py @@ -19,6 +19,7 @@ try: # Python 3.5.0 and 3.5.1 have incompatible typing modules from typing import Dict, Text # noqa pylint: disable=unused-import + from mypy_extensions import NoReturn # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks diff --git a/src/dynamodb_encryption_sdk/materials/wrapped.py b/src/dynamodb_encryption_sdk/materials/wrapped.py index daf8638a..f85924ea 100644 --- a/src/dynamodb_encryption_sdk/materials/wrapped.py +++ b/src/dynamodb_encryption_sdk/materials/wrapped.py @@ -101,9 +101,10 @@ def __attrs_post_init__(self): self._content_key_from_material_description() ) # noqa pylint: disable=attribute-defined-outside-init else: - self._content_key, self._material_description = ( - self._generate_content_key() - ) # noqa pylint: disable=attribute-defined-outside-init + ( + self._content_key, + self._material_description, + ) = self._generate_content_key() # noqa pylint: disable=attribute-defined-outside-init @staticmethod def _wrapping_transformation(algorithm): From 2fabdc732142bd4f595462cdbbce853e172a7576 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Fri, 21 Aug 2020 12:57:51 -0700 Subject: [PATCH 04/23] chore: hypothesis.HealthCheck.hung_test is deprecated --- test/functional/hypothesis_strategies.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/functional/hypothesis_strategies.py b/test/functional/hypothesis_strategies.py index 3550a411..d0fb7c74 100644 --- a/test/functional/hypothesis_strategies.py +++ b/test/functional/hypothesis_strategies.py @@ -22,7 +22,6 @@ suppress_health_check=( hypothesis.HealthCheck.too_slow, hypothesis.HealthCheck.data_too_large, - hypothesis.HealthCheck.hung_test, hypothesis.HealthCheck.large_base_example, ), deadline=None, From 933856712fa72b49c632428bca818c0ebc186938 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Fri, 21 Aug 2020 12:59:32 -0700 Subject: [PATCH 05/23] chore: flake8 linting --- test/functional/delegated_keys/test_jce.py | 2 +- test/functional/functional_test_utils.py | 6 ++---- test/functional/material_providers/store/test_meta.py | 7 +------ test/functional/material_providers/test_most_recent.py | 9 +++------ test/functional/test_structures.py | 10 ++++------ test/integration/integration_test_utils.py | 3 ++- test/integration/material_providers/store/test_meta.py | 2 +- .../integration/material_providers/test_most_recent.py | 8 +++----- test/unit/internal/formatting/test_attribute.py | 7 +++++-- 9 files changed, 22 insertions(+), 32 deletions(-) diff --git a/test/functional/delegated_keys/test_jce.py b/test/functional/delegated_keys/test_jce.py index ba880399..08f5446e 100644 --- a/test/functional/delegated_keys/test_jce.py +++ b/test/functional/delegated_keys/test_jce.py @@ -78,7 +78,7 @@ def build_short_key_cases(): @pytest.mark.parametrize("algorithm, key_bits, too_short, error_message", build_short_key_cases()) def test_warn_on_short_keys(caplog, algorithm, key_bits, too_short, error_message): with caplog.at_level(logging.DEBUG): - _test = JceNameLocalDelegatedKey.generate(algorithm, key_bits) + _test = JceNameLocalDelegatedKey.generate(algorithm, key_bits) # noqa=F401 logging_results = caplog.text assert (too_short and error_message in logging_results) or (not too_short and error_message not in logging_results) diff --git a/test/functional/functional_test_utils.py b/test/functional/functional_test_utils.py index 950aee47..1b9ab7fb 100644 --- a/test/functional/functional_test_utils.py +++ b/test/functional/functional_test_utils.py @@ -520,9 +520,7 @@ def cycle_batch_item_check( def cycle_batch_writer_check(raw_table, encrypted_table, initial_actions, initial_item): - """Check that cycling (plaintext->encrypted->decrypted) items with the Table batch writer - has the expected results. - """ + """Cycling (plaintext->encrypted->decrypted) items with the Table batch writer should have the expected results.""" check_attribute_actions = initial_actions.copy() check_attribute_actions.set_index_keys(*list(TEST_KEY.keys())) items = _generate_items(initial_item, _nop_transformer) @@ -757,7 +755,7 @@ def client_cycle_batch_items_check_scan_paginator( scan the table with raw client paginator to get encrypted items, scan the table with encrypted client paginator to get decrypted items, then verify that all items appear to have been encrypted correctly. - """ + """ # noqa=D401 kwargs = {} if region_name is not None: kwargs["region_name"] = region_name diff --git a/test/functional/material_providers/store/test_meta.py b/test/functional/material_providers/store/test_meta.py index 908fc9dc..57d5611a 100644 --- a/test/functional/material_providers/store/test_meta.py +++ b/test/functional/material_providers/store/test_meta.py @@ -11,18 +11,13 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. """Functional tests for ``dynamodb_encryption_sdk.material_providers.store.meta``.""" -import base64 -import os - -import boto3 import pytest -from moto import mock_dynamodb2 from dynamodb_encryption_sdk.exceptions import NoKnownVersionError from dynamodb_encryption_sdk.material_providers.store.meta import MetaStore, MetaStoreAttributeNames from dynamodb_encryption_sdk.material_providers.wrapped import WrappedCryptographicMaterialsProvider -from ...functional_test_utils import build_static_jce_cmp, mock_metastore +from ...functional_test_utils import mock_metastore # noqa=F401 pylint: disable=unused-import pytestmark = [pytest.mark.functional, pytest.mark.local] diff --git a/test/functional/material_providers/test_most_recent.py b/test/functional/material_providers/test_most_recent.py index 1d41ef83..9af56166 100644 --- a/test/functional/material_providers/test_most_recent.py +++ b/test/functional/material_providers/test_most_recent.py @@ -21,12 +21,9 @@ from dynamodb_encryption_sdk.material_providers.most_recent import MostRecentProvider from dynamodb_encryption_sdk.material_providers.store import ProviderStore -from ..functional_test_utils import ( # pylint: disable=unused-import - TEST_TABLE_NAME, - check_metastore_cache_use_encrypt, - example_table, - mock_metastore, -) +from ..functional_test_utils import example_table # noqa=F401 pylint: disable=unused-import +from ..functional_test_utils import mock_metastore # noqa=F401 pylint: disable=unused-import +from ..functional_test_utils import TEST_TABLE_NAME, check_metastore_cache_use_encrypt pytestmark = [pytest.mark.functional, pytest.mark.local] diff --git a/test/functional/test_structures.py b/test/functional/test_structures.py index 0c180fe9..76c1398f 100644 --- a/test/functional/test_structures.py +++ b/test/functional/test_structures.py @@ -18,12 +18,10 @@ from dynamodb_encryption_sdk.identifiers import CryptoAction from dynamodb_encryption_sdk.structures import AttributeActions, TableIndex, TableInfo -from .functional_test_utils import ( - TEST_TABLE_NAME, - example_table, - table_with_global_secondary_indexes, - table_with_local_secondary_indexes, -) +from .functional_test_utils import example_table # noqa=F401 pylint: disable=unused-import +from .functional_test_utils import table_with_global_secondary_indexes # noqa=F401 pylint: disable=unused-import +from .functional_test_utils import table_with_local_secondary_indexes # noqa=F401 pylint: disable=unused-import +from .functional_test_utils import TEST_TABLE_NAME pytestmark = [pytest.mark.functional, pytest.mark.local] diff --git a/test/integration/integration_test_utils.py b/test/integration/integration_test_utils.py index 7eb3ea7d..5c43e452 100644 --- a/test/integration/integration_test_utils.py +++ b/test/integration/integration_test_utils.py @@ -20,7 +20,8 @@ # convenience imports try: - from ..functional import functional_test_utils, hypothesis_strategies + from ..functional import hypothesis_strategies # noqa=F401 pylint: disable=unused-import + from ..functional import functional_test_utils except (ImportError, ValueError, SystemError): if "AWS_ENCRYPTION_SDK_EXAMPLES_TESTING" not in os.environ: raise diff --git a/test/integration/material_providers/store/test_meta.py b/test/integration/material_providers/store/test_meta.py index c092ba21..7ee28deb 100644 --- a/test/integration/material_providers/store/test_meta.py +++ b/test/integration/material_providers/store/test_meta.py @@ -16,7 +16,7 @@ from dynamodb_encryption_sdk.exceptions import NoKnownVersionError from dynamodb_encryption_sdk.material_providers.store.meta import MetaStore, MetaStoreAttributeNames -from ...integration_test_utils import temp_metastore # pylint: disable=unused-import +from ...integration_test_utils import temp_metastore # noqa=F401 pylint: disable=unused-import pytestmark = [pytest.mark.integ, pytest.mark.ddb_integ] diff --git a/test/integration/material_providers/test_most_recent.py b/test/integration/material_providers/test_most_recent.py index 90c2ccf8..ba843cac 100644 --- a/test/integration/material_providers/test_most_recent.py +++ b/test/integration/material_providers/test_most_recent.py @@ -13,11 +13,9 @@ """Load testing using MostRecentProvider and MetaStore.""" import pytest -from ..integration_test_utils import ( # pylint: disable=unused-import - ddb_table_name, - functional_test_utils, - temp_metastore, -) +from ..integration_test_utils import ddb_table_name # noqa=F401 pylint: disable=unused-import +from ..integration_test_utils import temp_metastore # noqa=F401 pylint: disable=unused-import +from ..integration_test_utils import functional_test_utils pytestmark = [pytest.mark.integ, pytest.mark.ddb_integ] diff --git a/test/unit/internal/formatting/test_attribute.py b/test/unit/internal/formatting/test_attribute.py index 71235e0c..5fc0617f 100644 --- a/test/unit/internal/formatting/test_attribute.py +++ b/test/unit/internal/formatting/test_attribute.py @@ -10,8 +10,11 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Unit tests for ``dynamodb_encryption_sdk.internal.formatting.serialize.attribute`` -and ``dynamodb_encryption_sdk.internal.formatting.deserialize.attribute``.""" +"""De/serialization unit tests. + +Tests for ``dynamodb_encryption_sdk.internal.formatting.serialize.attribute`` +and ``dynamodb_encryption_sdk.internal.formatting.deserialize.attribute``. +""" import pytest from dynamodb_encryption_sdk.internal.formatting.serialize.attribute import _sorted_key_map From e2733e242cb6c85b63a097cd2915a1df96a1c2f1 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Fri, 21 Aug 2020 16:06:52 -0700 Subject: [PATCH 06/23] chore: update pylint and flake8 configs --- setup.cfg | 2 +- src/pylintrc | 5 ++++- test/pylintrc | 5 ++++- tox.ini | 6 ++++-- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/setup.cfg b/setup.cfg index fdf72b58..b693d2e9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,7 +32,7 @@ log_level=DEBUG # Flake8 Configuration [flake8] -max_complexity = 10 +max_complexity = 11 max_line_length = 120 import_order_style = google application_import_names = dynamodb_encryption_sdk diff --git a/src/pylintrc b/src/pylintrc index bd252833..bc0406f6 100644 --- a/src/pylintrc +++ b/src/pylintrc @@ -3,8 +3,11 @@ disable = bad-continuation, # we let black handle this ungrouped-imports, # we let isort handle this - useless-object-inheritance, # we need to support Python 2, so no, not useless duplicate-code, # causes lots of problems with implementations of common interfaces + # All below are disabled because we need to support Python 2 + useless-object-inheritance, + raise-missing-from, + super-with-arguments, [BASIC] # Allow function names up to 50 characters diff --git a/test/pylintrc b/test/pylintrc index 123c1870..ce2bba60 100644 --- a/test/pylintrc +++ b/test/pylintrc @@ -6,11 +6,14 @@ disable = bad-continuation, # we let black handle this ungrouped-imports, # we let isort handle this no-member, # raised on patched objects with mock checks - useless-object-inheritance, # we need to support Python 2, so no, not useless duplicate-code, # unit tests for similar things tend to be similar protected-access, # raised when calling _ methods redefined-outer-name, # raised when using pytest-mock unused-argument, # raised when patches and fixtures are needed but not called + # All below are disabled because we need to support Python 2 + useless-object-inheritance, + raise-missing-from, + super-with-arguments, [DESIGN] max-args = 10 diff --git a/tox.ini b/tox.ini index 2b825184..854303a5 100644 --- a/tox.ini +++ b/tox.ini @@ -217,8 +217,10 @@ deps = {[testenv:flake8]deps} commands = flake8 \ # Ignore F811 redefinition errors in tests (breaks with pytest-mock use) - # Ignore D103 docstring requirements for tests - --ignore F811,D103 \ + # Ignore D101-107 docstring requirements for tests + # E203 is not PEP8 compliant https://github.com/ambv/black#slices + # W503 is not PEP8 compliant https://github.com/ambv/black#line-breaks--binary-operators + --ignore F811,D101,D102,D103,D107,E203,W503 \ test/ [testenv:flake8-examples] From a21d2f50e8cc89807eb91ec2e00ea9145e0494fe Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Fri, 21 Aug 2020 16:07:08 -0700 Subject: [PATCH 07/23] chore: linting fixes --- src/dynamodb_encryption_sdk/structures.py | 2 +- test/acceptance/acceptance_test_utils.py | 4 ++-- test/functional/functional_test_utils.py | 8 +++++--- test/functional/materials/test_raw.py | 4 ++-- test/integration/integration_test_utils.py | 4 ++-- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/dynamodb_encryption_sdk/structures.py b/src/dynamodb_encryption_sdk/structures.py index 0dd35461..61b329c6 100644 --- a/src/dynamodb_encryption_sdk/structures.py +++ b/src/dynamodb_encryption_sdk/structures.py @@ -223,7 +223,7 @@ class TableIndex(object): partition = attr.ib(validator=attr.validators.instance_of(six.string_types)) sort = attr.ib(validator=attr.validators.optional(attr.validators.instance_of(six.string_types)), default=None) - def __init__(self, partition, sort=None): + def __init__(self, partition, sort=None): # noqa=D107 # type: (Text, Optional[Text]) -> None # Workaround pending resolution of attrs/mypy interaction. # https://github.com/python/mypy/issues/2088 diff --git a/test/acceptance/acceptance_test_utils.py b/test/acceptance/acceptance_test_utils.py index 5d66bc65..482d182a 100644 --- a/test/acceptance/acceptance_test_utils.py +++ b/test/acceptance/acceptance_test_utils.py @@ -159,12 +159,12 @@ def _meta_table_prep(table_name, items_filename): table_data = json.load(f) request_items = {} - for table_name, items in table_data.items(): + for this_table_name, items in table_data.items(): requests = [] for item in items: _decode_item(item) requests.append({"PutRequest": {"Item": item}}) - request_items[table_name] = requests + request_items[this_table_name] = requests client.batch_write_item(RequestItems=request_items) diff --git a/test/functional/functional_test_utils.py b/test/functional/functional_test_utils.py index 1b9ab7fb..39ab9e02 100644 --- a/test/functional/functional_test_utils.py +++ b/test/functional/functional_test_utils.py @@ -178,7 +178,8 @@ class PassThroughCryptographicMaterialsProviderThatRequiresAttributes(Cryptograp def __init__(self, passthrough_cmp): self._passthrough_cmp = passthrough_cmp - def _assert_attributes_set(self, encryption_context): + @staticmethod + def _assert_attributes_set(encryption_context): # type: (EncryptionContext) -> None if not encryption_context.attributes: raise ValueError("Encryption context attributes MUST be set!") @@ -386,7 +387,7 @@ def diverse_item(): return copy.deepcopy(base_item) -_reserved_attributes = set([attr.value for attr in ReservedAttributes]) +_reserved_attributes = {attr.value for attr in ReservedAttributes} def return_requestitems_as_unprocessed(*args, **kwargs): @@ -529,7 +530,7 @@ def cycle_batch_writer_check(raw_table, encrypted_table, initial_actions, initia for item in items: writer.put_item(item) - ddb_keys = [key for key in TEST_BATCH_KEYS] + ddb_keys = TEST_BATCH_KEYS.copy() encrypted_items = [raw_table.get_item(Key=key, ConsistentRead=True)["Item"] for key in ddb_keys] check_many_encrypted_items( actual=encrypted_items, expected=items, attribute_actions=check_attribute_actions, transformer=_nop_transformer @@ -756,6 +757,7 @@ def client_cycle_batch_items_check_scan_paginator( scan the table with encrypted client paginator to get decrypted items, then verify that all items appear to have been encrypted correctly. """ # noqa=D401 + # pylint: disable=too-many-locals kwargs = {} if region_name is not None: kwargs["region_name"] = region_name diff --git a/test/functional/materials/test_raw.py b/test/functional/materials/test_raw.py index f093f811..65cb0c62 100644 --- a/test/functional/materials/test_raw.py +++ b/test/functional/materials/test_raw.py @@ -24,7 +24,7 @@ def test_no_encryption_key(): encryption_materials = RawEncryptionMaterials(signing_key=signing_key) with pytest.raises(AttributeError) as excinfo: - encryption_materials.encryption_key + encryption_materials.encryption_key # calls a property, so pylint: disable=pointless-statement excinfo.match("No encryption key available") @@ -34,6 +34,6 @@ def test_no_decryption_key(): decryption_materials = RawDecryptionMaterials(verification_key=verification_key) with pytest.raises(AttributeError) as excinfo: - decryption_materials.decryption_key + decryption_materials.decryption_key # calls a property, so pylint: disable=pointless-statement excinfo.match("No decryption key available") diff --git a/test/integration/integration_test_utils.py b/test/integration/integration_test_utils.py index 5c43e452..71106ef7 100644 --- a/test/integration/integration_test_utils.py +++ b/test/integration/integration_test_utils.py @@ -54,8 +54,8 @@ def _build_kms_cmp(require_attributes): inner_cmp = AwsKmsCryptographicMaterialsProvider(key_id=cmk_arn_value()) if require_attributes: return functional_test_utils.PassThroughCryptographicMaterialsProviderThatRequiresAttributes(inner_cmp) - else: - return inner_cmp + + return inner_cmp def set_parameterized_kms_cmps(metafunc, require_attributes=True): From fdda1191c52bf35d23e4a479f908ecaed11dd36d Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Fri, 21 Aug 2020 16:09:07 -0700 Subject: [PATCH 08/23] chore: update default Python envlist - remove 3.4 - add 3.8 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 854303a5..53c4ddc4 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist = - py{27,34,35,36,37}-{local,integ,ddb,examples}-fast, + py{27,35,36,37,38}-{local,integ,ddb,examples}-fast, nocmk, sourcebuildcheck, docs, bandit, doc8, readme, flake8{,-tests,-examples}, pylint{,-tests,-examples}, From f6b2a4ecfba5b17f5d9bd77a1b77c63756223bf4 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Mon, 24 Aug 2020 12:36:18 -0700 Subject: [PATCH 09/23] chore: Python 2 lists do not have copy() --- test/functional/functional_test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/functional_test_utils.py b/test/functional/functional_test_utils.py index 39ab9e02..56256296 100644 --- a/test/functional/functional_test_utils.py +++ b/test/functional/functional_test_utils.py @@ -530,7 +530,7 @@ def cycle_batch_writer_check(raw_table, encrypted_table, initial_actions, initia for item in items: writer.put_item(item) - ddb_keys = TEST_BATCH_KEYS.copy() + ddb_keys = copy.copy(TEST_BATCH_KEYS) encrypted_items = [raw_table.get_item(Key=key, ConsistentRead=True)["Item"] for key in ddb_keys] check_many_encrypted_items( actual=encrypted_items, expected=items, attribute_actions=check_attribute_actions, transformer=_nop_transformer From cf9237477f376b1ac42af5eaccccbabe053639e6 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Mon, 24 Aug 2020 13:33:17 -0700 Subject: [PATCH 10/23] chore: address all pylint issues aside from TODO references --- setup.py | 2 +- src/dynamodb_encryption_sdk/encrypted/item.py | 2 +- .../encrypted/resource.py | 4 +- .../encrypted/table.py | 2 +- src/dynamodb_encryption_sdk/identifiers.py | 4 +- .../internal/crypto/authentication.py | 11 +++-- .../internal/crypto/encryption.py | 6 ++- .../crypto/jce_bridge/authentication.py | 2 +- .../internal/crypto/jce_bridge/primitives.py | 14 ++---- .../internal/dynamodb_types.py | 1 + .../formatting/deserialize/attribute.py | 46 +++++++++++++------ .../formatting/material_description.py | 8 +++- .../formatting/serialize/attribute.py | 36 ++++++++++----- .../internal/identifiers.py | 2 +- .../material_providers/aws_kms.py | 1 + .../material_providers/most_recent.py | 10 ++-- 16 files changed, 97 insertions(+), 54 deletions(-) diff --git a/setup.py b/setup.py index a1c79400..2dc604cd 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ def get_version(): def get_requirements(): """Reads the requirements file.""" requirements = read("requirements.txt") - return [r for r in requirements.strip().splitlines()] + return requirements.strip().splitlines() setup( diff --git a/src/dynamodb_encryption_sdk/encrypted/item.py b/src/dynamodb_encryption_sdk/encrypted/item.py index 25f1ef50..b491a34a 100644 --- a/src/dynamodb_encryption_sdk/encrypted/item.py +++ b/src/dynamodb_encryption_sdk/encrypted/item.py @@ -171,7 +171,7 @@ def decrypt_dynamodb_item(item, crypto_config): :rtype: dict """ unique_actions = set([crypto_config.attribute_actions.default_action.name]) - unique_actions.update(set([action.name for action in crypto_config.attribute_actions.attribute_actions.values()])) + unique_actions.update({action.name for action in crypto_config.attribute_actions.attribute_actions.values()}) if crypto_config.attribute_actions.take_no_actions: # If we explicitly have been told not to do anything to this item, just copy it. diff --git a/src/dynamodb_encryption_sdk/encrypted/resource.py b/src/dynamodb_encryption_sdk/encrypted/resource.py index 58b5c776..b5b71f8b 100644 --- a/src/dynamodb_encryption_sdk/encrypted/resource.py +++ b/src/dynamodb_encryption_sdk/encrypted/resource.py @@ -41,7 +41,7 @@ @attr.s(init=False) class EncryptedTablesCollectionManager(object): - # pylint: disable=too-few-public-methods + # pylint: disable=too-few-public-methods,too-many-instance-attributes """Tables collection manager that provides :class:`EncryptedTable` objects. https://boto3.readthedocs.io/en/latest/reference/services/dynamodb.html#DynamoDB.ServiceResource.tables @@ -119,7 +119,7 @@ def _transform_table(self, method, **kwargs): @attr.s(init=False) class EncryptedResource(object): - # pylint: disable=too-few-public-methods + # pylint: disable=too-few-public-methods,too-many-instance-attributes """High-level helper class to provide a familiar interface to encrypted tables. >>> import boto3 diff --git a/src/dynamodb_encryption_sdk/encrypted/table.py b/src/dynamodb_encryption_sdk/encrypted/table.py index 51c69b74..128cb896 100644 --- a/src/dynamodb_encryption_sdk/encrypted/table.py +++ b/src/dynamodb_encryption_sdk/encrypted/table.py @@ -42,7 +42,7 @@ @attr.s(init=False) class EncryptedTable(object): - # pylint: disable=too-few-public-methods + # pylint: disable=too-few-public-methods,too-many-instance-attributes """High-level helper class to provide a familiar interface to encrypted tables. >>> import boto3 diff --git a/src/dynamodb_encryption_sdk/identifiers.py b/src/dynamodb_encryption_sdk/identifiers.py index 9187ecb9..e0156b06 100644 --- a/src/dynamodb_encryption_sdk/identifiers.py +++ b/src/dynamodb_encryption_sdk/identifiers.py @@ -35,12 +35,12 @@ def __gt__(self, other): def __lt__(self, other): # type: (CryptoAction) -> bool """Define CryptoAction equality.""" - return self.value < other.value + return self.value < other.value # pylint: disable=comparison-with-callable def __eq__(self, other): # type: (CryptoAction) -> bool """Define CryptoAction equality.""" - return self.value == other.value + return self.value == other.value # pylint: disable=comparison-with-callable class EncryptionKeyType(Enum): diff --git a/src/dynamodb_encryption_sdk/internal/crypto/authentication.py b/src/dynamodb_encryption_sdk/internal/crypto/authentication.py index 24fdc2a1..d5247688 100644 --- a/src/dynamodb_encryption_sdk/internal/crypto/authentication.py +++ b/src/dynamodb_encryption_sdk/internal/crypto/authentication.py @@ -56,7 +56,8 @@ def sign_item(encrypted_item, signing_key, crypto_config): attribute_actions=crypto_config.attribute_actions, ), ) - return {Tag.BINARY.dynamodb_tag: signature} + # for some reason pylint can't follow the Enum member attributes + return {Tag.BINARY.dynamodb_tag: signature} # pylint: disable=no-member def verify_item_signature(signature_attribute, encrypted_item, verification_key, crypto_config): @@ -68,7 +69,8 @@ def verify_item_signature(signature_attribute, encrypted_item, verification_key, :param DelegatedKey verification_key: DelegatedKey to use to calculate the signature :param CryptoConfig crypto_config: Cryptographic configuration """ - signature = signature_attribute[Tag.BINARY.dynamodb_tag] + # for some reason pylint can't follow the Enum member attributes + signature = signature_attribute[Tag.BINARY.dynamodb_tag] # pylint: disable=no-member verification_key.verify( algorithm=verification_key.algorithm, signature=signature, @@ -98,10 +100,11 @@ def _string_to_sign(item, table_name, attribute_actions): data_to_sign.extend(_hash_data(hasher=hasher, data=key.encode(TEXT_ENCODING))) + # for some reason pylint can't follow the Enum member attributes if action is CryptoAction.SIGN_ONLY: - data_to_sign.extend(SignatureValues.PLAINTEXT.sha256) + data_to_sign.extend(SignatureValues.PLAINTEXT.sha256) # pylint: disable=no-member else: - data_to_sign.extend(SignatureValues.ENCRYPTED.sha256) + data_to_sign.extend(SignatureValues.ENCRYPTED.sha256) # pylint: disable=no-member data_to_sign.extend(_hash_data(hasher=hasher, data=serialize_attribute(item[key]))) return bytes(data_to_sign) diff --git a/src/dynamodb_encryption_sdk/internal/crypto/encryption.py b/src/dynamodb_encryption_sdk/internal/crypto/encryption.py index 863ab142..3737d520 100644 --- a/src/dynamodb_encryption_sdk/internal/crypto/encryption.py +++ b/src/dynamodb_encryption_sdk/internal/crypto/encryption.py @@ -47,7 +47,8 @@ def encrypt_attribute(attribute_name, attribute, encryption_key, algorithm): encrypted_attribute = encryption_key.encrypt( algorithm=algorithm, name=attribute_name, plaintext=serialized_attribute ) - return {Tag.BINARY.dynamodb_tag: encrypted_attribute} + # for some reason pylint can't follow the Enum member attributes + return {Tag.BINARY.dynamodb_tag: encrypted_attribute} # pylint: disable=no-member def decrypt_attribute(attribute_name, attribute, decryption_key, algorithm): @@ -61,7 +62,8 @@ def decrypt_attribute(attribute_name, attribute, decryption_key, algorithm): :returns: Plaintext DynamoDB attribute :rtype: dict """ - encrypted_attribute = attribute[Tag.BINARY.dynamodb_tag] + # for some reason pylint can't follow the Enum member attributes + encrypted_attribute = attribute[Tag.BINARY.dynamodb_tag] # pylint: disable=no-member decrypted_attribute = decryption_key.decrypt( algorithm=algorithm, name=attribute_name, ciphertext=encrypted_attribute ) diff --git a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/authentication.py b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/authentication.py index a40c4a62..f0fe4e1d 100644 --- a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/authentication.py +++ b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/authentication.py @@ -139,7 +139,7 @@ def load_key(self, key, key_type, key_encoding): raise ValueError("Key type must be symmetric and encoding must be raw.") if len(key) * 8 < MinimumKeySizes.HMAC.value: - _LOGGER.warning("HMAC keys smaller than %d bits are unsafe" % MinimumKeySizes.HMAC.value) + _LOGGER.warning("HMAC keys smaller than %d bits are unsafe", MinimumKeySizes.HMAC.value) return key diff --git a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/primitives.py b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/primitives.py index 29497fc9..978e353b 100644 --- a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/primitives.py +++ b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/primitives.py @@ -121,8 +121,7 @@ class SimplePadding(JavaPadding): java_name = attr.ib(validator=attr.validators.instance_of(six.string_types)) padding = attr.ib(validator=callable_validator) - def __init__(self, java_name, padding): # type: Text # type: Callable # noqa=D107 - # type: (...) -> None + def __init__(self, java_name, padding): # type: (Text, Callable) -> None # noqa=D107 # Workaround pending resolution of attrs/mypy interaction. # https://github.com/python/mypy/issues/2088 # https://github.com/python-attrs/attrs/issues/215 @@ -148,8 +147,7 @@ class BlockSizePadding(JavaPadding): java_name = attr.ib(validator=attr.validators.instance_of(six.string_types)) padding = attr.ib(validator=callable_validator) - def __init__(self, java_name, padding): # type: Text # type: Callable # noqa=D107 - # type: (...) -> None + def __init__(self, java_name, padding): # type: (Text, Callable) -> None # noqa=D107 # Workaround pending resolution of attrs/mypy interaction. # https://github.com/python/mypy/issues/2088 # https://github.com/python-attrs/attrs/issues/215 @@ -227,8 +225,7 @@ class JavaMode(object): java_name = attr.ib(validator=attr.validators.instance_of(six.string_types)) mode = attr.ib(validator=callable_validator) - def __init__(self, java_name, mode): # type: Text # type: Callable # noqa=D107 - # type: (...) -> None + def __init__(self, java_name, mode): # type: (Text, Callable) -> None # noqa=D107 # Workaround pending resolution of attrs/mypy interaction. # https://github.com/python/mypy/issues/2088 # https://github.com/python-attrs/attrs/issues/215 @@ -256,8 +253,7 @@ class JavaEncryptionAlgorithm(object): java_name = attr.ib(validator=attr.validators.instance_of(six.string_types)) cipher = attr.ib() - def __init__(self, java_name, cipher): # type: Text # type: Callable # noqa=D107 - # type: (...) -> None + def __init__(self, java_name, cipher): # type: (Text, Callable) -> None # noqa=D107 # Workaround pending resolution of attrs/mypy interaction. # https://github.com/python/mypy/issues/2088 # https://github.com/python-attrs/attrs/issues/215 @@ -455,7 +451,7 @@ def load_rsa_key(key, key_type, key_encoding): loaded_key = loader(**kwargs) if loaded_key.key_size < MinimumKeySizes.RSA.value: - _LOGGER.warning("RSA keys smaller than %d bits are unsafe" % MinimumKeySizes.RSA.value) + _LOGGER.warning("RSA keys smaller than %d bits are unsafe", MinimumKeySizes.RSA.value) return loaded_key diff --git a/src/dynamodb_encryption_sdk/internal/dynamodb_types.py b/src/dynamodb_encryption_sdk/internal/dynamodb_types.py index 1096a645..964e7779 100644 --- a/src/dynamodb_encryption_sdk/internal/dynamodb_types.py +++ b/src/dynamodb_encryption_sdk/internal/dynamodb_types.py @@ -4,6 +4,7 @@ No guarantee is provided on the modules and APIs within this namespace staying consistent. Directly reference at your own risk. """ +# constant naming for types so pylint: disable=invalid-name try: # Python 3.5.0 and 3.5.1 have incompatible typing modules from typing import Any, AnyStr, ByteString, Dict, List, Text diff --git a/src/dynamodb_encryption_sdk/internal/formatting/deserialize/attribute.py b/src/dynamodb_encryption_sdk/internal/formatting/deserialize/attribute.py index fa2eceb2..41058916 100644 --- a/src/dynamodb_encryption_sdk/internal/formatting/deserialize/attribute.py +++ b/src/dynamodb_encryption_sdk/internal/formatting/deserialize/attribute.py @@ -65,6 +65,8 @@ def _transform_binary_value(value): def _deserialize_binary(stream): # type: (io.BytesIO) -> Dict[Text, bytes] + # pylint: disable=no-member + # for some reason pylint can't follow the Enum member attributes """Deserializes a binary object. :param stream: Stream containing serialized object @@ -85,6 +87,8 @@ def _transform_string_value(value): def _deserialize_string(stream): # type: (io.BytesIO) -> Dict[Text, dynamodb_types.STRING] + # pylint: disable=no-member + # for some reason pylint can't follow the Enum member attributes """Deserializes a string object. :param stream: Stream containing serialized object @@ -107,6 +111,8 @@ def _transform_number_value(value): def _deserialize_number(stream): # type: (io.BytesIO) -> Dict[Text, dynamodb_types.STRING] + # pylint: disable=no-member + # for some reason pylint can't follow the Enum member attributes """Deserializes a number object. :param stream: Stream containing serialized object @@ -120,6 +126,8 @@ def _deserialize_number(stream): def _deserialize_boolean(stream): # type: (io.BytesIO) -> Dict[Text, dynamodb_types.BOOLEAN] + # pylint: disable=no-member + # for some reason pylint can't follow the Enum member attributes """Deserializes a boolean object. :param stream: Stream containing serialized object @@ -131,6 +139,8 @@ def _deserialize_boolean(stream): def _deserialize_null(stream): # we want a consistent API but don't use stream, so pylint: disable=unused-argument # type: (io.BytesIO) -> Dict[Text, dynamodb_types.BOOLEAN] + # pylint: disable=no-member + # for some reason pylint can't follow the Enum member attributes """Deserializes a null object. :param stream: Stream containing serialized object @@ -152,6 +162,8 @@ def _deserialize_set(stream, member_transform): def _deserialize_binary_set(stream): # type: (io.BytesIO) -> Dict[Text, dynamodb_types.SET[dynamodb_types.BINARY]] + # pylint: disable=no-member + # for some reason pylint can't follow the Enum member attributes """Deserializes a binary set object. :param stream: Stream containing serialized object @@ -162,6 +174,8 @@ def _deserialize_binary_set(stream): def _deserialize_string_set(stream): # type: (io.BytesIO) -> Dict[Text, dynamodb_types.SET[dynamodb_types.STRING]] + # pylint: disable=no-member + # for some reason pylint can't follow the Enum member attributes """Deserializes a string set object. :param stream: Stream containing serialized object @@ -172,6 +186,8 @@ def _deserialize_string_set(stream): def _deserialize_number_set(stream): # type: (io.BytesIO) -> Dict[Text, dynamodb_types.SET[dynamodb_types.STRING]] + # pylint: disable=no-member + # for some reason pylint can't follow the Enum member attributes """Deserializes a number set object. :param stream: Stream containing serialized object @@ -182,6 +198,8 @@ def _deserialize_number_set(stream): def _deserialize_list(stream): # type: (io.BytesIO) -> Dict[Text, dynamodb_types.LIST] + # pylint: disable=no-member + # for some reason pylint can't follow the Enum member attributes """Deserializes a list object. :param stream: Stream containing serialized object @@ -199,19 +217,20 @@ def _deserialize_map(stream): :type stream: io.BytesIO :rtype: dict """ + # for some reason pylint can't follow the Enum member attributes member_count = decode_length(stream) members = {} # type: dynamodb_types.MAP for _ in range(member_count): key = _deserialize(stream) - if Tag.STRING.dynamodb_tag not in key: + if Tag.STRING.dynamodb_tag not in key: # pylint: disable=no-member raise DeserializationError( 'Malformed serialized map: found "{}" as map key.'.format(list(key.keys())[0]) ) value = _deserialize(stream) - members[key[Tag.STRING.dynamodb_tag]] = value + members[key[Tag.STRING.dynamodb_tag]] = value # pylint: disable=no-member - return {Tag.MAP.dynamodb_tag: members} + return {Tag.MAP.dynamodb_tag: members} # pylint: disable=no-member def _deserialize_function(tag): # type: (bytes) -> Callable @@ -221,17 +240,18 @@ def _deserialize_function(tag): :type tag: dynamodb_encryption_sdk.internal.identifiers.Tag :rtype: callable """ + # for some reason pylint can't follow the Enum member attributes deserialize_functions = { - Tag.BINARY.tag: _deserialize_binary, - Tag.BINARY_SET.tag: _deserialize_binary_set, - Tag.NUMBER.tag: _deserialize_number, - Tag.NUMBER_SET.tag: _deserialize_number_set, - Tag.STRING.tag: _deserialize_string, - Tag.STRING_SET.tag: _deserialize_string_set, - Tag.BOOLEAN.tag: _deserialize_boolean, - Tag.NULL.tag: _deserialize_null, - Tag.LIST.tag: _deserialize_list, - Tag.MAP.tag: _deserialize_map, + Tag.BINARY.tag: _deserialize_binary, # pylint: disable=no-member + Tag.BINARY_SET.tag: _deserialize_binary_set, # pylint: disable=no-member + Tag.NUMBER.tag: _deserialize_number, # pylint: disable=no-member + Tag.NUMBER_SET.tag: _deserialize_number_set, # pylint: disable=no-member + Tag.STRING.tag: _deserialize_string, # pylint: disable=no-member + Tag.STRING_SET.tag: _deserialize_string_set, # pylint: disable=no-member + Tag.BOOLEAN.tag: _deserialize_boolean, # pylint: disable=no-member + Tag.NULL.tag: _deserialize_null, # pylint: disable=no-member + Tag.LIST.tag: _deserialize_list, # pylint: disable=no-member + Tag.MAP.tag: _deserialize_map, # pylint: disable=no-member } try: return deserialize_functions[tag] diff --git a/src/dynamodb_encryption_sdk/internal/formatting/material_description.py b/src/dynamodb_encryption_sdk/internal/formatting/material_description.py index 0b7ca10e..6607c758 100644 --- a/src/dynamodb_encryption_sdk/internal/formatting/material_description.py +++ b/src/dynamodb_encryption_sdk/internal/formatting/material_description.py @@ -63,7 +63,8 @@ def serialize(material_description): 'Invalid name or value in material description: "{name}"="{value}"'.format(name=name, value=value) ) - return {Tag.BINARY.dynamodb_tag: bytes(material_description_bytes)} + # for some reason pylint can't follow the Enum member attributes + return {Tag.BINARY.dynamodb_tag: bytes(material_description_bytes)} # pylint: disable=no-member def deserialize(serialized_material_description): @@ -77,7 +78,10 @@ def deserialize(serialized_material_description): :raises InvalidMaterialDescriptionVersionError: if unknown version is found """ try: - _raw_material_description = serialized_material_description[Tag.BINARY.dynamodb_tag] + # for some reason pylint can't follow the Enum member attributes + _raw_material_description = serialized_material_description[ + Tag.BINARY.dynamodb_tag # pylint: disable=no-member + ] material_description_bytes = io.BytesIO(_raw_material_description) total_bytes = len(_raw_material_description) diff --git a/src/dynamodb_encryption_sdk/internal/formatting/serialize/attribute.py b/src/dynamodb_encryption_sdk/internal/formatting/serialize/attribute.py index 67651d3d..1ca416a1 100644 --- a/src/dynamodb_encryption_sdk/internal/formatting/serialize/attribute.py +++ b/src/dynamodb_encryption_sdk/internal/formatting/serialize/attribute.py @@ -80,6 +80,8 @@ def _transform_binary_value(value): def _serialize_binary(_attribute): # type: (dynamodb_types.BINARY) -> bytes + # pylint: disable=no-member + # for some reason pylint can't follow the Enum member attributes """ :param _attribute: Attribute to serialize :type _attribute: boto3.dynamodb.types.Binary @@ -105,6 +107,8 @@ def _transform_number_value(value): def _serialize_number(_attribute): # type: (str) -> bytes + # pylint: disable=no-member + # for some reason pylint can't follow the Enum member attributes """ :param _attribute: Attribute to serialize :type _attribute: numbers.Number @@ -125,6 +129,8 @@ def _transform_string_value(value): def _serialize_string(_attribute): # type: (dynamodb_types.STRING) -> bytes + # pylint: disable=no-member + # for some reason pylint can't follow the Enum member attributes """ :param _attribute: Attribute to serialize :type _attribute: six.string_types @@ -135,6 +141,8 @@ def _serialize_string(_attribute): def _serialize_boolean(_attribute): # type: (dynamodb_types.BOOLEAN) -> bytes + # pylint: disable=no-member + # for some reason pylint can't follow the Enum member attributes """ :param bool _attribute: Attribute to serialize :returns: Serialized _attribute @@ -145,6 +153,8 @@ def _serialize_boolean(_attribute): def _serialize_null(_attribute): # type: (dynamodb_types.NULL) -> bytes + # pylint: disable=no-member + # for some reason pylint can't follow the Enum member attributes """ :param _attribute: Attribute to serialize :type _attribute: types.NoneType @@ -204,6 +214,8 @@ def _serialize_string_set(_attribute): def _serialize_list(_attribute): # type: (dynamodb_types.LIST) -> bytes + # pylint: disable=no-member + # for some reason pylint can't follow the Enum member attributes """ :param list _attribute: Attribute to serialize :returns: Serialized _attribute @@ -227,7 +239,8 @@ def _serialize_map(_attribute): """ serialized_attribute = io.BytesIO() serialized_attribute.write(_RESERVED) - serialized_attribute.write(Tag.MAP.tag) + # for some reason pylint can't follow the Enum member attributes + serialized_attribute.write(Tag.MAP.tag) # pylint: disable=no-member serialized_attribute.write(encode_length(_attribute)) sorted_items = _sorted_key_map(item=_attribute, transform=_transform_string_value) @@ -241,17 +254,18 @@ def _serialize_map(_attribute): def _serialize_function(dynamodb_tag): # type: (str) -> Callable[[dynamodb_types.ATTRIBUTE], bytes] """Locates the appropriate serialization function for the specified DynamoDB attribute tag.""" + # for some reason pylint can't follow the Enum member attributes serialize_functions = { - Tag.BINARY.dynamodb_tag: _serialize_binary, - Tag.BINARY_SET.dynamodb_tag: _serialize_binary_set, - Tag.NUMBER.dynamodb_tag: _serialize_number, - Tag.NUMBER_SET.dynamodb_tag: _serialize_number_set, - Tag.STRING.dynamodb_tag: _serialize_string, - Tag.STRING_SET.dynamodb_tag: _serialize_string_set, - Tag.BOOLEAN.dynamodb_tag: _serialize_boolean, - Tag.NULL.dynamodb_tag: _serialize_null, - Tag.LIST.dynamodb_tag: _serialize_list, - Tag.MAP.dynamodb_tag: _serialize_map, + Tag.BINARY.dynamodb_tag: _serialize_binary, # pylint: disable=no-member + Tag.BINARY_SET.dynamodb_tag: _serialize_binary_set, # pylint: disable=no-member + Tag.NUMBER.dynamodb_tag: _serialize_number, # pylint: disable=no-member + Tag.NUMBER_SET.dynamodb_tag: _serialize_number_set, # pylint: disable=no-member + Tag.STRING.dynamodb_tag: _serialize_string, # pylint: disable=no-member + Tag.STRING_SET.dynamodb_tag: _serialize_string_set, # pylint: disable=no-member + Tag.BOOLEAN.dynamodb_tag: _serialize_boolean, # pylint: disable=no-member + Tag.NULL.dynamodb_tag: _serialize_null, # pylint: disable=no-member + Tag.LIST.dynamodb_tag: _serialize_list, # pylint: disable=no-member + Tag.MAP.dynamodb_tag: _serialize_map, # pylint: disable=no-member } try: return serialize_functions[dynamodb_tag] diff --git a/src/dynamodb_encryption_sdk/internal/identifiers.py b/src/dynamodb_encryption_sdk/internal/identifiers.py index b0474ec8..94d7bd41 100644 --- a/src/dynamodb_encryption_sdk/internal/identifiers.py +++ b/src/dynamodb_encryption_sdk/internal/identifiers.py @@ -73,7 +73,7 @@ def __init__(self, tag, dynamodb_tag, element_tag=None): """Sets up new Tag object. :param bytes tag: DynamoDB Encryption SDK tag - :param bytes dynamodb_tag: DynamoDB tag + :param str dynamodb_tag: DynamoDB tag :param bytes element_tag: The type of tag contained within attributes of this type """ self.tag = tag diff --git a/src/dynamodb_encryption_sdk/material_providers/aws_kms.py b/src/dynamodb_encryption_sdk/material_providers/aws_kms.py index 7bcad4c4..ea7a55f2 100644 --- a/src/dynamodb_encryption_sdk/material_providers/aws_kms.py +++ b/src/dynamodb_encryption_sdk/material_providers/aws_kms.py @@ -138,6 +138,7 @@ def from_material_description(cls, material_description, description_key, defaul @attr.s(init=False) class AwsKmsCryptographicMaterialsProvider(CryptographicMaterialsProvider): + # pylint: disable=too-many-instance-attributes """Cryptographic materials provider for use with the AWS Key Management Service (KMS). .. note:: diff --git a/src/dynamodb_encryption_sdk/material_providers/most_recent.py b/src/dynamodb_encryption_sdk/material_providers/most_recent.py index 80892882..c5ce27a5 100644 --- a/src/dynamodb_encryption_sdk/material_providers/most_recent.py +++ b/src/dynamodb_encryption_sdk/material_providers/most_recent.py @@ -194,7 +194,7 @@ def _ttl_action(self): if time_since_updated < self._version_ttl: return TtlActions.LIVE - elif time_since_updated < self._version_ttl + _GRACE_PERIOD: + if time_since_updated < self._version_ttl + _GRACE_PERIOD: return TtlActions.GRACE_PERIOD _LOGGER.debug("TTL Expired because known version has expired") @@ -255,12 +255,14 @@ def _get_most_recent_version(self, allow_local): provider = self._cache.get(max_version) except KeyError: provider = self._get_provider(max_version) - received_version = self._provider_store.version_from_material_description(provider._material_description) + received_version = self._provider_store.version_from_material_description( + provider._material_description # pylint: disable=protected-access + ) # TODO: ^ should we promote material description from hidden? _LOGGER.debug("Caching materials provider version %d", received_version) - self._version = received_version - self._last_updated = time.time() + self._version = received_version # pylint: disable=attribute-defined-outside-init + self._last_updated = time.time() # pylint: disable=attribute-defined-outside-init self._cache.put(received_version, provider) finally: self._lock.release() From 2f1bdaba884489d435fbceec762d55c5c78d1b6e Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Mon, 24 Aug 2020 13:37:33 -0700 Subject: [PATCH 11/23] chore: unlock mypy version --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 53c4ddc4..2ad19ad7 100644 --- a/tox.ini +++ b/tox.ini @@ -169,7 +169,7 @@ commands = basepython = python3 deps = coverage - mypy<=0.560 + mypy mypy_extensions typing>=3.6.2 From 0fa073ef94cbfe5ce95bb9f589496e0dc4bb03ab Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Mon, 24 Aug 2020 13:42:28 -0700 Subject: [PATCH 12/23] chore: fix type annotation syntax errors --- .../internal/crypto/jce_bridge/authentication.py | 8 ++++---- .../internal/crypto/jce_bridge/encryption.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/authentication.py b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/authentication.py index f0fe4e1d..24fdbd44 100644 --- a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/authentication.py +++ b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/authentication.py @@ -105,9 +105,9 @@ class JavaMac(JavaAuthenticator): hash_type = attr.ib(validator=callable_validator) def __init__( - self, java_name, algorithm_type, hash_type # type: Text # type: Callable # type: Callable + self, java_name, algorithm_type, hash_type ): # noqa=D107 - # type: (...) -> None + # type: (Text, Callable, Callable) -> None # Workaround pending resolution of attrs/mypy interaction. # https://github.com/python/mypy/issues/2088 # https://github.com/python-attrs/attrs/issues/215 @@ -209,9 +209,9 @@ class JavaSignature(JavaAuthenticator): padding_type = attr.ib(validator=callable_validator) def __init__( - self, java_name, algorithm_type, hash_type, padding_type # type: Text # type: Callable # type: Callable + self, java_name, algorithm_type, hash_type, padding_type ): # noqa=D107 - # type: (...) -> None + # type: (Text, Any, Callable, Callable) -> None # Workaround pending resolution of attrs/mypy interaction. # https://github.com/python/mypy/issues/2088 # https://github.com/python-attrs/attrs/issues/215 diff --git a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/encryption.py b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/encryption.py index b09f60a1..318dc0d4 100644 --- a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/encryption.py +++ b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/encryption.py @@ -48,9 +48,9 @@ class JavaCipher(object): padding = attr.ib(validator=attr.validators.instance_of(JavaPadding)) def __init__( - self, cipher, mode, padding # type: JavaEncryptionAlgorithm # type: JavaMode # type: JavaPadding + self, cipher, mode, padding ): # noqa=D107 - # type: (...) -> None + # type: (JavaEncryptionAlgorithm, JavaMode, JavaPadding) -> None # Workaround pending resolution of attrs/mypy interaction. # https://github.com/python/mypy/issues/2088 # https://github.com/python-attrs/attrs/issues/215 From c9015f32093ba7c544bea0ff37ffa9f574717a5c Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Mon, 24 Aug 2020 14:01:21 -0700 Subject: [PATCH 13/23] chore: move TODOs into GitHub issues --- .../encrypted/client.py | 3 ++- .../crypto/jce_bridge/authentication.py | 15 ++++++++------- .../internal/crypto/jce_bridge/primitives.py | 4 ++-- .../internal/dynamodb_types.py | 9 ++++++--- .../formatting/material_description.py | 1 - src/dynamodb_encryption_sdk/internal/utils.py | 18 ++++++++++++------ .../material_providers/most_recent.py | 1 - .../material_providers/store/meta.py | 11 ++++++----- src/dynamodb_encryption_sdk/transform.py | 6 ++++-- 9 files changed, 40 insertions(+), 28 deletions(-) diff --git a/src/dynamodb_encryption_sdk/encrypted/client.py b/src/dynamodb_encryption_sdk/encrypted/client.py index ebee5d90..bd8f2c58 100644 --- a/src/dynamodb_encryption_sdk/encrypted/client.py +++ b/src/dynamodb_encryption_sdk/encrypted/client.py @@ -96,7 +96,8 @@ def __getattr__(self, name): def paginate(self, **kwargs): # type: (**Any) -> Iterator[Dict] - # TODO: narrow this down + # narrow this down + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 """Create an iterator that will paginate through responses from the underlying paginator, transparently decrypting any returned items. """ diff --git a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/authentication.py b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/authentication.py index 24fdbd44..dfde7f04 100644 --- a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/authentication.py +++ b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/authentication.py @@ -49,7 +49,8 @@ class JavaAuthenticator(object): @abc.abstractmethod def load_key(self, key, key_type, key_encoding): # (bytes, EncryptionKeyType, KeyEncodingType) -> Any - # TODO: narrow down the output type + # narrow down the output type + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 """Load a key from bytes. :param bytes key: Raw key bytes to load @@ -237,25 +238,25 @@ def validate_algorithm(self, algorithm): def load_key(self, key, key_type, key_encoding): # (bytes, EncryptionKeyType, KeyEncodingType) -> Any - # TODO: narrow down the output type + # narrow down the output type + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 """Load a key object from the provided raw key bytes. :param bytes key: Raw key bytes to load :param EncryptionKeyType key_type: Type of key to load :param KeyEncodingType key_encoding: Encoding used to serialize ``key`` :returns: Loaded key - :rtype: TODO: :raises ValueError: if ``key_type`` and ``key_encoding`` are not a valid pairing """ return load_rsa_key(key, key_type, key_encoding) def sign(self, key, data): # type: (Any, bytes) -> bytes - # TODO: narrow down the key type + # narrow down the key type + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 """Sign ``data`` using loaded ``key``. :param key: Loaded key - :type key: TODO: :param bytes data: Data to sign :returns: Calculated signature :rtype: bytes @@ -272,11 +273,11 @@ def sign(self, key, data): def verify(self, key, signature, data): # type: (Any, bytes, bytes) -> None - # TODO: narrow down the key type + # narrow down the key type + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 """Verify ``signature`` over ``data`` using ``key``. :param key: Loaded key - :type key: TODO: :param bytes signature: Signature to verify :param bytes data: Data over which to verify signature :raises SignatureVerificationError: if unable to verify ``signature`` diff --git a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/primitives.py b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/primitives.py index 978e353b..564bed80 100644 --- a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/primitives.py +++ b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/primitives.py @@ -429,14 +429,14 @@ def decrypt(self, key, data, mode, padding): def load_rsa_key(key, key_type, key_encoding): # (bytes, EncryptionKeyType, KeyEncodingType) -> Any - # TODO: narrow down the output type + # narrow down the output type + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 """Load an RSA key object from the provided raw key bytes. :param bytes key: Raw key bytes to load :param EncryptionKeyType key_type: Type of key to load :param KeyEncodingType key_encoding: Encoding used to serialize ``key`` :returns: Loaded key - :rtype: TODO: :raises ValueError: if ``key_type`` and ``key_encoding`` are not a valid pairing """ try: diff --git a/src/dynamodb_encryption_sdk/internal/dynamodb_types.py b/src/dynamodb_encryption_sdk/internal/dynamodb_types.py index 964e7779..0509a59b 100644 --- a/src/dynamodb_encryption_sdk/internal/dynamodb_types.py +++ b/src/dynamodb_encryption_sdk/internal/dynamodb_types.py @@ -8,13 +8,16 @@ try: # Python 3.5.0 and 3.5.1 have incompatible typing modules from typing import Any, AnyStr, ByteString, Dict, List, Text - ATTRIBUTE = Dict[Text, Any] # TODO: narrow this down + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 + ATTRIBUTE = Dict[Text, Any] # narrow this down ITEM = Dict[Text, ATTRIBUTE] RAW_ATTRIBUTE = ITEM NULL = bool # DynamoDB TypeSerializer converts none to {'NULL': True} BOOLEAN = bool - NUMBER = int # TODO: This misses long on Python 2...figure out something for this - STRING = AnyStr # TODO: can be unicode but should not be bytes + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 + NUMBER = int # This misses long on Python 2...figure out something for this + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 + STRING = AnyStr # can be unicode but should not be bytes BINARY = ByteString BINARY_ATTRIBUTE = Dict[Text, BINARY] SET = List # DynamoDB TypeSerializer converts sets into lists diff --git a/src/dynamodb_encryption_sdk/internal/formatting/material_description.py b/src/dynamodb_encryption_sdk/internal/formatting/material_description.py index 6607c758..1f6af4a2 100644 --- a/src/dynamodb_encryption_sdk/internal/formatting/material_description.py +++ b/src/dynamodb_encryption_sdk/internal/formatting/material_description.py @@ -53,7 +53,6 @@ def serialize(material_description): """ material_description_bytes = bytearray(_MATERIAL_DESCRIPTION_VERSION) - # TODO: verify Java sorting order for name, value in sorted(material_description.items(), key=lambda x: x[0]): try: material_description_bytes.extend(encode_value(to_bytes(name))) diff --git a/src/dynamodb_encryption_sdk/internal/utils.py b/src/dynamodb_encryption_sdk/internal/utils.py index ae8ebe16..988576b2 100644 --- a/src/dynamodb_encryption_sdk/internal/utils.py +++ b/src/dynamodb_encryption_sdk/internal/utils.py @@ -174,7 +174,8 @@ def _item_transformer(crypto_transformer): def decrypt_list_of_items(crypto_config, decrypt_method, items): # type: (CryptoConfig, Callable, Iterable[Any]) -> Iterable[Any] - # TODO: narrow this down + # narrow this down + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 """Iterate through a list of encrypted items, decrypting each item and yielding the plaintext item. :param CryptoConfig crypto_config: :class:`CryptoConfig` to use @@ -190,7 +191,8 @@ def decrypt_list_of_items(crypto_config, decrypt_method, items): def decrypt_multi_get(decrypt_method, crypto_config_method, read_method, **kwargs): # type: (Callable, Callable, Callable, **Any) -> Dict - # TODO: narrow this down + # narrow this down + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 """Transparently decrypt multiple items after getting them from the table with a scan or query method. :param callable decrypt_method: Method to use to decrypt items @@ -211,7 +213,8 @@ def decrypt_multi_get(decrypt_method, crypto_config_method, read_method, **kwarg def decrypt_get_item(decrypt_method, crypto_config_method, read_method, **kwargs): # type: (Callable, Callable, Callable, **Any) -> Dict - # TODO: narrow this down + # narrow this down + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 """Transparently decrypt an item after getting it from the table. :param callable decrypt_method: Method to use to decrypt item @@ -234,7 +237,8 @@ def decrypt_get_item(decrypt_method, crypto_config_method, read_method, **kwargs def decrypt_batch_get_item(decrypt_method, crypto_config_method, read_method, **kwargs): # type: (Callable, Callable, Callable, **Any) -> Dict - # TODO: narrow this down + # narrow this down + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 """Transparently decrypt multiple items after getting them in a batch request. :param callable decrypt_method: Method to use to decrypt items @@ -265,7 +269,8 @@ def decrypt_batch_get_item(decrypt_method, crypto_config_method, read_method, ** def encrypt_put_item(encrypt_method, crypto_config_method, write_method, **kwargs): # type: (Callable, Callable, Callable, **Any) -> Dict - # TODO: narrow this down + # narrow this down + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 """Transparently encrypt an item before putting it to the table. :param callable encrypt_method: Method to use to encrypt items @@ -285,7 +290,8 @@ def encrypt_put_item(encrypt_method, crypto_config_method, write_method, **kwarg def encrypt_batch_write_item(encrypt_method, crypto_config_method, write_method, **kwargs): # type: (Callable, Callable, Callable, **Any) -> Dict - # TODO: narrow this down + # narrow this down + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 """Transparently encrypt multiple items before putting them in a batch request. :param callable encrypt_method: Method to use to encrypt items diff --git a/src/dynamodb_encryption_sdk/material_providers/most_recent.py b/src/dynamodb_encryption_sdk/material_providers/most_recent.py index c5ce27a5..ce074301 100644 --- a/src/dynamodb_encryption_sdk/material_providers/most_recent.py +++ b/src/dynamodb_encryption_sdk/material_providers/most_recent.py @@ -258,7 +258,6 @@ def _get_most_recent_version(self, allow_local): received_version = self._provider_store.version_from_material_description( provider._material_description # pylint: disable=protected-access ) - # TODO: ^ should we promote material description from hidden? _LOGGER.debug("Caching materials provider version %d", received_version) self._version = received_version # pylint: disable=attribute-defined-outside-init diff --git a/src/dynamodb_encryption_sdk/material_providers/store/meta.py b/src/dynamodb_encryption_sdk/material_providers/store/meta.py index c16cd5e7..567c57be 100644 --- a/src/dynamodb_encryption_sdk/material_providers/store/meta.py +++ b/src/dynamodb_encryption_sdk/material_providers/store/meta.py @@ -121,7 +121,7 @@ def create_table(cls, client, table_name, read_units, write_units): ProvisionedThroughput={"ReadCapacityUnits": read_units, "WriteCapacityUnits": write_units}, ) except botocore.exceptions.ClientError: - raise Exception("TODO: Could not create table") + raise Exception("Could not create table") def _load_materials(self, material_name, version): # type: (Text, int) -> Tuple[JceNameLocalDelegatedKey, JceNameLocalDelegatedKey] @@ -152,9 +152,10 @@ def _load_materials(self, material_name, version): key_encoding=KeyEncodingType.RAW, ) except KeyError: - raise Exception("TODO: Invalid record") + raise Exception("Invalid record") - # TODO: handle if the material type version is not in the item + # need to handle if the material type version is not in the item + # https://github.com/aws/aws-dynamodb-encryption-python/issues/140 if item[MetaStoreAttributeNames.MATERIAL_TYPE_VERSION.value] != MetaStoreValues.MATERIAL_TYPE_VERSION.value: raise InvalidVersionError( 'Unsupported material type: "{}"'.format(item[MetaStoreAttributeNames.MATERIAL_TYPE_VERSION.value]) @@ -301,12 +302,12 @@ def version_from_material_description(self, material_description): try: info = material_description[_MATERIAL_DESCRIPTION_META_FIELD] except KeyError: - raise Exception("TODO: No info found") + raise Exception("No info found") try: return int(info.split("#", 1)[1]) except (IndexError, ValueError): - raise Exception("TODO: Malformed info") + raise Exception("Malformed info") def max_version(self, material_name): # (Text) -> int diff --git a/src/dynamodb_encryption_sdk/transform.py b/src/dynamodb_encryption_sdk/transform.py index 76b22c6c..347024d3 100644 --- a/src/dynamodb_encryption_sdk/transform.py +++ b/src/dynamodb_encryption_sdk/transform.py @@ -24,7 +24,8 @@ def dict_to_ddb(item): # type: (Dict[str, Any]) -> Dict[str, Any] - # TODO: narrow these types down + # narrow these types down + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 """Converts a native Python dictionary to a raw DynamoDB item. :param dict item: Native item @@ -37,7 +38,8 @@ def dict_to_ddb(item): def ddb_to_dict(item): # type: (Dict[str, Any]) -> Dict[str, Any] - # TODO: narrow these types down + # narrow these types down + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 """Converts a raw DynamoDB item to a native Python dictionary. :param dict item: DynamoDB item From 25e250207b31ae77a28a7fc47a386c4a1b880337 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Mon, 24 Aug 2020 15:10:47 -0700 Subject: [PATCH 14/23] chore: move test TODOs to GitHub issues --- test/functional/hypothesis_strategies.py | 7 +++---- test/functional/test_structures.py | 2 -- test/unit/material_providers/test_aws_kms.py | 3 --- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/test/functional/hypothesis_strategies.py b/test/functional/hypothesis_strategies.py index d0fb7c74..6a39d4cf 100644 --- a/test/functional/hypothesis_strategies.py +++ b/test/functional/hypothesis_strategies.py @@ -31,9 +31,7 @@ # _MIN_NUMBER = Decimal('1E-128') # The DDB min is 1E-130, but DYNAMODB_CONTEXT Emin is -128 # _MAX_NUMBER = Decimal('9.9999999999999999999999999999999999999E+125') -# TODO: I would like to test the full range of possible number values, but boto3 does not -# correctly handle conversion of large edge case values at this time. We will work to fix -# that, but in the meantime, we will just use the happy path numbers. +# boto3 does not correctly handle conversion of large edge case values at this time _MIN_NUMBER = Decimal("1E-38") _MAX_NUMBER = Decimal("9.{}E37".format("9" * 37)) @@ -67,7 +65,8 @@ def _negative(val): ddb_set_types = ddb_string_set | ddb_number_set | ddb_binary_set ddb_attribute_names = text(min_size=1, max_size=255) -# TODO: List and Map types have a max depth of 32 +# List and Map types have a max depth of 32 +# https://github.com/aws/aws-dynamodb-encryption-python/issues/141 ddb_map_type = deferred( lambda: dictionaries( keys=ddb_attribute_names, values=(ddb_scalar_types | ddb_set_types | ddb_list_type | ddb_map_type), min_size=1 diff --git a/test/functional/test_structures.py b/test/functional/test_structures.py index 76c1398f..23a02825 100644 --- a/test/functional/test_structures.py +++ b/test/functional/test_structures.py @@ -26,8 +26,6 @@ pytestmark = [pytest.mark.functional, pytest.mark.local] -# TODO: There is a way to parameterize fixtures, but the existing docs on that are very unclear. -# This will get us what we need for now, but we should come back to this to clean this up later. def test_tableinfo_refresh_indexes_no_secondary_indexes(example_table): client = boto3.client("dynamodb", region_name="us-west-2") table = TableInfo(name=TEST_TABLE_NAME) diff --git a/test/unit/material_providers/test_aws_kms.py b/test/unit/material_providers/test_aws_kms.py index efedfd90..edcd301d 100644 --- a/test/unit/material_providers/test_aws_kms.py +++ b/test/unit/material_providers/test_aws_kms.py @@ -316,7 +316,6 @@ def test_select_id(default_kms_cmp): assert test is default_kms_cmp._key_id -# TODO: vectorize @pytest.mark.parametrize( "attribute, expected_value", ( @@ -338,7 +337,6 @@ def test_attribute_to_value_wrong_type(default_kms_cmp): excinfo.match(r"Attribute of type *") -# TODO: vectorize @pytest.mark.parametrize( "encryption_context, additional_expected_keypairs", ( @@ -413,7 +411,6 @@ def test_decrypt_initial_material_fail(default_kms_cmp, patch_kms_client): excinfo.match("Failed to unwrap AWS KMS protected materials") -# TODO: vectorize @pytest.mark.parametrize( "description, method_name, key_name", (("AES/256", "_encryption_key", "encryption_key"), ("HmacSHA256/256", "_mac_key", "mac_key")), From f122f598f45850ccc278097fafab593da5670d32 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Mon, 24 Aug 2020 18:12:27 -0700 Subject: [PATCH 15/23] chore: autoformat --- .../internal/crypto/jce_bridge/authentication.py | 8 ++------ .../internal/crypto/jce_bridge/encryption.py | 4 +--- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/authentication.py b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/authentication.py index dfde7f04..b2244ac7 100644 --- a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/authentication.py +++ b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/authentication.py @@ -105,9 +105,7 @@ class JavaMac(JavaAuthenticator): algorithm_type = attr.ib(validator=callable_validator) hash_type = attr.ib(validator=callable_validator) - def __init__( - self, java_name, algorithm_type, hash_type - ): # noqa=D107 + def __init__(self, java_name, algorithm_type, hash_type): # noqa=D107 # type: (Text, Callable, Callable) -> None # Workaround pending resolution of attrs/mypy interaction. # https://github.com/python/mypy/issues/2088 @@ -209,9 +207,7 @@ class JavaSignature(JavaAuthenticator): hash_type = attr.ib(validator=callable_validator) padding_type = attr.ib(validator=callable_validator) - def __init__( - self, java_name, algorithm_type, hash_type, padding_type - ): # noqa=D107 + def __init__(self, java_name, algorithm_type, hash_type, padding_type): # noqa=D107 # type: (Text, Any, Callable, Callable) -> None # Workaround pending resolution of attrs/mypy interaction. # https://github.com/python/mypy/issues/2088 diff --git a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/encryption.py b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/encryption.py index 318dc0d4..6a123597 100644 --- a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/encryption.py +++ b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/encryption.py @@ -47,9 +47,7 @@ class JavaCipher(object): mode = attr.ib(validator=attr.validators.instance_of(JavaMode)) padding = attr.ib(validator=attr.validators.instance_of(JavaPadding)) - def __init__( - self, cipher, mode, padding - ): # noqa=D107 + def __init__(self, cipher, mode, padding): # noqa=D107 # type: (JavaEncryptionAlgorithm, JavaMode, JavaPadding) -> None # Workaround pending resolution of attrs/mypy interaction. # https://github.com/python/mypy/issues/2088 From aa24b33948bd70375a8ae597041aa292811fb422 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Mon, 24 Aug 2020 18:13:55 -0700 Subject: [PATCH 16/23] chore: rework moto use - fixes issues with multiple service mocks in Python 2 - module scope avoids resetting the mocked service for tests that use multiple mocked tables --- test/functional/encrypted/test_client.py | 12 +++--- test/functional/encrypted/test_resource.py | 10 +++-- test/functional/encrypted/test_table.py | 16 +++++--- test/functional/functional_test_utils.py | 40 +++++++++---------- .../material_providers/test_most_recent.py | 1 + test/functional/test_structures.py | 9 +++-- 6 files changed, 50 insertions(+), 38 deletions(-) diff --git a/test/functional/encrypted/test_client.py b/test/functional/encrypted/test_client.py index 83f0f250..40c055df 100644 --- a/test/functional/encrypted/test_client.py +++ b/test/functional/encrypted/test_client.py @@ -14,8 +14,10 @@ import hypothesis import pytest -from ..functional_test_utils import example_table # noqa pylint: disable=unused-import +from ..functional_test_utils import example_table # noqa=F401 pylint: disable=unused-import +from ..functional_test_utils import mock_ddb_service # noqa=F401 pylint: disable=unused-import from ..functional_test_utils import ( + TEST_REGION_NAME, TEST_TABLE_NAME, build_static_jce_cmp, client_batch_items_unprocessed_check, @@ -39,25 +41,25 @@ def pytest_generate_tests(metafunc): def _client_cycle_single_item_check(materials_provider, initial_actions, initial_item): return client_cycle_single_item_check( - materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, "us-west-2" + materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, TEST_REGION_NAME ) def _client_cycle_batch_items_check(materials_provider, initial_actions, initial_item): return client_cycle_batch_items_check( - materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, "us-west-2" + materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, TEST_REGION_NAME ) def _client_cycle_batch_items_check_scan_paginator(materials_provider, initial_actions, initial_item): return client_cycle_batch_items_check_scan_paginator( - materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, "us-west-2" + materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, TEST_REGION_NAME ) def _client_batch_items_unprocessed_check(materials_provider, initial_actions, initial_item): client_batch_items_unprocessed_check( - materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, "us-west-2" + materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, TEST_REGION_NAME ) diff --git a/test/functional/encrypted/test_resource.py b/test/functional/encrypted/test_resource.py index 3c6a01e2..95ed9d28 100644 --- a/test/functional/encrypted/test_resource.py +++ b/test/functional/encrypted/test_resource.py @@ -13,8 +13,10 @@ """Functional tests for ``dynamodb_encryption_sdk.encrypted.resource``.""" import pytest -from ..functional_test_utils import example_table # noqa pylint: disable=unused-import +from ..functional_test_utils import example_table # noqa=F401 pylint: disable=unused-import +from ..functional_test_utils import mock_ddb_service # noqa=F401 pylint: disable=unused-import from ..functional_test_utils import ( + TEST_REGION_NAME, TEST_TABLE_NAME, build_static_jce_cmp, resource_batch_items_unprocessed_check, @@ -34,12 +36,14 @@ def pytest_generate_tests(metafunc): def _resource_cycle_batch_items_check(materials_provider, initial_actions, initial_item): - resource_cycle_batch_items_check(materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, "us-west-2") + resource_cycle_batch_items_check( + materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, TEST_REGION_NAME + ) def _resource_batch_items_unprocessed_check(materials_provider, initial_actions, initial_item): resource_batch_items_unprocessed_check( - materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, "us-west-2" + materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, TEST_REGION_NAME ) diff --git a/test/functional/encrypted/test_table.py b/test/functional/encrypted/test_table.py index fe9339ad..736edb3b 100644 --- a/test/functional/encrypted/test_table.py +++ b/test/functional/encrypted/test_table.py @@ -14,8 +14,10 @@ import hypothesis import pytest -from ..functional_test_utils import example_table # noqa pylint: disable=unused-import +from ..functional_test_utils import example_table # noqa=F401 pylint: disable=unused-import +from ..functional_test_utils import mock_ddb_service # noqa=F401 pylint: disable=unused-import from ..functional_test_utils import ( + TEST_REGION_NAME, TEST_TABLE_NAME, build_static_jce_cmp, set_parametrized_actions, @@ -37,7 +39,7 @@ def pytest_generate_tests(metafunc): def _table_cycle_check(materials_provider, initial_actions, initial_item): - return table_cycle_check(materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, "us-west-2") + return table_cycle_check(materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, TEST_REGION_NAME) def test_ephemeral_item_cycle(example_table, some_cmps, parametrized_actions, parametrized_item): @@ -47,14 +49,16 @@ def test_ephemeral_item_cycle(example_table, some_cmps, parametrized_actions, pa def test_ephemeral_item_cycle_batch_writer(example_table, some_cmps, parametrized_actions, parametrized_item): """Test a small number of curated CMPs against a small number of curated items.""" - table_cycle_batch_writer_check(some_cmps, parametrized_actions, parametrized_item, TEST_TABLE_NAME, "us-west-2") + table_cycle_batch_writer_check( + some_cmps, parametrized_actions, parametrized_item, TEST_TABLE_NAME, TEST_REGION_NAME + ) def test_batch_writer_unprocessed(example_table, parametrized_actions, parametrized_item): """Test Unprocessed Items handling with a single ephemeral static CMP against a small number of curated items.""" cmp = build_static_jce_cmp("AES", 256, "HmacSHA256", 256) table_batch_writer_unprocessed_items_check( - cmp, parametrized_actions, parametrized_item, TEST_TABLE_NAME, "us-west-2" + cmp, parametrized_actions, parametrized_item, TEST_TABLE_NAME, TEST_REGION_NAME ) @@ -67,7 +71,9 @@ def test_ephemeral_item_cycle_slow(example_table, all_the_cmps, parametrized_act @pytest.mark.slow def test_ephemeral_item_cycle_batch_writer_slow(example_table, all_the_cmps, parametrized_actions, parametrized_item): """Test a small number of curated CMPs against a small number of curated items.""" - table_cycle_batch_writer_check(all_the_cmps, parametrized_actions, parametrized_item, TEST_TABLE_NAME, "us-west-2") + table_cycle_batch_writer_check( + all_the_cmps, parametrized_actions, parametrized_item, TEST_TABLE_NAME, TEST_REGION_NAME + ) @pytest.mark.travis_isolation diff --git a/test/functional/functional_test_utils.py b/test/functional/functional_test_utils.py index 56256296..8902c626 100644 --- a/test/functional/functional_test_utils.py +++ b/test/functional/functional_test_utils.py @@ -78,11 +78,16 @@ TEST_BATCH_KEYS = [{name: value["value"] for name, value in key.items()} for key in TEST_BATCH_INDEXES] +@pytest.fixture(scope="module") +def mock_ddb_service(): + """Centralize service mock to avoid resetting service for tests that use multiple tables.""" + with mock_dynamodb2(): + yield boto3.client("dynamodb", region_name=TEST_REGION_NAME) + + @pytest.fixture -def example_table(): - mock_dynamodb2().start(reset=False) - ddb = boto3.client("dynamodb", region_name=TEST_REGION_NAME) - ddb.create_table( +def example_table(mock_ddb_service): + mock_ddb_service.create_table( TableName=TEST_TABLE_NAME, KeySchema=[ {"AttributeName": "partition_attribute", "KeyType": "HASH"}, @@ -93,16 +98,13 @@ def example_table(): ], ProvisionedThroughput={"ReadCapacityUnits": 100, "WriteCapacityUnits": 100}, ) - yield - ddb.delete_table(TableName=TEST_TABLE_NAME) - mock_dynamodb2().stop() + yield mock_ddb_service + mock_ddb_service.delete_table(TableName=TEST_TABLE_NAME) @pytest.fixture -def table_with_local_secondary_indexes(): - mock_dynamodb2().start(reset=False) - ddb = boto3.client("dynamodb", region_name=TEST_REGION_NAME) - ddb.create_table( +def table_with_local_secondary_indexes(mock_ddb_service): + mock_ddb_service.create_table( TableName=TEST_TABLE_NAME, KeySchema=[ {"AttributeName": "partition_attribute", "KeyType": "HASH"}, @@ -126,16 +128,13 @@ def table_with_local_secondary_indexes(): ], ProvisionedThroughput={"ReadCapacityUnits": 100, "WriteCapacityUnits": 100}, ) - yield - ddb.delete_table(TableName=TEST_TABLE_NAME) - mock_dynamodb2().stop() + yield mock_ddb_service + mock_ddb_service.delete_table(TableName=TEST_TABLE_NAME) @pytest.fixture -def table_with_global_secondary_indexes(): - mock_dynamodb2().start(reset=False) - ddb = boto3.client("dynamodb", region_name=TEST_REGION_NAME) - ddb.create_table( +def table_with_global_secondary_indexes(mock_ddb_service): + mock_ddb_service.create_table( TableName=TEST_TABLE_NAME, KeySchema=[ {"AttributeName": "partition_attribute", "KeyType": "HASH"}, @@ -161,9 +160,8 @@ def table_with_global_secondary_indexes(): ], ProvisionedThroughput={"ReadCapacityUnits": 100, "WriteCapacityUnits": 100}, ) - yield - ddb.delete_table(TableName=TEST_TABLE_NAME) - mock_dynamodb2().stop() + yield mock_ddb_service + mock_ddb_service.delete_table(TableName=TEST_TABLE_NAME) class PassThroughCryptographicMaterialsProviderThatRequiresAttributes(CryptographicMaterialsProvider): diff --git a/test/functional/material_providers/test_most_recent.py b/test/functional/material_providers/test_most_recent.py index 9af56166..967b1da5 100644 --- a/test/functional/material_providers/test_most_recent.py +++ b/test/functional/material_providers/test_most_recent.py @@ -22,6 +22,7 @@ from dynamodb_encryption_sdk.material_providers.store import ProviderStore from ..functional_test_utils import example_table # noqa=F401 pylint: disable=unused-import +from ..functional_test_utils import mock_ddb_service # noqa=F401 pylint: disable=unused-import from ..functional_test_utils import mock_metastore # noqa=F401 pylint: disable=unused-import from ..functional_test_utils import TEST_TABLE_NAME, check_metastore_cache_use_encrypt diff --git a/test/functional/test_structures.py b/test/functional/test_structures.py index 23a02825..aec90179 100644 --- a/test/functional/test_structures.py +++ b/test/functional/test_structures.py @@ -19,29 +19,30 @@ from dynamodb_encryption_sdk.structures import AttributeActions, TableIndex, TableInfo from .functional_test_utils import example_table # noqa=F401 pylint: disable=unused-import +from .functional_test_utils import mock_ddb_service # noqa=F401 pylint: disable=unused-import from .functional_test_utils import table_with_global_secondary_indexes # noqa=F401 pylint: disable=unused-import from .functional_test_utils import table_with_local_secondary_indexes # noqa=F401 pylint: disable=unused-import -from .functional_test_utils import TEST_TABLE_NAME +from .functional_test_utils import TEST_REGION_NAME, TEST_TABLE_NAME pytestmark = [pytest.mark.functional, pytest.mark.local] def test_tableinfo_refresh_indexes_no_secondary_indexes(example_table): - client = boto3.client("dynamodb", region_name="us-west-2") + client = boto3.client("dynamodb", region_name=TEST_REGION_NAME) table = TableInfo(name=TEST_TABLE_NAME) table.refresh_indexed_attributes(client) def test_tableinfo_refresh_indexes_with_gsis(table_with_global_secondary_indexes): - client = boto3.client("dynamodb", region_name="us-west-2") + client = boto3.client("dynamodb", region_name=TEST_REGION_NAME) table = TableInfo(name=TEST_TABLE_NAME) table.refresh_indexed_attributes(client) def test_tableinfo_refresh_indexes_with_lsis(table_with_local_secondary_indexes): - client = boto3.client("dynamodb", region_name="us-west-2") + client = boto3.client("dynamodb", region_name=TEST_REGION_NAME) table = TableInfo(name=TEST_TABLE_NAME) table.refresh_indexed_attributes(client) From d4b128a4684af52af8ff6b8d24c1c4f357ce5e50 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Mon, 24 Aug 2020 18:27:00 -0700 Subject: [PATCH 17/23] chore: force nocmk environment to black all environment variables --- tox.ini | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tox.ini b/tox.ini index 2ad19ad7..6fe21863 100644 --- a/tox.ini +++ b/tox.ini @@ -88,6 +88,11 @@ commands = [testenv:nocmk] basepython = python3 sitepackages = False +######################################################### +# Do not pass through or set any environment variables! # +passenv = +setenv = +######################################################### deps = -rtest/requirements.txt commands = {[testenv:base-command]commands} -m "local and not slow and not veryslow and not nope" From 3f10f435add0248cb52ca0a75113e0e92dbcf0ca Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Mon, 24 Aug 2020 18:40:24 -0700 Subject: [PATCH 18/23] chore: add GitHub Actions workflows --- .github/workflows/ci_static-analysis.yaml | 44 +++++++++++ .github/workflows/ci_tests.yaml | 93 +++++++++++++++++++++++ 2 files changed, 137 insertions(+) create mode 100644 .github/workflows/ci_static-analysis.yaml create mode 100644 .github/workflows/ci_tests.yaml diff --git a/.github/workflows/ci_static-analysis.yaml b/.github/workflows/ci_static-analysis.yaml new file mode 100644 index 00000000..b5d52954 --- /dev/null +++ b/.github/workflows/ci_static-analysis.yaml @@ -0,0 +1,44 @@ +# This workflow runs static analysis checks on pull requests. +name: static analysis + +on: + pull_request: + push: + # Run once a day + schedule: + - cron: '0 0 * * *' + +jobs: + analysis: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + category: +# Disabled pending completion of integration +# https://github.com/aws/aws-dynamodb-encryption-python/issues/66 +# - mypy-py2 +# - mypy-py3 + - bandit + - doc8 + - readme + - docs + - flake8 + - pylint + - flake8-tests + - flake8-examples + - pylint-tests + - pylint-examples + - black-check + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + with: + python-version: 3.x + - run: | + python -m pip install --upgrade pip + pip install --upgrade -r ci-requirements.txt + - name: check + env: + TOXENV: ${{ matrix.category }} + run: tox -- -vv diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml new file mode 100644 index 00000000..72d22138 --- /dev/null +++ b/.github/workflows/ci_tests.yaml @@ -0,0 +1,93 @@ +# This workflow runs tests on pull requests. +name: tests + +on: + pull_request: + push: + # Run once a day + schedule: + - cron: '0 0 * * *' + +jobs: + tests: + runs-on: ${{ matrix.platform.os }} + strategy: + fail-fast: true + matrix: + platform: + - os: ubuntu-latest + architecture: x64 + - os: windows-latest + architecture: x64 + # x86 builds are only meaningful for Windows + - os: windows-latest + architecture: x86 + - os: macos-latest + architecture: x64 + python: + - 2.7 + - 3.5 + - 3.6 + - 3.7 + - 3.8 + - 3.x + category: + - local-slow + - nocmk + - sourcebuildcheck +# These require credentials. +# Enable them once we sort how to provide them. +# - integ-slow +# - examples + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python }} + architecture: ${{ matrix.platform.architecture }} + - run: | + python -m pip install --upgrade pip + pip install --upgrade -r ci-requirements.txt + - name: run test + env: + TOXENV: ${{ matrix.category }} + run: tox -- -vv + upstream-py3: + runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + category: + - nocmk + - test-upstream-requirements-py37 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + with: + python-version: 3.7 + - run: | + python -m pip install --upgrade pip + pip install --upgrade -r ci-requirements.txt + - name: run test + env: + TOXENV: ${{ matrix.category }} + run: tox -- -vv + upstream-py2: + runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + category: + - test-upstream-requirements-py27 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + with: + python-version: 2.7 + - run: | + python -m pip install --upgrade pip + pip install --upgrade -r ci-requirements.txt + - name: run test + env: + TOXENV: ${{ matrix.category }} + run: tox -- -vv From 72a38a79726bd2b004c7cfdf17572b8c923d462d Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Mon, 24 Aug 2020 18:41:58 -0700 Subject: [PATCH 19/23] chore: move sourcebuildcheck and nocmk into upstream-py3 job --- .github/workflows/ci_tests.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 72d22138..b3b57fb9 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -33,8 +33,6 @@ jobs: - 3.x category: - local-slow - - nocmk - - sourcebuildcheck # These require credentials. # Enable them once we sort how to provide them. # - integ-slow @@ -59,6 +57,7 @@ jobs: matrix: category: - nocmk + - sourcebuildcheck - test-upstream-requirements-py37 steps: - uses: actions/checkout@v2 From 824d38d33015767ccfa3b4a3c1b6f11eae67de53 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Mon, 24 Aug 2020 18:42:29 -0700 Subject: [PATCH 20/23] chore: add ci-requirements.txt --- ci-requirements.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 ci-requirements.txt diff --git a/ci-requirements.txt b/ci-requirements.txt new file mode 100644 index 00000000..053148f8 --- /dev/null +++ b/ci-requirements.txt @@ -0,0 +1 @@ +tox From c00699eabdd68c297200bbc056628f4fa88f4a9b Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Tue, 25 Aug 2020 13:58:10 -0700 Subject: [PATCH 21/23] chore: work around bug in Python 2 Hypothesis behavior by only running fast tests for Python 2 --- .github/workflows/ci_tests.yaml | 41 ++++++++++++++++++++++++++++++++- test/requirements.txt | 3 ++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index b3b57fb9..b94c4f9e 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -9,6 +9,46 @@ on: - cron: '0 0 * * *' jobs: + # Hypothesis no longer supports Python 2 and + # there is a bug that appears with our slow tests + # only on Python 2. + # Until we also drop Python 2 support, + # the workaround is just that we don't run the slow tests + # on Python 2. + py2-tests: + runs-on: ${{ matrix.platform.os }} + strategy: + fail-fast: true + matrix: + platform: + - os: ubuntu-latest + architecture: x64 + - os: windows-latest + architecture: x64 + # x86 builds are only meaningful for Windows + - os: windows-latest + architecture: x86 + - os: macos-latest + architecture: x64 + category: + - local-fast + # These require credentials. + # Enable them once we sort how to provide them. + # - integ-fast + # - examples + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + with: + python-version: 2.7 + architecture: ${{ matrix.platform.architecture }} + - run: | + python -m pip install --upgrade pip + pip install --upgrade -r ci-requirements.txt + - name: run test + env: + TOXENV: ${{ matrix.category }} + run: tox -- -vv tests: runs-on: ${{ matrix.platform.os }} strategy: @@ -25,7 +65,6 @@ jobs: - os: macos-latest architecture: x64 python: - - 2.7 - 3.5 - 3.6 - 3.7 diff --git a/test/requirements.txt b/test/requirements.txt index 652ce531..2d87d831 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,4 +1,5 @@ -hypothesis>=3.63.0 +hypothesis>=5.0.0;python_version>='3' +hypothesis==4.57.1;python_version=='2.7' mock moto>=1.3.8 pytest>=3.4.0 From 3c404b3bf4ae169259155a2250d04a707a08e867 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Tue, 25 Aug 2020 14:10:18 -0700 Subject: [PATCH 22/23] chore: fix sourcebuildcheck script - The ls command was getting a relative path when the script ran but a bare filename in manual checks. --- test/source-build-check.sh | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/test/source-build-check.sh b/test/source-build-check.sh index 91403af4..22e31a83 100755 --- a/test/source-build-check.sh +++ b/test/source-build-check.sh @@ -4,20 +4,26 @@ #NOTE: Humans should not run this file directly. If you want to run this check, use the # tox to run the "sourcebuildcheck" test environment. -WORKINGDIR=$1 -DISTDIR=$2 +WORKINGDIR=${1} +DISTDIR=${2} + +echo "Performing source build check" +echo "Using working directory ${WORKINGDIR}" +echo "Using dist directory ${DISTDIR}" echo "Locating the source build and copying it into the working directory." -DISTFILE=`ls $DISTDIR/dynamodb-encryption-sdk-*.tar.gz | tail -1` -cp $DISTFILE $WORKINGDIR -DISTFILE=`ls $WORKINGDIR/dynamodb-encryption-sdk-*.tar.gz | tail -1` +DISTFILE=$(ls ${DISTDIR}/dynamodb-encryption-sdk-*.tar.gz | tail -1) +echo "Found source build at ${DISTFILE}" +cp ${DISTFILE} ${WORKINGDIR} echo "Extracting the source build." -cd $WORKINGDIR -tar xzvf $DISTFILE -rm $DISTFILE -EXTRACTEDDIR=`ls | tail -1` -cd $EXTRACTEDDIR +cd ${WORKINGDIR} +NEWDISTFILE=$(ls dynamodb-encryption-sdk-*.tar.gz | tail -1) +echo "Using distfile ${NEWDISTFILE}" +tar xzvf ${NEWDISTFILE} +rm ${NEWDISTFILE} +EXTRACTEDDIR=$(ls | tail -1) +cd ${EXTRACTEDDIR} echo "Installing requirements from extracted source build." pip install -r test/requirements.txt From ceb0b5f4749a943c1be7e973984a2a71ca2db05d Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Tue, 25 Aug 2020 14:59:32 -0700 Subject: [PATCH 23/23] chore: pruning known runs from Travis that fail due to known infrastructure issues --- .travis.yml | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 070d4a39..df8e4800 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,20 +2,17 @@ sudo: false language: python matrix: include: + # Hypothesis no longer supports Python 2 and + # there is a bug that appears with our slow tests + # only on Python 2. + # Until we also drop Python 2 support, + # the workaround is just that we don't run the slow tests + # on Python 2. # CPython 2.7 - python: 2.7 - env: TOXENV=py27-travis-local-slow + env: TOXENV=py27-travis-local-fast - python: 2.7 env: TOXENV=py27-travis-integ-slow - - python: 2.7 - env: TOXENV=py27-travis-isolation - # CPython 3.4 - - python: 3.4 - env: TOXENV=py34-travis-local-slow - - python: 3.4 - env: TOXENV=py34-travis-integ-slow - - python: 3.4 - env: TOXENV=py34-travis-isolation # CPython 3.5 - python: 3.5 env: TOXENV=py35-travis-local-slow