From 7360edd46964ae0a0d7127a1cf51bd180973af33 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 31 Jan 2024 15:18:21 -0800 Subject: [PATCH 001/376] passing hierarchy keyring example --- examples/src/basic_encryption.py | 98 +++++++++++++++++-- requirements.txt | 2 +- setup.py | 8 ++ .../internal/crypto/authentication.py | 8 +- src/aws_encryption_sdk/streaming_client.py | 83 +++++++++++++--- 5 files changed, 179 insertions(+), 20 deletions(-) diff --git a/examples/src/basic_encryption.py b/examples/src/basic_encryption.py index cfe8ac791..f48f7e4a1 100644 --- a/examples/src/basic_encryption.py +++ b/examples/src/basic_encryption.py @@ -13,7 +13,40 @@ """Example showing basic encryption and decryption of a value already in memory.""" import aws_encryption_sdk from aws_encryption_sdk import CommitmentPolicy +import aws_cryptographic_materialproviders +import boto3 +from aws_encryption_sdk.cmm_handler import (CMMHandler) + +import sys + +module_root_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(module_root_dir) + +import aws_cryptographic_materialproviders + +from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.client import AwsCryptographicMaterialProviders +from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.config import MaterialProvidersConfig +from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.models import ( + CreateAwsKmsHierarchicalKeyringInput, + CacheTypeDefault, + DefaultCache, + GetBranchKeyIdInput, + GetBranchKeyIdOutput, + CreateDefaultCryptographicMaterialsManagerInput, +) +from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.references import ( + IKeyring, + IBranchKeyIdSupplier, +) + +from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_keystore.client import KeyStore +from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_keystore.config import KeyStoreConfig +from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_keystore.models import ( + CreateKeyInput, + KMSConfigurationKmsKeyArn, +) def cycle_string(key_arn, source_plaintext, botocore_session=None): """Encrypts and then decrypts a string under a KMS customer master key (CMK). @@ -25,23 +58,72 @@ def cycle_string(key_arn, source_plaintext, botocore_session=None): """ # Set up an encryption client with an explicit commitment policy. Note that if you do not explicitly choose a # commitment policy, REQUIRE_ENCRYPT_REQUIRE_DECRYPT is used by default. - client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + client = aws_encryption_sdk.EncryptionSDKClient() # Create a KMS master key provider. Note that because we are planning on decrypting using this same provider, # we MUST provide the ARN of the KMS Key. If we provide a raw key id or a key alias, decryption will fail. kms_kwargs = dict(key_ids=[key_arn]) if botocore_session is not None: kms_kwargs["botocore_session"] = botocore_session - master_key_provider = aws_encryption_sdk.StrictAwsKmsMasterKeyProvider(**kms_kwargs) + # master_key_provider = aws_encryption_sdk.StrictAwsKmsMasterKeyProvider(**kms_kwargs) + + ##### + + + key_store_table_name="KeyStoreDdbTable" + logical_key_store_name="KeyStoreDdbTable" + keystore_kms_key_id="arn:aws:kms:us-west-2:370957321024:key/9d989aa2-2f9c-438c-a745-cc57d3ad0126" + + ddb_client = boto3.client('dynamodb') + kms_client = boto3.client('kms') + + keystore: KeyStore = KeyStore( + config=KeyStoreConfig( + ddb_client=ddb_client, + ddb_table_name=key_store_table_name, + logical_key_store_name=logical_key_store_name, + kms_client=kms_client, + kms_configuration=KMSConfigurationKmsKeyArn(value=keystore_kms_key_id), + ) + ) + + new_branch_key_id: str = keystore.create_key(input=CreateKeyInput()).branch_key_identifier + print(f"DEBUG: {new_branch_key_id=}") + + mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + config=MaterialProvidersConfig() + ) + + keyring_input: CreateAwsKmsHierarchicalKeyringInput = CreateAwsKmsHierarchicalKeyringInput( + key_store=keystore, + branch_key_id=new_branch_key_id, + ttl_seconds=600, + cache=CacheTypeDefault(value=DefaultCache(entry_capacity=100)), + ) + + hierarchical_keyring: IKeyring = mat_prov.create_aws_kms_hierarchical_keyring( + input=keyring_input + ) + # This is as far as we can go in the linked Java example without the ESDK. + # We can't use this keyring until it's integrated with the ESDK :( + # Peek at it with print statement for now + print(f"DEBUG: {hierarchical_keyring=}") + + ##### + + cmm = mat_prov.create_default_cryptographic_materials_manager(CreateDefaultCryptographicMaterialsManagerInput(keyring=hierarchical_keyring)) + + cmm_handler: CMMHandler = CMMHandler(cmm) # Encrypt the plaintext source data - ciphertext, encryptor_header = client.encrypt(source=source_plaintext, key_provider=master_key_provider) + ciphertext, encryptor_header = client.encrypt(source=source_plaintext, materials_manager=cmm_handler) # Decrypt the ciphertext - cycled_plaintext, decrypted_header = client.decrypt(source=ciphertext, key_provider=master_key_provider) + cycled_plaintext, decrypted_header = client.decrypt(source=ciphertext, materials_manager=cmm_handler) + cycled_plaintext_str = str(cycled_plaintext, encoding="ascii") # Verify that the "cycled" (encrypted, then decrypted) plaintext is identical to the source plaintext - assert cycled_plaintext == source_plaintext + assert cycled_plaintext_str == source_plaintext # Verify that the encryption context used in the decrypt operation includes all key pairs from # the encrypt operation. (The SDK can add pairs, so don't require an exact match.) @@ -49,5 +131,9 @@ def cycle_string(key_arn, source_plaintext, botocore_session=None): # In production, always use a meaningful encryption context. In this sample, we omit the # encryption context (no key pairs). assert all( - pair in decrypted_header.encryption_context.items() for pair in encryptor_header.encryption_context.items() + (str(k, encoding="ascii"), str(v, encoding="ascii")) in decrypted_header.encryption_context.items() for (k, v) in encryptor_header.encryption_context.items() ) + +# hack in a test +import botocore +cycle_string("arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f", "abcdefg", botocore_session=botocore.session.Session()) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 2f4323845..13466216c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ boto3>=1.10.0 cryptography>=3.4.0 attrs>=17.4.0 -wrapt>=1.10.11 +wrapt>=1.16.0 diff --git a/setup.py b/setup.py index 7cc111bac..cafc16979 100644 --- a/setup.py +++ b/setup.py @@ -39,6 +39,11 @@ def get_requirements(): keywords="aws-encryption-sdk aws kms encryption", license="Apache License 2.0", install_requires=get_requirements(), + # TODO: Point at main once Python is merged into main. + # PyPI will not accept a package that declares dependencies using direct URLs. + extras_require={ + "MPL": ["aws-cryptographic-material-providers @ git+https://github.com/aws/aws-cryptographic-material-providers-library.git@lucmcdon/python-mpl#subdirectory=AwsCryptographicMaterialProviders/runtimes/python"], + }, classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", @@ -49,6 +54,9 @@ def get_requirements(): "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: Implementation :: CPython", "Topic :: Security", "Topic :: Security :: Cryptography", diff --git a/src/aws_encryption_sdk/internal/crypto/authentication.py b/src/aws_encryption_sdk/internal/crypto/authentication.py index f90ac77e0..ad5cf1b2a 100644 --- a/src/aws_encryption_sdk/internal/crypto/authentication.py +++ b/src/aws_encryption_sdk/internal/crypto/authentication.py @@ -76,7 +76,8 @@ def from_key_bytes(cls, algorithm, key_bytes): :param bytes key_bytes: Raw signing key :rtype: aws_encryption_sdk.internal.crypto.Signer """ - key = serialization.load_der_private_key(data=key_bytes, password=None, backend=default_backend()) + # key = serialization.load_der_private_key(data=key_bytes, password=None, backend=default_backend()) + key = serialization.load_pem_private_key(data=key_bytes, password=None, backend=default_backend()) return cls(algorithm, key) def key_bytes(self): @@ -140,6 +141,7 @@ def from_encoded_point(cls, algorithm, encoded_point): :returns: Instance of Verifier generated from encoded point :rtype: aws_encryption_sdk.internal.crypto.Verifier """ + print(f"from_encoded_point {encoded_point=}") return cls( algorithm=algorithm, key=_ecc_public_numbers_from_compressed_point( @@ -157,8 +159,10 @@ def from_key_bytes(cls, algorithm, key_bytes): :returns: Instance of Verifier generated from encoded point :rtype: aws_encryption_sdk.internal.crypto.Verifier """ + print(f"{algorithm=}") + print(f"{key_bytes=}") return cls( - algorithm=algorithm, key=serialization.load_der_public_key(data=key_bytes, backend=default_backend()) + algorithm=algorithm, key=serialization.load_pem_public_key(data=key_bytes, backend=default_backend()) ) def key_bytes(self): diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 1119cb740..afed52e0f 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -66,6 +66,21 @@ from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager from aws_encryption_sdk.materials_managers.default import DefaultCryptoMaterialsManager from aws_encryption_sdk.structures import MessageHeader +try: + import aws_cryptographic_materialproviders + from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.client import AwsCryptographicMaterialProviders + from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.config import MaterialProvidersConfig + from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.models import ( + CreateDefaultCryptographicMaterialsManagerInput + ) + from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.references import ( + IKeyring, + ) + from aws_encryption_sdk.cmm_handler import CMMHandler + + _has_mpl = True +except ImportError as e: + _has_mpl = False _LOGGER = logging.getLogger(__name__) @@ -113,6 +128,10 @@ class _ClientConfig(object): # pylint: disable=too-many-instance-attributes key_provider = attr.ib( hash=True, default=None, validator=attr.validators.optional(attr.validators.instance_of(MasterKeyProvider)) ) + if _has_mpl: + keyring = attr.ib( + hash=True, default=None, validator=attr.validators.optional(attr.validators.instance_of(IKeyring)) + ) source_length = attr.ib( hash=True, default=None, validator=attr.validators.optional(attr.validators.instance_of(six.integer_types)) ) @@ -122,13 +141,38 @@ class _ClientConfig(object): # pylint: disable=too-many-instance-attributes def __attrs_post_init__(self): """Normalize inputs to crypto material manager.""" - both_cmm_and_mkp_defined = self.materials_manager is not None and self.key_provider is not None - neither_cmm_nor_mkp_defined = self.materials_manager is None and self.key_provider is None + if _has_mpl: + all_cmm_and_mkp_and_keyring_defined = all([ + self.materials_manager is not None, + self.key_provider is not None, + self.keyring is not None, + ]) + none_cmm_nor_mkp_nor_keyring_defined = all([ + self.materials_manager is None, + self.key_provider is None, + self.keyring is None, + ]) + + if all_cmm_and_mkp_and_keyring_defined or none_cmm_nor_mkp_nor_keyring_defined: + raise TypeError("Exactly one of keyring, materials_manager, or key_provider must be provided") + if self.materials_manager is None: + if self.key_provider is not None: + self.materials_manager = DefaultCryptoMaterialsManager(master_key_provider=self.key_provider) + elif self.keyring is not None: + mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + config=MaterialProvidersConfig() + ) + cmm = mat_prov.create_default_cryptographic_materials_manager(CreateDefaultCryptographicMaterialsManagerInput(keyring=self.keyring)) + cmm_handler: CryptoMaterialsManager = CMMHandler(cmm) + self.materials_manager = cmm_handler + elif not _has_mpl: + both_cmm_and_mkp_defined = self.materials_manager is not None and self.key_provider is not None + neither_cmm_nor_mkp_defined = self.materials_manager is None and self.key_provider is None - if both_cmm_and_mkp_defined or neither_cmm_nor_mkp_defined: - raise TypeError("Exactly one of materials_manager or key_provider must be provided") - if self.materials_manager is None: - self.materials_manager = DefaultCryptoMaterialsManager(master_key_provider=self.key_provider) + if both_cmm_and_mkp_defined or neither_cmm_nor_mkp_defined: + raise TypeError("Exactly one of materials_manager or key_provider must be provided") + if self.materials_manager is None: + self.materials_manager = DefaultCryptoMaterialsManager(master_key_provider=self.key_provider) class _EncryptionStream(io.IOBase): @@ -343,6 +387,8 @@ class EncryptorConfig(_ClientConfig): :param key_provider: `MasterKeyProvider` from which to obtain data keys for encryption (either `materials_manager` or `key_provider` required) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider + :param keyring: `IKeyring` TODO-MPL content + :type keyring: TODO-MPL :param int source_length: Length of source data (optional) .. note:: @@ -394,6 +440,8 @@ class StreamEncryptor(_EncryptionStream): # pylint: disable=too-many-instance-a :param key_provider: `MasterKeyProvider` from which to obtain data keys for encryption (either `materials_manager` or `key_provider` required) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider + :param keyring: `IKeyring` TODO-MPL content + :type keyring: TODO-MPL :param int source_length: Length of source data (optional) .. note:: @@ -729,11 +777,13 @@ class DecryptorConfig(_ClientConfig): :param source: Source data to encrypt or decrypt :type source: str, bytes, io.IOBase, or file :param materials_manager: `CryptoMaterialsManager` from which to obtain cryptographic materials - (either `materials_manager` or `key_provider` required) + (either `keyring`, `materials_manager` or `key_provider` required) :type materials_manager: aws_encryption_sdk.materials_managers.base.CryptoMaterialsManager :param key_provider: `MasterKeyProvider` from which to obtain data keys for decryption - (either `materials_manager` or `key_provider` required) + (either `keyring`, `materials_manager` or `key_provider` required) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider + :param keyring: `IKeyring` TODO-MPL content + :type keyring: TODO-MPL :param int source_length: Length of source data (optional) .. note:: @@ -770,6 +820,8 @@ class StreamDecryptor(_EncryptionStream): # pylint: disable=too-many-instance-a :param key_provider: `MasterKeyProvider` from which to obtain data keys for decryption (either `materials_manager` or `key_provider` required) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider + :param keyring: `IKeyring` TODO-MPL content + :type keyring: TODO-MPL :param int source_length: Length of source data (optional) .. note:: @@ -831,9 +883,18 @@ def _read_header(self): if decryption_materials.verification_key is None: self.verifier = None else: - self.verifier = Verifier.from_key_bytes( - algorithm=header.algorithm, key_bytes=decryption_materials.verification_key - ) + # MPL verification key is NOT key bytes, it is bytes of the compressed point + # TODO-MPL: clean this up, least-privilege violation + import base64 + if hasattr(self.config.materials_manager, "mpl_cmm"): + self.verifier = Verifier.from_encoded_point( + algorithm=header.algorithm, + encoded_point=base64.b64encode(decryption_materials.verification_key) + ) + else: + self.verifier = Verifier.from_key_bytes( + algorithm=header.algorithm, key_bytes=decryption_materials.verification_key + ) if self.verifier is not None: self.verifier.update(raw_header) From 53c46ece22fec60a2d4d653a4720f66fe706ccca Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 2 Feb 2024 09:19:14 -0800 Subject: [PATCH 002/376] cleanup --- examples/src/basic_encryption.py | 98 +--------------- requirements.txt | 2 +- setup.py | 1 - .../internal/crypto/authentication.py | 3 - src/aws_encryption_sdk/streaming_client.py | 105 +++++++++++------- 5 files changed, 74 insertions(+), 135 deletions(-) diff --git a/examples/src/basic_encryption.py b/examples/src/basic_encryption.py index f48f7e4a1..cfe8ac791 100644 --- a/examples/src/basic_encryption.py +++ b/examples/src/basic_encryption.py @@ -13,40 +13,7 @@ """Example showing basic encryption and decryption of a value already in memory.""" import aws_encryption_sdk from aws_encryption_sdk import CommitmentPolicy -import aws_cryptographic_materialproviders -import boto3 -from aws_encryption_sdk.cmm_handler import (CMMHandler) - -import sys - -module_root_dir = '/'.join(__file__.split("/")[:-1]) - -sys.path.append(module_root_dir) - -import aws_cryptographic_materialproviders - -from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.client import AwsCryptographicMaterialProviders -from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.config import MaterialProvidersConfig -from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.models import ( - CreateAwsKmsHierarchicalKeyringInput, - CacheTypeDefault, - DefaultCache, - GetBranchKeyIdInput, - GetBranchKeyIdOutput, - CreateDefaultCryptographicMaterialsManagerInput, -) -from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.references import ( - IKeyring, - IBranchKeyIdSupplier, -) - -from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_keystore.client import KeyStore -from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_keystore.config import KeyStoreConfig -from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_keystore.models import ( - CreateKeyInput, - KMSConfigurationKmsKeyArn, -) def cycle_string(key_arn, source_plaintext, botocore_session=None): """Encrypts and then decrypts a string under a KMS customer master key (CMK). @@ -58,72 +25,23 @@ def cycle_string(key_arn, source_plaintext, botocore_session=None): """ # Set up an encryption client with an explicit commitment policy. Note that if you do not explicitly choose a # commitment policy, REQUIRE_ENCRYPT_REQUIRE_DECRYPT is used by default. - client = aws_encryption_sdk.EncryptionSDKClient() + client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) # Create a KMS master key provider. Note that because we are planning on decrypting using this same provider, # we MUST provide the ARN of the KMS Key. If we provide a raw key id or a key alias, decryption will fail. kms_kwargs = dict(key_ids=[key_arn]) if botocore_session is not None: kms_kwargs["botocore_session"] = botocore_session - # master_key_provider = aws_encryption_sdk.StrictAwsKmsMasterKeyProvider(**kms_kwargs) - - ##### - - - key_store_table_name="KeyStoreDdbTable" - logical_key_store_name="KeyStoreDdbTable" - keystore_kms_key_id="arn:aws:kms:us-west-2:370957321024:key/9d989aa2-2f9c-438c-a745-cc57d3ad0126" - - ddb_client = boto3.client('dynamodb') - kms_client = boto3.client('kms') - - keystore: KeyStore = KeyStore( - config=KeyStoreConfig( - ddb_client=ddb_client, - ddb_table_name=key_store_table_name, - logical_key_store_name=logical_key_store_name, - kms_client=kms_client, - kms_configuration=KMSConfigurationKmsKeyArn(value=keystore_kms_key_id), - ) - ) - - new_branch_key_id: str = keystore.create_key(input=CreateKeyInput()).branch_key_identifier - print(f"DEBUG: {new_branch_key_id=}") - - mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( - config=MaterialProvidersConfig() - ) - - keyring_input: CreateAwsKmsHierarchicalKeyringInput = CreateAwsKmsHierarchicalKeyringInput( - key_store=keystore, - branch_key_id=new_branch_key_id, - ttl_seconds=600, - cache=CacheTypeDefault(value=DefaultCache(entry_capacity=100)), - ) - - hierarchical_keyring: IKeyring = mat_prov.create_aws_kms_hierarchical_keyring( - input=keyring_input - ) - # This is as far as we can go in the linked Java example without the ESDK. - # We can't use this keyring until it's integrated with the ESDK :( - # Peek at it with print statement for now - print(f"DEBUG: {hierarchical_keyring=}") - - ##### - - cmm = mat_prov.create_default_cryptographic_materials_manager(CreateDefaultCryptographicMaterialsManagerInput(keyring=hierarchical_keyring)) - - cmm_handler: CMMHandler = CMMHandler(cmm) + master_key_provider = aws_encryption_sdk.StrictAwsKmsMasterKeyProvider(**kms_kwargs) # Encrypt the plaintext source data - ciphertext, encryptor_header = client.encrypt(source=source_plaintext, materials_manager=cmm_handler) + ciphertext, encryptor_header = client.encrypt(source=source_plaintext, key_provider=master_key_provider) # Decrypt the ciphertext - cycled_plaintext, decrypted_header = client.decrypt(source=ciphertext, materials_manager=cmm_handler) - cycled_plaintext_str = str(cycled_plaintext, encoding="ascii") + cycled_plaintext, decrypted_header = client.decrypt(source=ciphertext, key_provider=master_key_provider) # Verify that the "cycled" (encrypted, then decrypted) plaintext is identical to the source plaintext - assert cycled_plaintext_str == source_plaintext + assert cycled_plaintext == source_plaintext # Verify that the encryption context used in the decrypt operation includes all key pairs from # the encrypt operation. (The SDK can add pairs, so don't require an exact match.) @@ -131,9 +49,5 @@ def cycle_string(key_arn, source_plaintext, botocore_session=None): # In production, always use a meaningful encryption context. In this sample, we omit the # encryption context (no key pairs). assert all( - (str(k, encoding="ascii"), str(v, encoding="ascii")) in decrypted_header.encryption_context.items() for (k, v) in encryptor_header.encryption_context.items() + pair in decrypted_header.encryption_context.items() for pair in encryptor_header.encryption_context.items() ) - -# hack in a test -import botocore -cycle_string("arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f", "abcdefg", botocore_session=botocore.session.Session()) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 13466216c..2f4323845 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ boto3>=1.10.0 cryptography>=3.4.0 attrs>=17.4.0 -wrapt>=1.16.0 +wrapt>=1.10.11 diff --git a/setup.py b/setup.py index cafc16979..8ffd74015 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,6 @@ def get_requirements(): "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", "Programming Language :: Python :: Implementation :: CPython", "Topic :: Security", "Topic :: Security :: Cryptography", diff --git a/src/aws_encryption_sdk/internal/crypto/authentication.py b/src/aws_encryption_sdk/internal/crypto/authentication.py index ad5cf1b2a..b9692eb16 100644 --- a/src/aws_encryption_sdk/internal/crypto/authentication.py +++ b/src/aws_encryption_sdk/internal/crypto/authentication.py @@ -141,7 +141,6 @@ def from_encoded_point(cls, algorithm, encoded_point): :returns: Instance of Verifier generated from encoded point :rtype: aws_encryption_sdk.internal.crypto.Verifier """ - print(f"from_encoded_point {encoded_point=}") return cls( algorithm=algorithm, key=_ecc_public_numbers_from_compressed_point( @@ -159,8 +158,6 @@ def from_key_bytes(cls, algorithm, key_bytes): :returns: Instance of Verifier generated from encoded point :rtype: aws_encryption_sdk.internal.crypto.Verifier """ - print(f"{algorithm=}") - print(f"{key_bytes=}") return cls( algorithm=algorithm, key=serialization.load_pem_public_key(data=key_bytes, backend=default_backend()) ) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index afed52e0f..176b92334 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -139,40 +139,61 @@ class _ClientConfig(object): # pylint: disable=too-many-instance-attributes hash=True, default=LINE_LENGTH, validator=attr.validators.instance_of(six.integer_types) ) # DEPRECATED: Value is no longer configurable here. Parameter left here to avoid breaking consumers. + def _has_mpl_attrs_post_init(self): + + def _exactly_one_arg_is_not_None(*args): + ''' + Private helper function. + Returns `True` if exactly one item in the list is not `None`. + Returns `False` otherwise. + ''' + # Have not found any `not None` + found_one = False + for arg in args: + if arg is not None: + if found_one == False: + # Have not already found a `not None`, found a `not None` => only one `not None` (so far) + found_one = True + else: + # Already found a `not None`, found another `not None` => not exactly one `not None` + return False + return found_one + + if not _exactly_one_arg_is_not_None(self.materials_manager, self.key_provider, self.keyring): + raise TypeError("Exactly one of keyring, materials_manager, or key_provider must be provided") + if self.materials_manager is None: + if self.key_provider is not None: + # No CMM, provided (legacy) native `key_provider` => create (legacy) native DefaultCryptoMaterialsManager + self.materials_manager = DefaultCryptoMaterialsManager(master_key_provider=self.key_provider) + elif self.keyring is not None: + # No CMM, provided MPL keyring => create MPL's DefaultCryptographicMaterialsManager + try: + assert isinstance(self.keyring, IKeyring) + except AssertionError as e: + raise ValueError(f"Argument provided to keyring MUST be a {IKeyring}. Found {keyring.__class__.__name__=}") + + mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + config=MaterialProvidersConfig() + ) + cmm = mat_prov.create_default_cryptographic_materials_manager(CreateDefaultCryptographicMaterialsManagerInput(keyring=self.keyring)) + cmm_handler: CryptoMaterialsManager = CMMHandler(cmm) + self.materials_manager = cmm_handler + + def _no_mpl_attrs_post_init(self): + both_cmm_and_mkp_defined = self.materials_manager is not None and self.key_provider is not None + neither_cmm_nor_mkp_defined = self.materials_manager is None and self.key_provider is None + + if both_cmm_and_mkp_defined or neither_cmm_nor_mkp_defined: + raise TypeError("Exactly one of materials_manager or key_provider must be provided") + if self.materials_manager is None: + self.materials_manager = DefaultCryptoMaterialsManager(master_key_provider=self.key_provider) + def __attrs_post_init__(self): """Normalize inputs to crypto material manager.""" if _has_mpl: - all_cmm_and_mkp_and_keyring_defined = all([ - self.materials_manager is not None, - self.key_provider is not None, - self.keyring is not None, - ]) - none_cmm_nor_mkp_nor_keyring_defined = all([ - self.materials_manager is None, - self.key_provider is None, - self.keyring is None, - ]) - - if all_cmm_and_mkp_and_keyring_defined or none_cmm_nor_mkp_nor_keyring_defined: - raise TypeError("Exactly one of keyring, materials_manager, or key_provider must be provided") - if self.materials_manager is None: - if self.key_provider is not None: - self.materials_manager = DefaultCryptoMaterialsManager(master_key_provider=self.key_provider) - elif self.keyring is not None: - mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( - config=MaterialProvidersConfig() - ) - cmm = mat_prov.create_default_cryptographic_materials_manager(CreateDefaultCryptographicMaterialsManagerInput(keyring=self.keyring)) - cmm_handler: CryptoMaterialsManager = CMMHandler(cmm) - self.materials_manager = cmm_handler + self._has_mpl_attrs_post_init() elif not _has_mpl: - both_cmm_and_mkp_defined = self.materials_manager is not None and self.key_provider is not None - neither_cmm_nor_mkp_defined = self.materials_manager is None and self.key_provider is None - - if both_cmm_and_mkp_defined or neither_cmm_nor_mkp_defined: - raise TypeError("Exactly one of materials_manager or key_provider must be provided") - if self.materials_manager is None: - self.materials_manager = DefaultCryptoMaterialsManager(master_key_provider=self.key_provider) + self._no_mpl_attrs_post_init() class _EncryptionStream(io.IOBase): @@ -387,8 +408,10 @@ class EncryptorConfig(_ClientConfig): :param key_provider: `MasterKeyProvider` from which to obtain data keys for encryption (either `materials_manager` or `key_provider` required) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider - :param keyring: `IKeyring` TODO-MPL content - :type keyring: TODO-MPL + :param keyring: `IKeyring` from the aws_cryptographic_materialproviders library + which handles encryption and decryption + :type keyring: + aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.references.IKeyring :param int source_length: Length of source data (optional) .. note:: @@ -440,8 +463,10 @@ class StreamEncryptor(_EncryptionStream): # pylint: disable=too-many-instance-a :param key_provider: `MasterKeyProvider` from which to obtain data keys for encryption (either `materials_manager` or `key_provider` required) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider - :param keyring: `IKeyring` TODO-MPL content - :type keyring: TODO-MPL + :param keyring: `IKeyring` from the aws_cryptographic_materialproviders library + which handles encryption and decryption + :type keyring: + aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.references.IKeyring :param int source_length: Length of source data (optional) .. note:: @@ -782,8 +807,10 @@ class DecryptorConfig(_ClientConfig): :param key_provider: `MasterKeyProvider` from which to obtain data keys for decryption (either `keyring`, `materials_manager` or `key_provider` required) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider - :param keyring: `IKeyring` TODO-MPL content - :type keyring: TODO-MPL + :param keyring: `IKeyring` from the aws_cryptographic_materialproviders library + which handles encryption and decryption + :type keyring: + aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.references.IKeyring :param int source_length: Length of source data (optional) .. note:: @@ -820,8 +847,10 @@ class StreamDecryptor(_EncryptionStream): # pylint: disable=too-many-instance-a :param key_provider: `MasterKeyProvider` from which to obtain data keys for decryption (either `materials_manager` or `key_provider` required) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider - :param keyring: `IKeyring` TODO-MPL content - :type keyring: TODO-MPL + :param keyring: `IKeyring` from the aws_cryptographic_materialproviders library + which handles encryption and decryption + :type keyring: + aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.references.IKeyring :param int source_length: Length of source data (optional) .. note:: From 3f5a503ab7f866c7750cabdc79dbf7f8f75a34e6 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 2 Feb 2024 09:21:46 -0800 Subject: [PATCH 003/376] add --- setup.py | 3 +- src/aws_encryption_sdk/cmm_handler.py | 138 ++++++++++++++++ src/aws_encryption_sdk/materials_handlers.py | 165 +++++++++++++++++++ 3 files changed, 304 insertions(+), 2 deletions(-) create mode 100644 src/aws_encryption_sdk/cmm_handler.py create mode 100644 src/aws_encryption_sdk/materials_handlers.py diff --git a/setup.py b/setup.py index 8ffd74015..c4c277096 100644 --- a/setup.py +++ b/setup.py @@ -39,8 +39,7 @@ def get_requirements(): keywords="aws-encryption-sdk aws kms encryption", license="Apache License 2.0", install_requires=get_requirements(), - # TODO: Point at main once Python is merged into main. - # PyPI will not accept a package that declares dependencies using direct URLs. + # TODO: Point at MPL main branch once Python MPL is merged into main. extras_require={ "MPL": ["aws-cryptographic-material-providers @ git+https://github.com/aws/aws-cryptographic-material-providers-library.git@lucmcdon/python-mpl#subdirectory=AwsCryptographicMaterialProviders/runtimes/python"], }, diff --git a/src/aws_encryption_sdk/cmm_handler.py b/src/aws_encryption_sdk/cmm_handler.py new file mode 100644 index 000000000..82aad2248 --- /dev/null +++ b/src/aws_encryption_sdk/cmm_handler.py @@ -0,0 +1,138 @@ +# These dependencies are only loaded if you install the MPL. +try: + from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.references import ( + ICryptographicMaterialsManager, + ) + from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.models import ( + GetEncryptionMaterialsInput, + GetEncryptionMaterialsOutput, + DecryptMaterialsInput, + DecryptMaterialsOutput, + EncryptedDataKey as MPL_EncryptedDataKey, + CommitmentPolicyESDK, + AlgorithmSuiteIdESDK, + ) +except ImportError as e: + print(f"WARNING: MPL import failed with {e=}") + +from aws_encryption_sdk.materials_managers import ( + DecryptionMaterialsRequest, + EncryptionMaterialsRequest, +) +from aws_encryption_sdk.materials_managers.base import ( + CryptoMaterialsManager, +) +from aws_encryption_sdk.materials_handlers import ( + EncryptionMaterialsHandler, + DecryptionMaterialsHandler, +) +from aws_encryption_sdk.structures import ( + EncryptedDataKey as Native_EncryptedDataKey, +) +from aws_encryption_sdk.identifiers import ( + Algorithm, + AlgorithmSuite, + CommitmentPolicy, +) + +# TODO-MPL Should this implement interface..? seems like yes since it implements all of interface methods +class CMMHandler(CryptoMaterialsManager): + native_cmm: CryptoMaterialsManager + mpl_cmm: 'ICryptographicMaterialsManager' + + def __init__( + self, + cmm: 'CryptoMaterialsManager | ICryptographicMaterialsManager' + ): + if isinstance(cmm, CryptoMaterialsManager): + self.native_cmm = cmm + elif isinstance(cmm, ICryptographicMaterialsManager): + self.mpl_cmm = cmm + else: + raise ValueError(f"Invalid CMM passed to CMMHander: {cmm=}") + + def get_encryption_materials( + self, + request: EncryptionMaterialsRequest + ) -> EncryptionMaterialsHandler: + ''' + Returns an EncryptionMaterialsHandler based on the configured CMM. + ''' + if (hasattr(self, "native_cmm") and not hasattr(self, "mpl_cmm")): + return EncryptionMaterialsHandler(self.native_cmm.get_encryption_materials(request)) + else: + input: GetEncryptionMaterialsInput = CMMHandler._create_mpl_get_encryption_materials_input_from_request(request) + print(f"get_encryption_materials {input=}") + output: GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(input) + print(f"get_encryption_materials {output=}") + return EncryptionMaterialsHandler(output.encryption_materials) + + @staticmethod + def _create_mpl_get_encryption_materials_input_from_request( + request: EncryptionMaterialsRequest + ) -> 'GetEncryptionMaterialsInput': + print(f"_create_mpl_get_encryption_materials_input_from_request {request=}") + print(f"{CMMHandler._map_native_commitment_policy_to_mpl_commitment_policy(request.commitment_policy)=}") + print(f"_create_mpl_get_encryption_materials_input_from_request {request.encryption_context=}") + output: GetEncryptionMaterialsInput = GetEncryptionMaterialsInput( + encryption_context=request.encryption_context, + commitment_policy=CMMHandler._map_native_commitment_policy_to_mpl_commitment_policy(request.commitment_policy), + # TODO double check this + # optional... maybe this needs to be kwargs?? + # algorithm_suite_id=request.algorithm.algorithm_id, + max_plaintext_length=request.plaintext_length, + ) + print(f"_create_mpl_get_encryption_materials_input_from_request {output=}") + return output + + @staticmethod + def _map_native_commitment_policy_to_mpl_commitment_policy( + native_commitment_policy: CommitmentPolicy + ) -> CommitmentPolicyESDK: + if native_commitment_policy == CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT: + return CommitmentPolicyESDK(value="FORBID_ENCRYPT_ALLOW_DECRYPT") + elif native_commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT: + return CommitmentPolicyESDK(value="REQUIRE_ENCRYPT_ALLOW_DECRYPT") + elif native_commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT: + return CommitmentPolicyESDK(value="REQUIRE_ENCRYPT_REQUIRE_DECRYPT") + else: + raise ValueError(f"Invalid {native_commitment_policy=}") + + def decrypt_materials( + self, + request: DecryptionMaterialsRequest + ) -> DecryptionMaterialsHandler: + ''' + Returns a DecryptionMaterialsHandler based on the configured CMM. + ''' + print(f"decrypt_materials {request=}") + if (hasattr(self, "native_cmm") and not hasattr(self, "mpl_cmm")): + return DecryptionMaterialsHandler(self.native_cmm.decrypt_materials(request)) + else: + input: 'DecryptMaterialsInput' = CMMHandler._create_mpl_decrypt_materials_input_from_request(request) + output: 'DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(input) + print(f"decrypt_materials {output.decryption_materials.verification_key=}") + return DecryptionMaterialsHandler(output.decryption_materials) + + @staticmethod + def _native_algorithm_id_to_mpl_algorithm_id(native_algorithm_id: str) -> AlgorithmSuiteIdESDK: + # MPL algorithm suite ID = hexstr(native_algorithm_id) padded to 4 digits post-`x`. + return AlgorithmSuiteIdESDK(f"{native_algorithm_id:#0{6}x}") + + @staticmethod + def _create_mpl_decrypt_materials_input_from_request( + request: DecryptionMaterialsRequest + ) -> 'DecryptMaterialsInput': + key_blob_list: list[Native_EncryptedDataKey] = request.encrypted_data_keys + list_edks = [MPL_EncryptedDataKey( + key_provider_id=key_blob.key_provider.provider_id, + key_provider_info=key_blob.key_provider.key_info, + ciphertext=key_blob.encrypted_data_key, + ) for key_blob in key_blob_list] + output: DecryptMaterialsInput = DecryptMaterialsInput( + algorithm_suite_id=CMMHandler._native_algorithm_id_to_mpl_algorithm_id(request.algorithm.algorithm_id), + commitment_policy=CMMHandler._map_native_commitment_policy_to_mpl_commitment_policy(request.commitment_policy), + encrypted_data_keys=list_edks, + encryption_context=request.encryption_context, + ) + return output diff --git a/src/aws_encryption_sdk/materials_handlers.py b/src/aws_encryption_sdk/materials_handlers.py new file mode 100644 index 000000000..bf3073ad3 --- /dev/null +++ b/src/aws_encryption_sdk/materials_handlers.py @@ -0,0 +1,165 @@ +# These dependencies are only loaded if you install the MPL. +try: + from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.models import ( + DecryptionMaterials as MPL_DecryptionMaterials, + EncryptionMaterials as MPL_EncryptionMaterials, + EncryptedDataKey as MPL_EncryptedDataKey, + ) +except ImportError as e: + pass + +from aws_encryption_sdk.materials_managers import ( + DecryptionMaterials as Native_DecryptionMaterials, + EncryptionMaterials as Native_EncryptionMaterials, +) +from aws_encryption_sdk.identifiers import ( + Algorithm, + AlgorithmSuite, +) +from aws_encryption_sdk.structures import ( + DataKey, + EncryptedDataKey as Native_EncryptedDataKey, + MasterKeyInfo, +) +from aws_encryption_sdk.internal.crypto.authentication import ( + Signer +) + +class EncryptionMaterialsHandler: + native_materials: Native_EncryptionMaterials + mpl_materials: 'MPL_EncryptionMaterials' + + @staticmethod + def _mpl_algorithm_id_to_native_algorithm_id(mpl_algorithm_id: str): + # MPL algorithm suite ID == "ALG_" + native algorithm suite ID. + return int(mpl_algorithm_id, 16) + + def __init__( + self, + materials: 'Native_EncryptionMaterials | MPL_EncryptionMaterials' + ): + if isinstance(materials, Native_EncryptionMaterials): + self.native_materials = materials + elif isinstance(materials, MPL_EncryptionMaterials): + self.mpl_materials = materials + else: + raise ValueError(f"Invalid EncryptionMaterials passed to EncryptionMaterialsHandler: {materials=}") + @property + def algorithm(self) -> Algorithm: + if hasattr(self, "native_materials"): + return self.native_materials.algorithm + else: + print(f"algorithm {self.mpl_materials.algorithm_suite.id.value=}") + return AlgorithmSuite.get_by_id( + EncryptionMaterialsHandler._mpl_algorithm_id_to_native_algorithm_id( + self.mpl_materials.algorithm_suite.id.value + ) + ) + + @property + def encryption_context(self) -> dict[str, str]: + if hasattr(self, "native_materials"): + return self.native_materials.encryption_context + else: + return self.mpl_materials.encryption_context + + @property + def encrypted_data_keys(self) -> list[Native_EncryptedDataKey]: + if hasattr(self, "native_materials"): + return self.native_materials.encrypted_data_keys + else: + mpl_edk_list: list[MPL_EncryptedDataKey] = self.mpl_materials.encrypted_data_keys + key_blob_list: set[Native_EncryptedDataKey] = {Native_EncryptedDataKey( + key_provider=MasterKeyInfo( + provider_id=mpl_edk.key_provider_id, + key_info=mpl_edk.key_provider_info, + ), + encrypted_data_key=mpl_edk.ciphertext, + ) for mpl_edk in mpl_edk_list} + return key_blob_list + + @property + def data_encryption_key(self) -> DataKey: + if hasattr(self, "native_materials"): + return self.native_materials.data_encryption_key + else: + # TODO-MPL This impl is probably wrong + mpl_dek = self.mpl_materials.plaintext_data_key + return DataKey( + # key_provider=None, # No MasterKeyInfo object for plaintext data key + key_provider=MasterKeyInfo( + provider_id="", + key_info=b'' + ), + data_key=mpl_dek, + encrypted_data_key=b'', # No encrypted DEK + ) + + @property + def signing_key(self) -> bytes: + if hasattr(self, "native_materials"): + return self.native_materials.signing_key + else: + print(f"sign {self.mpl_materials.signing_key=}") + return self.mpl_materials.signing_key + # if self.mpl_materials.signing_key is None: + # return Signer.from_key_bytes( + # algorithm=AlgorithmSuite.get_by_id(self.mpl_materials.algorithm_suite.id.value), + # bytes=self.mpl_materials.signing_key + # ) + + def get_required_encryption_context_keys(self) -> list[str]: + if hasattr(self, "native_materials"): + return [] + else: + return self.mpl_materials.required_encryption_context_keys + +class DecryptionMaterialsHandler: + native_materials: Native_DecryptionMaterials + mpl_materials: 'MPL_DecryptionMaterials' + + def __init__( + self, + materials: 'Native_DecryptionMaterials | MPL_DecryptionMaterials' + ): + if isinstance(materials, Native_DecryptionMaterials): + self.native_materials = materials + elif isinstance(materials, MPL_DecryptionMaterials): + self.mpl_materials = materials + else: + raise ValueError(f"Invalid DecryptionMaterials passed to DecryptionMaterialsHandler: {materials=}") + + def get_encryption_context(self) -> dict[str, str]: + if hasattr(self, "native_materials"): + return {} # TODO-MPL This impl is probably wrong + else: + return self.mpl_materials.encryption_context + + @property + def data_key(self) -> DataKey: + if hasattr(self, "native_materials"): + return self.native_materials.data_key + else: + # TODO-MPL This impl is probably wrong + return DataKey( + key_provider=MasterKeyInfo( + provider_id="", + key_info=b'' + ), + data_key=self.mpl_materials.plaintext_data_key, + encrypted_data_key=b'', + ) + + @property + def verification_key(self) -> bytes: + if hasattr(self, "native_materials"): + return self.native_materials.verification_key + else: + print(f"ver {self.mpl_materials.verification_key=}") + return self.mpl_materials.verification_key + + def get_required_encryption_context_keys(self) -> list[str]: + if hasattr(self, "native_materials"): + return [] + else: + return self.mpl_materials.required_encryption_context_keys \ No newline at end of file From 16cf5c1f38ae1aaa0e7e5f68fc58cd0c6f3532b2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 2 Feb 2024 14:41:53 -0800 Subject: [PATCH 004/376] changes, cleanup: --- examples/src/basic_encryption.py | 5 + examples/src/keyrings/hierarchical_keyring.py | 248 ++++++++++++++++++ examples/src/keyrings/module_.py | 0 examples/src/module_.py | 0 src/aws_encryption_sdk/cmm_handler.py | 55 ++-- src/aws_encryption_sdk/materials_handlers.py | 56 ++-- src/aws_encryption_sdk/streaming_client.py | 18 +- .../test_streaming_client_stream_decryptor.py | 2 +- 8 files changed, 314 insertions(+), 70 deletions(-) create mode 100644 examples/src/keyrings/hierarchical_keyring.py create mode 100644 examples/src/keyrings/module_.py create mode 100644 examples/src/module_.py diff --git a/examples/src/basic_encryption.py b/examples/src/basic_encryption.py index cfe8ac791..7b729feab 100644 --- a/examples/src/basic_encryption.py +++ b/examples/src/basic_encryption.py @@ -51,3 +51,8 @@ def cycle_string(key_arn, source_plaintext, botocore_session=None): assert all( pair in decrypted_header.encryption_context.items() for pair in encryptor_header.encryption_context.items() ) + +cycle_string( + "arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f", + "abcdefg", +) \ No newline at end of file diff --git a/examples/src/keyrings/hierarchical_keyring.py b/examples/src/keyrings/hierarchical_keyring.py new file mode 100644 index 000000000..e8f662b73 --- /dev/null +++ b/examples/src/keyrings/hierarchical_keyring.py @@ -0,0 +1,248 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + + + +"""Example showing basic encryption and decryption of a value already in memory.""" +import aws_encryption_sdk +from aws_encryption_sdk import CommitmentPolicy +import boto3 + +import sys + +from aws_encryption_sdk.exceptions import ( + AWSEncryptionSDKClientError, + SerializationError, +) + +module_root_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(module_root_dir) + +import aws_cryptographic_materialproviders + +from aws_cryptographic_materialproviders.mpl.client import AwsCryptographicMaterialProviders +from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig +from aws_cryptographic_materialproviders.mpl.models import ( + CreateAwsKmsHierarchicalKeyringInput, + CacheTypeDefault, + DefaultCache, + GetBranchKeyIdInput, + GetBranchKeyIdOutput, +) +from aws_cryptographic_materialproviders.mpl.references import ( + IKeyring, + IBranchKeyIdSupplier, +) + +from aws_cryptographic_materialproviders.keystore.client import KeyStore +from aws_cryptographic_materialproviders.keystore.config import KeyStoreConfig +from aws_cryptographic_materialproviders.keystore.models import ( + CreateKeyInput, + KMSConfigurationKmsKeyArn, +) + +EXAMPLE_DATA: bytes = b"Hello World" + +def encrypt_and_decrypt_with_keyring( + key_store_table_name: str, + logical_key_store_name: str, + kms_key_id: str + ): + + # 1. Instantiate the encryption SDK client. + # This builds the client with the REQUIRE_ENCRYPT_REQUIRE_DECRYPT commitment policy, + # which enforces that this client only encrypts using committing algorithm suites and enforces + # that this client will only decrypt encrypted messages that were created with a committing + # algorithm suite. + # This is the default commitment policy if you were to build the client as + # `client = aws_encryption_sdk.EncryptionSDKClient()`. + + client = aws_encryption_sdk.EncryptionSDKClient( + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT + ) + + # 2. Create boto3 clients for DynamoDB and KMS. + ddb_client = boto3.client('dynamodb') + kms_client = boto3.client('kms') + + # 3. Configure your KeyStore resource. + # This SHOULD be the same configuration that you used + # to initially create and populate your KeyStore. + keystore: KeyStore = KeyStore( + config=KeyStoreConfig( + ddb_client=ddb_client, + ddb_table_name=key_store_table_name, + logical_key_store_name=logical_key_store_name, + kms_client=kms_client, + kms_configuration=KMSConfigurationKmsKeyArn( + value=kms_key_id + ), + ) + ) + + # 4. Call CreateKey to create two new active branch keys + branch_key_id_A: str = keystore.create_key(input=CreateKeyInput()).branch_key_identifier + branch_key_id_B: str = keystore.create_key(input=CreateKeyInput()).branch_key_identifier + + class ExampleBranchKeyIdSupplier(IBranchKeyIdSupplier): + branch_key_id_for_tenant_A: str + branch_key_id_for_tenant_B: str + + def __init__(self, tenant_1_id, tenant_2_id): + self.branch_key_id_for_tenant_A = tenant_1_id + self.branch_key_id_for_tenant_B = tenant_2_id + + def get_branch_key_id( + self, + input: GetBranchKeyIdInput + ) -> GetBranchKeyIdOutput: + encryption_context: dict[str, str] = input.encryption_context + + if b"tenant" not in encryption_context: + raise ValueError("EncryptionContext invalid, does not contain expected tenant key value pair.") + + tenant_key_id: str = encryption_context.get(b"tenant") + branch_key_id: str + + if tenant_key_id == b"TenantA": + branch_key_id = self.branch_key_id_for_tenant_A + elif tenant_key_id == b"TenantB": + branch_key_id = self.branch_key_id_for_tenant_B + else: + raise ValueError(f"Item does not contain valid tenant ID: {tenant_key_id=}") + + return GetBranchKeyIdOutput(branch_key_id=branch_key_id) + + # 5. Create a branch key supplier that maps the branch key id to a more readable format + branch_key_id_supplier: IBranchKeyIdSupplier = ExampleBranchKeyIdSupplier( + tenant_1_id=branch_key_id_A, + tenant_2_id=branch_key_id_B, + ) + + # 6. Create the Hierarchical Keyring. + mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + config=MaterialProvidersConfig() + ) + + keyring_input: CreateAwsKmsHierarchicalKeyringInput = CreateAwsKmsHierarchicalKeyringInput( + key_store=keystore, + branch_key_id_supplier=branch_key_id_supplier, + ttl_seconds=600, + cache=CacheTypeDefault( + value=DefaultCache( + entry_capacity=100 + ) + ), + ) + + hierarchical_keyring: IKeyring = mat_prov.create_aws_kms_hierarchical_keyring( + input=keyring_input + ) + + # The Branch Key Id supplier uses the encryption context to determine which branch key id will + # be used to encrypt data. + # Create encryption context for TenantA + encryption_context_A: dict[str, str] = { + "tenant": "TenantA", + "encryption": "context", + "is not": "secret", + "but adds": "useful metadata", + "that can help you": "be confident that", + "the data you are handling": "is what you think it is", + } + + # Create encryption context for TenantB + encryption_context_B: dict[str, str] = { + "tenant": "TenantB", + "encryption": "context", + "is not": "secret", + "but adds": "useful metadata", + "that can help you": "be confident that", + "the data you are handling": "is what you think it is", + } + + # Encrypt the data for encryptionContextA & encryptionContextB + ciphertext_A, _ = client.encrypt( + source=EXAMPLE_DATA, + keyring=hierarchical_keyring, + encryption_context=encryption_context_A + ) + ciphertext_B, _ = client.encrypt( + source=EXAMPLE_DATA, + keyring=hierarchical_keyring, + encryption_context=encryption_context_B + ) + + # To attest that TenantKeyB cannot decrypt a message written by TenantKeyA + # let's construct more restrictive hierarchical keyrings. + keyring_input_A: CreateAwsKmsHierarchicalKeyringInput = CreateAwsKmsHierarchicalKeyringInput( + key_store=keystore, + branch_key_id=branch_key_id_A, + ttl_seconds=600, + cache=CacheTypeDefault( + value=DefaultCache( + entry_capacity=100 + ) + ), + ) + + hierarchical_keyring_A: IKeyring = mat_prov.create_aws_kms_hierarchical_keyring( + input=keyring_input_A + ) + + keyring_input_B: CreateAwsKmsHierarchicalKeyringInput = CreateAwsKmsHierarchicalKeyringInput( + key_store=keystore, + branch_key_id=branch_key_id_B, + ttl_seconds=600, + cache=CacheTypeDefault( + value=DefaultCache( + entry_capacity=100 + ) + ), + ) + + hierarchical_keyring_B: IKeyring = mat_prov.create_aws_kms_hierarchical_keyring( + input=keyring_input_B + ) + + # TODO: Run the decrypt, get expected exception type + # This should fail + try: + client.decrypt( + source=ciphertext_A, + keyring=hierarchical_keyring_B + ) + except AWSEncryptionSDKClientError: + pass + + # # This should fail + try: + client.decrypt( + source=ciphertext_B, + keyring=hierarchical_keyring_A + ) + except AWSEncryptionSDKClientError: + pass + + # These should succeed + plaintext_bytes_A, _ = client.decrypt( + source=ciphertext_A, + keyring=hierarchical_keyring_A + ) + assert plaintext_bytes_A == EXAMPLE_DATA + plaintext_bytes_B, _ = client.decrypt( + source=ciphertext_B, + keyring=hierarchical_keyring_B + ) + assert plaintext_bytes_B == EXAMPLE_DATA + +# Also, a thread-safe example ig + +# hack in a test +import botocore +encrypt_and_decrypt_with_keyring( + "KeyStoreDdbTable", + "KeyStoreDdbTable", + "arn:aws:kms:us-west-2:370957321024:key/9d989aa2-2f9c-438c-a745-cc57d3ad0126" +) \ No newline at end of file diff --git a/examples/src/keyrings/module_.py b/examples/src/keyrings/module_.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/src/module_.py b/examples/src/module_.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/aws_encryption_sdk/cmm_handler.py b/src/aws_encryption_sdk/cmm_handler.py index 82aad2248..d634dd571 100644 --- a/src/aws_encryption_sdk/cmm_handler.py +++ b/src/aws_encryption_sdk/cmm_handler.py @@ -1,9 +1,12 @@ # These dependencies are only loaded if you install the MPL. try: - from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.references import ( + from aws_cryptographic_materialproviders.mpl.errors import ( + AwsCryptographicMaterialProvidersException + ) + from aws_cryptographic_materialproviders.mpl.references import ( ICryptographicMaterialsManager, ) - from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.models import ( + from aws_cryptographic_materialproviders.mpl.models import ( GetEncryptionMaterialsInput, GetEncryptionMaterialsOutput, DecryptMaterialsInput, @@ -13,8 +16,11 @@ AlgorithmSuiteIdESDK, ) except ImportError as e: - print(f"WARNING: MPL import failed with {e=}") + pass +from aws_encryption_sdk.exceptions import ( + AWSEncryptionSDKClientError, +) from aws_encryption_sdk.materials_managers import ( DecryptionMaterialsRequest, EncryptionMaterialsRequest, @@ -30,8 +36,6 @@ EncryptedDataKey as Native_EncryptedDataKey, ) from aws_encryption_sdk.identifiers import ( - Algorithm, - AlgorithmSuite, CommitmentPolicy, ) @@ -40,6 +44,9 @@ class CMMHandler(CryptoMaterialsManager): native_cmm: CryptoMaterialsManager mpl_cmm: 'ICryptographicMaterialsManager' + def _is_using_native_cmm(self): + return hasattr(self, "native_cmm") and not hasattr(self, "mpl_cmm") + def __init__( self, cmm: 'CryptoMaterialsManager | ICryptographicMaterialsManager' @@ -56,24 +63,24 @@ def get_encryption_materials( request: EncryptionMaterialsRequest ) -> EncryptionMaterialsHandler: ''' - Returns an EncryptionMaterialsHandler based on the configured CMM. + Returns an EncryptionMaterialsHandler for the configured CMM. ''' - if (hasattr(self, "native_cmm") and not hasattr(self, "mpl_cmm")): + if (self._is_using_native_cmm()): return EncryptionMaterialsHandler(self.native_cmm.get_encryption_materials(request)) else: - input: GetEncryptionMaterialsInput = CMMHandler._create_mpl_get_encryption_materials_input_from_request(request) - print(f"get_encryption_materials {input=}") - output: GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(input) - print(f"get_encryption_materials {output=}") - return EncryptionMaterialsHandler(output.encryption_materials) + try: + input: GetEncryptionMaterialsInput = CMMHandler._create_mpl_get_encryption_materials_input_from_request(request) + output: GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(input) + return EncryptionMaterialsHandler(output.encryption_materials) + except AwsCryptographicMaterialProvidersException as e: + # Wrap MPL error into the ESDK error type + # so customers only have to catch ESDK error types. + raise AWSEncryptionSDKClientError(e) @staticmethod def _create_mpl_get_encryption_materials_input_from_request( request: EncryptionMaterialsRequest ) -> 'GetEncryptionMaterialsInput': - print(f"_create_mpl_get_encryption_materials_input_from_request {request=}") - print(f"{CMMHandler._map_native_commitment_policy_to_mpl_commitment_policy(request.commitment_policy)=}") - print(f"_create_mpl_get_encryption_materials_input_from_request {request.encryption_context=}") output: GetEncryptionMaterialsInput = GetEncryptionMaterialsInput( encryption_context=request.encryption_context, commitment_policy=CMMHandler._map_native_commitment_policy_to_mpl_commitment_policy(request.commitment_policy), @@ -82,7 +89,6 @@ def _create_mpl_get_encryption_materials_input_from_request( # algorithm_suite_id=request.algorithm.algorithm_id, max_plaintext_length=request.plaintext_length, ) - print(f"_create_mpl_get_encryption_materials_input_from_request {output=}") return output @staticmethod @@ -103,16 +109,19 @@ def decrypt_materials( request: DecryptionMaterialsRequest ) -> DecryptionMaterialsHandler: ''' - Returns a DecryptionMaterialsHandler based on the configured CMM. + Returns a DecryptionMaterialsHandler for the configured CMM. ''' - print(f"decrypt_materials {request=}") - if (hasattr(self, "native_cmm") and not hasattr(self, "mpl_cmm")): + if (self._is_using_native_cmm()): return DecryptionMaterialsHandler(self.native_cmm.decrypt_materials(request)) else: - input: 'DecryptMaterialsInput' = CMMHandler._create_mpl_decrypt_materials_input_from_request(request) - output: 'DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(input) - print(f"decrypt_materials {output.decryption_materials.verification_key=}") - return DecryptionMaterialsHandler(output.decryption_materials) + try: + input: 'DecryptMaterialsInput' = CMMHandler._create_mpl_decrypt_materials_input_from_request(request) + output: 'DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(input) + return DecryptionMaterialsHandler(output.decryption_materials) + except AwsCryptographicMaterialProvidersException as e: + # Wrap MPL error into the ESDK error type + # so customers only have to catch ESDK error types. + raise AWSEncryptionSDKClientError(e) @staticmethod def _native_algorithm_id_to_mpl_algorithm_id(native_algorithm_id: str) -> AlgorithmSuiteIdESDK: diff --git a/src/aws_encryption_sdk/materials_handlers.py b/src/aws_encryption_sdk/materials_handlers.py index bf3073ad3..1f34eba03 100644 --- a/src/aws_encryption_sdk/materials_handlers.py +++ b/src/aws_encryption_sdk/materials_handlers.py @@ -1,6 +1,6 @@ # These dependencies are only loaded if you install the MPL. try: - from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.models import ( + from aws_cryptographic_materialproviders.mpl.models import ( DecryptionMaterials as MPL_DecryptionMaterials, EncryptionMaterials as MPL_EncryptionMaterials, EncryptedDataKey as MPL_EncryptedDataKey, @@ -25,15 +25,19 @@ Signer ) +def _mpl_algorithm_id_to_native_algorithm_id(mpl_algorithm_id: str): + # MPL algorithm suite ID == hex(native algorithm suite ID) + return int(mpl_algorithm_id, 16) + class EncryptionMaterialsHandler: + ''' + In instances where encryption materials may be provided by either + the native `aws_encryption_sdk.materials_managers.Native_EncryptionMaterials` + or the MPL's `aws_cryptographic_materialproviders.mpl.models` + ''' native_materials: Native_EncryptionMaterials mpl_materials: 'MPL_EncryptionMaterials' - @staticmethod - def _mpl_algorithm_id_to_native_algorithm_id(mpl_algorithm_id: str): - # MPL algorithm suite ID == "ALG_" + native algorithm suite ID. - return int(mpl_algorithm_id, 16) - def __init__( self, materials: 'Native_EncryptionMaterials | MPL_EncryptionMaterials' @@ -49,9 +53,8 @@ def algorithm(self) -> Algorithm: if hasattr(self, "native_materials"): return self.native_materials.algorithm else: - print(f"algorithm {self.mpl_materials.algorithm_suite.id.value=}") return AlgorithmSuite.get_by_id( - EncryptionMaterialsHandler._mpl_algorithm_id_to_native_algorithm_id( + _mpl_algorithm_id_to_native_algorithm_id( self.mpl_materials.algorithm_suite.id.value ) ) @@ -83,10 +86,12 @@ def data_encryption_key(self) -> DataKey: if hasattr(self, "native_materials"): return self.native_materials.data_encryption_key else: - # TODO-MPL This impl is probably wrong + # TODO-MPL This impl is probably wrong, but works for for now + # If this works for all features, great! Remove this comment before launch. + # Otherwise, fix the implementation. mpl_dek = self.mpl_materials.plaintext_data_key return DataKey( - # key_provider=None, # No MasterKeyInfo object for plaintext data key + # key_provider is unused, but the return type is DataKey key_provider=MasterKeyInfo( provider_id="", key_info=b'' @@ -100,20 +105,8 @@ def signing_key(self) -> bytes: if hasattr(self, "native_materials"): return self.native_materials.signing_key else: - print(f"sign {self.mpl_materials.signing_key=}") return self.mpl_materials.signing_key - # if self.mpl_materials.signing_key is None: - # return Signer.from_key_bytes( - # algorithm=AlgorithmSuite.get_by_id(self.mpl_materials.algorithm_suite.id.value), - # bytes=self.mpl_materials.signing_key - # ) - def get_required_encryption_context_keys(self) -> list[str]: - if hasattr(self, "native_materials"): - return [] - else: - return self.mpl_materials.required_encryption_context_keys - class DecryptionMaterialsHandler: native_materials: Native_DecryptionMaterials mpl_materials: 'MPL_DecryptionMaterials' @@ -128,19 +121,15 @@ def __init__( self.mpl_materials = materials else: raise ValueError(f"Invalid DecryptionMaterials passed to DecryptionMaterialsHandler: {materials=}") - - def get_encryption_context(self) -> dict[str, str]: - if hasattr(self, "native_materials"): - return {} # TODO-MPL This impl is probably wrong - else: - return self.mpl_materials.encryption_context @property def data_key(self) -> DataKey: if hasattr(self, "native_materials"): return self.native_materials.data_key else: - # TODO-MPL This impl is probably wrong + # TODO-MPL This impl is probably wrong, but works for for now + # If this works for all features, great! Remove this comment before launch. + # Otherwise, fix the implementation. return DataKey( key_provider=MasterKeyInfo( provider_id="", @@ -155,11 +144,4 @@ def verification_key(self) -> bytes: if hasattr(self, "native_materials"): return self.native_materials.verification_key else: - print(f"ver {self.mpl_materials.verification_key=}") - return self.mpl_materials.verification_key - - def get_required_encryption_context_keys(self) -> list[str]: - if hasattr(self, "native_materials"): - return [] - else: - return self.mpl_materials.required_encryption_context_keys \ No newline at end of file + return self.mpl_materials.verification_key \ No newline at end of file diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 176b92334..e6cf00635 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -68,12 +68,12 @@ from aws_encryption_sdk.structures import MessageHeader try: import aws_cryptographic_materialproviders - from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.client import AwsCryptographicMaterialProviders - from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.config import MaterialProvidersConfig - from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.models import ( + from aws_cryptographic_materialproviders.mpl.client import AwsCryptographicMaterialProviders + from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig + from aws_cryptographic_materialproviders.mpl.models import ( CreateDefaultCryptographicMaterialsManagerInput ) - from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.references import ( + from aws_cryptographic_materialproviders.mpl.references import ( IKeyring, ) from aws_encryption_sdk.cmm_handler import CMMHandler @@ -411,7 +411,7 @@ class EncryptorConfig(_ClientConfig): :param keyring: `IKeyring` from the aws_cryptographic_materialproviders library which handles encryption and decryption :type keyring: - aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.references.IKeyring + aws_cryptographic_materialproviders.mpl.references.IKeyring :param int source_length: Length of source data (optional) .. note:: @@ -466,7 +466,7 @@ class StreamEncryptor(_EncryptionStream): # pylint: disable=too-many-instance-a :param keyring: `IKeyring` from the aws_cryptographic_materialproviders library which handles encryption and decryption :type keyring: - aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.references.IKeyring + aws_cryptographic_materialproviders.mpl.references.IKeyring :param int source_length: Length of source data (optional) .. note:: @@ -810,7 +810,7 @@ class DecryptorConfig(_ClientConfig): :param keyring: `IKeyring` from the aws_cryptographic_materialproviders library which handles encryption and decryption :type keyring: - aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.references.IKeyring + aws_cryptographic_materialproviders.mpl.references.IKeyring :param int source_length: Length of source data (optional) .. note:: @@ -850,7 +850,7 @@ class StreamDecryptor(_EncryptionStream): # pylint: disable=too-many-instance-a :param keyring: `IKeyring` from the aws_cryptographic_materialproviders library which handles encryption and decryption :type keyring: - aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.references.IKeyring + aws_cryptographic_materialproviders.mpl.references.IKeyring :param int source_length: Length of source data (optional) .. note:: @@ -1082,7 +1082,7 @@ def close(self): """Closes out the stream.""" _LOGGER.debug("Closing stream") if not hasattr(self, "footer"): - raise SerializationError("Footer not read") + raise SerializationError("Footer not read, message may be corrupted or data key may be incorrect") super(StreamDecryptor, self).close() diff --git a/test/unit/test_streaming_client_stream_decryptor.py b/test/unit/test_streaming_client_stream_decryptor.py index 157755094..94b22b092 100644 --- a/test/unit/test_streaming_client_stream_decryptor.py +++ b/test/unit/test_streaming_client_stream_decryptor.py @@ -767,4 +767,4 @@ def test_close_no_footer(self, mock_close): ) with pytest.raises(SerializationError) as excinfo: test_decryptor.close() - excinfo.match("Footer not read") + excinfo.match("Footer not read, message may be corrupted or data key may be incorrect") From 5b5aa07af75e37d2ad42e7ba850345097d55fcc0 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 2 Feb 2024 15:04:57 -0800 Subject: [PATCH 005/376] changes, cleanup --- src/aws_encryption_sdk/streaming_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index e6cf00635..55be3b917 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -170,7 +170,7 @@ def _exactly_one_arg_is_not_None(*args): try: assert isinstance(self.keyring, IKeyring) except AssertionError as e: - raise ValueError(f"Argument provided to keyring MUST be a {IKeyring}. Found {keyring.__class__.__name__=}") + raise ValueError(f"Argument provided to keyring MUST be a {IKeyring}. Found {self.keyring.__class__.__name__=}") mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( config=MaterialProvidersConfig() From 03e19caff1eaa264873fab500beb8e62c890b583 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 2 Feb 2024 15:19:08 -0800 Subject: [PATCH 006/376] flake8 --- src/aws_encryption_sdk/cmm_handler.py | 31 ++++++---- src/aws_encryption_sdk/materials_handlers.py | 27 +++++---- src/aws_encryption_sdk/streaming_client.py | 62 +++++++++++--------- 3 files changed, 68 insertions(+), 52 deletions(-) diff --git a/src/aws_encryption_sdk/cmm_handler.py b/src/aws_encryption_sdk/cmm_handler.py index d634dd571..f7f95b0c9 100644 --- a/src/aws_encryption_sdk/cmm_handler.py +++ b/src/aws_encryption_sdk/cmm_handler.py @@ -15,7 +15,7 @@ CommitmentPolicyESDK, AlgorithmSuiteIdESDK, ) -except ImportError as e: +except ImportError: pass from aws_encryption_sdk.exceptions import ( @@ -39,6 +39,7 @@ CommitmentPolicy, ) + # TODO-MPL Should this implement interface..? seems like yes since it implements all of interface methods class CMMHandler(CryptoMaterialsManager): native_cmm: CryptoMaterialsManager @@ -57,7 +58,7 @@ def __init__( self.mpl_cmm = cmm else: raise ValueError(f"Invalid CMM passed to CMMHander: {cmm=}") - + def get_encryption_materials( self, request: EncryptionMaterialsRequest @@ -69,28 +70,32 @@ def get_encryption_materials( return EncryptionMaterialsHandler(self.native_cmm.get_encryption_materials(request)) else: try: - input: GetEncryptionMaterialsInput = CMMHandler._create_mpl_get_encryption_materials_input_from_request(request) + input: GetEncryptionMaterialsInput = CMMHandler._create_mpl_get_encryption_materials_input_from_request( + request + ) output: GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(input) return EncryptionMaterialsHandler(output.encryption_materials) except AwsCryptographicMaterialProvidersException as e: # Wrap MPL error into the ESDK error type # so customers only have to catch ESDK error types. raise AWSEncryptionSDKClientError(e) - + @staticmethod def _create_mpl_get_encryption_materials_input_from_request( request: EncryptionMaterialsRequest ) -> 'GetEncryptionMaterialsInput': output: GetEncryptionMaterialsInput = GetEncryptionMaterialsInput( encryption_context=request.encryption_context, - commitment_policy=CMMHandler._map_native_commitment_policy_to_mpl_commitment_policy(request.commitment_policy), + commitment_policy=CMMHandler._map_native_commitment_policy_to_mpl_commitment_policy( + request.commitment_policy + ), # TODO double check this # optional... maybe this needs to be kwargs?? # algorithm_suite_id=request.algorithm.algorithm_id, max_plaintext_length=request.plaintext_length, ) return output - + @staticmethod def _map_native_commitment_policy_to_mpl_commitment_policy( native_commitment_policy: CommitmentPolicy @@ -103,7 +108,7 @@ def _map_native_commitment_policy_to_mpl_commitment_policy( return CommitmentPolicyESDK(value="REQUIRE_ENCRYPT_REQUIRE_DECRYPT") else: raise ValueError(f"Invalid {native_commitment_policy=}") - + def decrypt_materials( self, request: DecryptionMaterialsRequest @@ -122,12 +127,12 @@ def decrypt_materials( # Wrap MPL error into the ESDK error type # so customers only have to catch ESDK error types. raise AWSEncryptionSDKClientError(e) - + @staticmethod def _native_algorithm_id_to_mpl_algorithm_id(native_algorithm_id: str) -> AlgorithmSuiteIdESDK: # MPL algorithm suite ID = hexstr(native_algorithm_id) padded to 4 digits post-`x`. return AlgorithmSuiteIdESDK(f"{native_algorithm_id:#0{6}x}") - + @staticmethod def _create_mpl_decrypt_materials_input_from_request( request: DecryptionMaterialsRequest @@ -139,8 +144,12 @@ def _create_mpl_decrypt_materials_input_from_request( ciphertext=key_blob.encrypted_data_key, ) for key_blob in key_blob_list] output: DecryptMaterialsInput = DecryptMaterialsInput( - algorithm_suite_id=CMMHandler._native_algorithm_id_to_mpl_algorithm_id(request.algorithm.algorithm_id), - commitment_policy=CMMHandler._map_native_commitment_policy_to_mpl_commitment_policy(request.commitment_policy), + algorithm_suite_id=CMMHandler._native_algorithm_id_to_mpl_algorithm_id( + request.algorithm.algorithm_id + ), + commitment_policy=CMMHandler._map_native_commitment_policy_to_mpl_commitment_policy( + request.commitment_policy + ), encrypted_data_keys=list_edks, encryption_context=request.encryption_context, ) diff --git a/src/aws_encryption_sdk/materials_handlers.py b/src/aws_encryption_sdk/materials_handlers.py index 1f34eba03..a03138e78 100644 --- a/src/aws_encryption_sdk/materials_handlers.py +++ b/src/aws_encryption_sdk/materials_handlers.py @@ -5,7 +5,7 @@ EncryptionMaterials as MPL_EncryptionMaterials, EncryptedDataKey as MPL_EncryptedDataKey, ) -except ImportError as e: +except ImportError: pass from aws_encryption_sdk.materials_managers import ( @@ -21,14 +21,13 @@ EncryptedDataKey as Native_EncryptedDataKey, MasterKeyInfo, ) -from aws_encryption_sdk.internal.crypto.authentication import ( - Signer -) + def _mpl_algorithm_id_to_native_algorithm_id(mpl_algorithm_id: str): # MPL algorithm suite ID == hex(native algorithm suite ID) return int(mpl_algorithm_id, 16) + class EncryptionMaterialsHandler: ''' In instances where encryption materials may be provided by either @@ -48,6 +47,7 @@ def __init__( self.mpl_materials = materials else: raise ValueError(f"Invalid EncryptionMaterials passed to EncryptionMaterialsHandler: {materials=}") + @property def algorithm(self) -> Algorithm: if hasattr(self, "native_materials"): @@ -58,14 +58,14 @@ def algorithm(self) -> Algorithm: self.mpl_materials.algorithm_suite.id.value ) ) - + @property def encryption_context(self) -> dict[str, str]: if hasattr(self, "native_materials"): return self.native_materials.encryption_context else: return self.mpl_materials.encryption_context - + @property def encrypted_data_keys(self) -> list[Native_EncryptedDataKey]: if hasattr(self, "native_materials"): @@ -80,7 +80,7 @@ def encrypted_data_keys(self) -> list[Native_EncryptedDataKey]: encrypted_data_key=mpl_edk.ciphertext, ) for mpl_edk in mpl_edk_list} return key_blob_list - + @property def data_encryption_key(self) -> DataKey: if hasattr(self, "native_materials"): @@ -97,16 +97,17 @@ def data_encryption_key(self) -> DataKey: key_info=b'' ), data_key=mpl_dek, - encrypted_data_key=b'', # No encrypted DEK + encrypted_data_key=b'', # No encrypted DEK ) - + @property def signing_key(self) -> bytes: if hasattr(self, "native_materials"): return self.native_materials.signing_key else: return self.mpl_materials.signing_key - + + class DecryptionMaterialsHandler: native_materials: Native_DecryptionMaterials mpl_materials: 'MPL_DecryptionMaterials' @@ -121,7 +122,7 @@ def __init__( self.mpl_materials = materials else: raise ValueError(f"Invalid DecryptionMaterials passed to DecryptionMaterialsHandler: {materials=}") - + @property def data_key(self) -> DataKey: if hasattr(self, "native_materials"): @@ -138,10 +139,10 @@ def data_key(self) -> DataKey: data_key=self.mpl_materials.plaintext_data_key, encrypted_data_key=b'', ) - + @property def verification_key(self) -> bytes: if hasattr(self, "native_materials"): return self.native_materials.verification_key else: - return self.mpl_materials.verification_key \ No newline at end of file + return self.mpl_materials.verification_key diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 55be3b917..661b3fa21 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -67,7 +67,6 @@ from aws_encryption_sdk.materials_managers.default import DefaultCryptoMaterialsManager from aws_encryption_sdk.structures import MessageHeader try: - import aws_cryptographic_materialproviders from aws_cryptographic_materialproviders.mpl.client import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig from aws_cryptographic_materialproviders.mpl.models import ( @@ -77,14 +76,33 @@ IKeyring, ) from aws_encryption_sdk.cmm_handler import CMMHandler - + _has_mpl = True -except ImportError as e: +except ImportError: _has_mpl = False _LOGGER = logging.getLogger(__name__) +def _exactly_one_arg_is_not_None(*args): + ''' + Private helper function. + Returns `True` if exactly one item in the list is not `None`. + Returns `False` otherwise. + ''' + # Have not found any `not None` + found_one = False + for arg in args: + if arg is not None: + if found_one is False: + # Have not already found a `not None`, found a `not None` => only one `not None` (so far) + found_one = True + else: + # Already found a `not None`, found another `not None` => not exactly one `not None` + return False + return found_one + + @attr.s(hash=True) # pylint: disable=too-many-instance-attributes @six.add_metaclass(abc.ABCMeta) class _ClientConfig(object): # pylint: disable=too-many-instance-attributes @@ -140,42 +158,30 @@ class _ClientConfig(object): # pylint: disable=too-many-instance-attributes ) # DEPRECATED: Value is no longer configurable here. Parameter left here to avoid breaking consumers. def _has_mpl_attrs_post_init(self): - - def _exactly_one_arg_is_not_None(*args): - ''' - Private helper function. - Returns `True` if exactly one item in the list is not `None`. - Returns `False` otherwise. - ''' - # Have not found any `not None` - found_one = False - for arg in args: - if arg is not None: - if found_one == False: - # Have not already found a `not None`, found a `not None` => only one `not None` (so far) - found_one = True - else: - # Already found a `not None`, found another `not None` => not exactly one `not None` - return False - return found_one - if not _exactly_one_arg_is_not_None(self.materials_manager, self.key_provider, self.keyring): raise TypeError("Exactly one of keyring, materials_manager, or key_provider must be provided") if self.materials_manager is None: if self.key_provider is not None: - # No CMM, provided (legacy) native `key_provider` => create (legacy) native DefaultCryptoMaterialsManager - self.materials_manager = DefaultCryptoMaterialsManager(master_key_provider=self.key_provider) + # No CMM, provided legacy native `key_provider` => create legacy native DefaultCryptoMaterialsManager + self.materials_manager = DefaultCryptoMaterialsManager( + master_key_provider=self.key_provider + ) elif self.keyring is not None: # No CMM, provided MPL keyring => create MPL's DefaultCryptographicMaterialsManager try: assert isinstance(self.keyring, IKeyring) - except AssertionError as e: - raise ValueError(f"Argument provided to keyring MUST be a {IKeyring}. Found {self.keyring.__class__.__name__=}") - + except AssertionError: + raise ValueError(f"Argument provided to keyring MUST be a {IKeyring}. \ + Found {self.keyring.__class__.__name__=}") + mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( config=MaterialProvidersConfig() ) - cmm = mat_prov.create_default_cryptographic_materials_manager(CreateDefaultCryptographicMaterialsManagerInput(keyring=self.keyring)) + cmm = mat_prov.create_default_cryptographic_materials_manager( + CreateDefaultCryptographicMaterialsManagerInput( + keyring=self.keyring + ) + ) cmm_handler: CryptoMaterialsManager = CMMHandler(cmm) self.materials_manager = cmm_handler From b5d33275a462e9311b08359a722496d0737a81be Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 2 Feb 2024 15:41:21 -0800 Subject: [PATCH 007/376] flake8 --- src/aws_encryption_sdk/cmm_handler.py | 25 +++++++-- src/aws_encryption_sdk/materials_handlers.py | 56 ++++++++++++++++++-- 2 files changed, 72 insertions(+), 9 deletions(-) diff --git a/src/aws_encryption_sdk/cmm_handler.py b/src/aws_encryption_sdk/cmm_handler.py index f7f95b0c9..bb60a4fa1 100644 --- a/src/aws_encryption_sdk/cmm_handler.py +++ b/src/aws_encryption_sdk/cmm_handler.py @@ -1,3 +1,5 @@ +"""Retrieves encryption/decryption materials from an underlying materials provider.""" + # These dependencies are only loaded if you install the MPL. try: from aws_cryptographic_materialproviders.mpl.errors import ( @@ -42,6 +44,15 @@ # TODO-MPL Should this implement interface..? seems like yes since it implements all of interface methods class CMMHandler(CryptoMaterialsManager): + """ + In instances where encryption materials may be provided by either + an implementation of the native + `aws_encryption_sdk.materials_managers.base.CryptoMaterialsManager` + or an implementation of the MPL's + `aws_cryptographic_materialproviders.mpl.references.ICryptographicMaterialsManager`, + this provides the correct materials based on the underlying materials manager. + """ + native_cmm: CryptoMaterialsManager mpl_cmm: 'ICryptographicMaterialsManager' @@ -57,15 +68,17 @@ def __init__( elif isinstance(cmm, ICryptographicMaterialsManager): self.mpl_cmm = cmm else: - raise ValueError(f"Invalid CMM passed to CMMHander: {cmm=}") + raise ValueError(f"Invalid CMM passed to CMMHandler: {cmm=}") def get_encryption_materials( self, request: EncryptionMaterialsRequest ) -> EncryptionMaterialsHandler: - ''' + """ Returns an EncryptionMaterialsHandler for the configured CMM. - ''' + :param request: Request for encryption materials + """ + if (self._is_using_native_cmm()): return EncryptionMaterialsHandler(self.native_cmm.get_encryption_materials(request)) else: @@ -113,9 +126,11 @@ def decrypt_materials( self, request: DecryptionMaterialsRequest ) -> DecryptionMaterialsHandler: - ''' + """ Returns a DecryptionMaterialsHandler for the configured CMM. - ''' + :param request: Request for decryption materials + """ + if (self._is_using_native_cmm()): return DecryptionMaterialsHandler(self.native_cmm.decrypt_materials(request)) else: diff --git a/src/aws_encryption_sdk/materials_handlers.py b/src/aws_encryption_sdk/materials_handlers.py index a03138e78..180dec3bb 100644 --- a/src/aws_encryption_sdk/materials_handlers.py +++ b/src/aws_encryption_sdk/materials_handlers.py @@ -1,3 +1,4 @@ +"""Provides encryption/decryption materials from an underlying materials provider.""" # These dependencies are only loaded if you install the MPL. try: from aws_cryptographic_materialproviders.mpl.models import ( @@ -29,11 +30,13 @@ def _mpl_algorithm_id_to_native_algorithm_id(mpl_algorithm_id: str): class EncryptionMaterialsHandler: - ''' + """ In instances where encryption materials may be provided by either - the native `aws_encryption_sdk.materials_managers.Native_EncryptionMaterials` - or the MPL's `aws_cryptographic_materialproviders.mpl.models` - ''' + the native `aws_encryption_sdk.materials_managers.EncryptionMaterials` + or the MPL's `aws_cryptographic_materialproviders.mpl.models.EncryptionMaterials`, + this provides the correct materials based on the configured materials provider. + """ + native_materials: Native_EncryptionMaterials mpl_materials: 'MPL_EncryptionMaterials' @@ -41,6 +44,11 @@ def __init__( self, materials: 'Native_EncryptionMaterials | MPL_EncryptionMaterials' ): + """ + Create EncryptionMaterialsHandler. + :param materials: Underlying encryption materials + """ + if isinstance(materials, Native_EncryptionMaterials): self.native_materials = materials elif isinstance(materials, MPL_EncryptionMaterials): @@ -50,6 +58,10 @@ def __init__( @property def algorithm(self) -> Algorithm: + """ + Materials' native Algorithm. + """ + if hasattr(self, "native_materials"): return self.native_materials.algorithm else: @@ -61,6 +73,10 @@ def algorithm(self) -> Algorithm: @property def encryption_context(self) -> dict[str, str]: + """ + Materials' encryption context. + """ + if hasattr(self, "native_materials"): return self.native_materials.encryption_context else: @@ -68,6 +84,10 @@ def encryption_context(self) -> dict[str, str]: @property def encrypted_data_keys(self) -> list[Native_EncryptedDataKey]: + """ + Materials' encrypted data keys. + """ + if hasattr(self, "native_materials"): return self.native_materials.encrypted_data_keys else: @@ -83,6 +103,10 @@ def encrypted_data_keys(self) -> list[Native_EncryptedDataKey]: @property def data_encryption_key(self) -> DataKey: + """ + Materials' data encryption key. + """ + if hasattr(self, "native_materials"): return self.native_materials.data_encryption_key else: @@ -102,6 +126,10 @@ def data_encryption_key(self) -> DataKey: @property def signing_key(self) -> bytes: + """ + Materials' signing key. + """ + if hasattr(self, "native_materials"): return self.native_materials.signing_key else: @@ -109,6 +137,13 @@ def signing_key(self) -> bytes: class DecryptionMaterialsHandler: + """ + In instances where decryption materials may be provided by either + the native `aws_encryption_sdk.materials_managers.DecryptionMaterials` + or the MPL's `aws_cryptographic_materialproviders.mpl.models.DecryptionMaterials`, + this provides the correct materials based on the configured materials provider. + """ + native_materials: Native_DecryptionMaterials mpl_materials: 'MPL_DecryptionMaterials' @@ -116,6 +151,11 @@ def __init__( self, materials: 'Native_DecryptionMaterials | MPL_DecryptionMaterials' ): + """ + Create DecryptionMaterialsHandler. + :param materials: Underlying decryption materials + """ + if isinstance(materials, Native_DecryptionMaterials): self.native_materials = materials elif isinstance(materials, MPL_DecryptionMaterials): @@ -125,6 +165,10 @@ def __init__( @property def data_key(self) -> DataKey: + """ + Materials' data key. + """ + if hasattr(self, "native_materials"): return self.native_materials.data_key else: @@ -142,6 +186,10 @@ def data_key(self) -> DataKey: @property def verification_key(self) -> bytes: + """ + Materials' verification key. + """ + if hasattr(self, "native_materials"): return self.native_materials.verification_key else: From b13cd191a8874dc0091e844c4cb0789d81591764 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 2 Feb 2024 15:45:55 -0800 Subject: [PATCH 008/376] flake8 --- src/aws_encryption_sdk/cmm_handler.py | 7 ++-- src/aws_encryption_sdk/materials_handlers.py | 37 ++++---------------- src/aws_encryption_sdk/streaming_client.py | 4 +-- 3 files changed, 14 insertions(+), 34 deletions(-) diff --git a/src/aws_encryption_sdk/cmm_handler.py b/src/aws_encryption_sdk/cmm_handler.py index bb60a4fa1..17d59792a 100644 --- a/src/aws_encryption_sdk/cmm_handler.py +++ b/src/aws_encryption_sdk/cmm_handler.py @@ -63,6 +63,11 @@ def __init__( self, cmm: 'CryptoMaterialsManager | ICryptographicMaterialsManager' ): + """ + Create DecryptionMaterialsHandler. + :param cmm: Underlying cryptographic materials manager + """ + if isinstance(cmm, CryptoMaterialsManager): self.native_cmm = cmm elif isinstance(cmm, ICryptographicMaterialsManager): @@ -78,7 +83,6 @@ def get_encryption_materials( Returns an EncryptionMaterialsHandler for the configured CMM. :param request: Request for encryption materials """ - if (self._is_using_native_cmm()): return EncryptionMaterialsHandler(self.native_cmm.get_encryption_materials(request)) else: @@ -130,7 +134,6 @@ def decrypt_materials( Returns a DecryptionMaterialsHandler for the configured CMM. :param request: Request for decryption materials """ - if (self._is_using_native_cmm()): return DecryptionMaterialsHandler(self.native_cmm.decrypt_materials(request)) else: diff --git a/src/aws_encryption_sdk/materials_handlers.py b/src/aws_encryption_sdk/materials_handlers.py index 180dec3bb..d54e4517b 100644 --- a/src/aws_encryption_sdk/materials_handlers.py +++ b/src/aws_encryption_sdk/materials_handlers.py @@ -48,7 +48,6 @@ def __init__( Create EncryptionMaterialsHandler. :param materials: Underlying encryption materials """ - if isinstance(materials, Native_EncryptionMaterials): self.native_materials = materials elif isinstance(materials, MPL_EncryptionMaterials): @@ -58,10 +57,7 @@ def __init__( @property def algorithm(self) -> Algorithm: - """ - Materials' native Algorithm. - """ - + """Materials' native Algorithm.""" if hasattr(self, "native_materials"): return self.native_materials.algorithm else: @@ -73,10 +69,7 @@ def algorithm(self) -> Algorithm: @property def encryption_context(self) -> dict[str, str]: - """ - Materials' encryption context. - """ - + """Materials' encryption context.""" if hasattr(self, "native_materials"): return self.native_materials.encryption_context else: @@ -84,10 +77,7 @@ def encryption_context(self) -> dict[str, str]: @property def encrypted_data_keys(self) -> list[Native_EncryptedDataKey]: - """ - Materials' encrypted data keys. - """ - + """Materials' encrypted data keys.""" if hasattr(self, "native_materials"): return self.native_materials.encrypted_data_keys else: @@ -103,10 +93,7 @@ def encrypted_data_keys(self) -> list[Native_EncryptedDataKey]: @property def data_encryption_key(self) -> DataKey: - """ - Materials' data encryption key. - """ - + """Materials' data encryption key.""" if hasattr(self, "native_materials"): return self.native_materials.data_encryption_key else: @@ -126,10 +113,7 @@ def data_encryption_key(self) -> DataKey: @property def signing_key(self) -> bytes: - """ - Materials' signing key. - """ - + """Materials' signing key.""" if hasattr(self, "native_materials"): return self.native_materials.signing_key else: @@ -155,7 +139,6 @@ def __init__( Create DecryptionMaterialsHandler. :param materials: Underlying decryption materials """ - if isinstance(materials, Native_DecryptionMaterials): self.native_materials = materials elif isinstance(materials, MPL_DecryptionMaterials): @@ -165,10 +148,7 @@ def __init__( @property def data_key(self) -> DataKey: - """ - Materials' data key. - """ - + """Materials' data key.""" if hasattr(self, "native_materials"): return self.native_materials.data_key else: @@ -186,10 +166,7 @@ def data_key(self) -> DataKey: @property def verification_key(self) -> bytes: - """ - Materials' verification key. - """ - + """Materials' verification key.""" if hasattr(self, "native_materials"): return self.native_materials.verification_key else: diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 661b3fa21..ec19b6dd5 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -85,11 +85,11 @@ def _exactly_one_arg_is_not_None(*args): - ''' + """ Private helper function. Returns `True` if exactly one item in the list is not `None`. Returns `False` otherwise. - ''' + """ # Have not found any `not None` found_one = False for arg in args: From 51065cb186bb3e649117a3a9871b2b5943066a52 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 2 Feb 2024 15:48:34 -0800 Subject: [PATCH 009/376] flake8 --- setup.py | 4 +++- src/aws_encryption_sdk/cmm_handler.py | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index c4c277096..353781800 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,9 @@ def get_requirements(): install_requires=get_requirements(), # TODO: Point at MPL main branch once Python MPL is merged into main. extras_require={ - "MPL": ["aws-cryptographic-material-providers @ git+https://github.com/aws/aws-cryptographic-material-providers-library.git@lucmcdon/python-mpl#subdirectory=AwsCryptographicMaterialProviders/runtimes/python"], + "MPL": ["aws-cryptographic-material-providers @\ + git+https://github.com/aws/aws-cryptographic-material-providers-library.git@\ + lucmcdon/python-mpl#subdirectory=AwsCryptographicMaterialProviders/runtimes/python"], }, classifiers=[ "Development Status :: 5 - Production/Stable", diff --git a/src/aws_encryption_sdk/cmm_handler.py b/src/aws_encryption_sdk/cmm_handler.py index 17d59792a..887d9d79e 100644 --- a/src/aws_encryption_sdk/cmm_handler.py +++ b/src/aws_encryption_sdk/cmm_handler.py @@ -67,7 +67,6 @@ def __init__( Create DecryptionMaterialsHandler. :param cmm: Underlying cryptographic materials manager """ - if isinstance(cmm, CryptoMaterialsManager): self.native_cmm = cmm elif isinstance(cmm, ICryptographicMaterialsManager): From fc4d254d7f7601d86fc9954ed69d0817869a43e2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 2 Feb 2024 15:51:15 -0800 Subject: [PATCH 010/376] flake8 --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 353781800..4cd8027cd 100644 --- a/setup.py +++ b/setup.py @@ -41,9 +41,9 @@ def get_requirements(): install_requires=get_requirements(), # TODO: Point at MPL main branch once Python MPL is merged into main. extras_require={ - "MPL": ["aws-cryptographic-material-providers @\ - git+https://github.com/aws/aws-cryptographic-material-providers-library.git@\ - lucmcdon/python-mpl#subdirectory=AwsCryptographicMaterialProviders/runtimes/python"], + "MPL": ["aws-cryptographic-material-providers @" \ + "git+https://github.com/aws/aws-cryptographic-material-providers-library.git@" \ + "lucmcdon/python-mpl#subdirectory=AwsCryptographicMaterialProviders/runtimes/python"], }, classifiers=[ "Development Status :: 5 - Production/Stable", From a8e52d310c777905326fc98cbf667217116de303 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 5 Feb 2024 13:23:21 -0800 Subject: [PATCH 011/376] fix pem/der --- .../internal/crypto/authentication.py | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/src/aws_encryption_sdk/internal/crypto/authentication.py b/src/aws_encryption_sdk/internal/crypto/authentication.py index b9692eb16..8c1b9af31 100644 --- a/src/aws_encryption_sdk/internal/crypto/authentication.py +++ b/src/aws_encryption_sdk/internal/crypto/authentication.py @@ -68,25 +68,31 @@ class Signer(_PrehashingAuthenticator): """ @classmethod - def from_key_bytes(cls, algorithm, key_bytes): + def from_key_bytes(cls, algorithm, key_bytes, encoding=serialization.Encoding.DER): """Builds a `Signer` from an algorithm suite and a raw signing key. :param algorithm: Algorithm on which to base signer :type algorithm: aws_encryption_sdk.identifiers.Algorithm :param bytes key_bytes: Raw signing key + :param encoding: Encoding used for key bytes + :type encoding: cryptography.hazmat.primitives.serialization.encoding :rtype: aws_encryption_sdk.internal.crypto.Signer """ - # key = serialization.load_der_private_key(data=key_bytes, password=None, backend=default_backend()) - key = serialization.load_pem_private_key(data=key_bytes, password=None, backend=default_backend()) + if encoding == serialization.Encoding.DER: + key = serialization.load_der_private_key(data=key_bytes, password=None, backend=default_backend()) + elif encoding == serialization.Encoding.PEM: + key = serialization.load_pem_private_key(data=key_bytes, password=None, backend=default_backend()) + else: + raise ValueError("Unsupported signing key encoding: {}".format(encoding)) return cls(algorithm, key) - def key_bytes(self): + def key_bytes(self, encoding=serialization.Encoding.DER): """Returns the raw signing key. :rtype: bytes """ return self.key.private_bytes( - encoding=serialization.Encoding.DER, + encoding=encoding, format=serialization.PrivateFormat.PKCS8, encryption_algorithm=serialization.NoEncryption(), ) @@ -149,19 +155,27 @@ def from_encoded_point(cls, algorithm, encoded_point): ) @classmethod - def from_key_bytes(cls, algorithm, key_bytes): + def from_key_bytes(cls, algorithm, key_bytes, encoding=serialization.Encoding.DER): """Creates a `Verifier` object based on the supplied algorithm and raw verification key. :param algorithm: Algorithm on which to base verifier :type algorithm: aws_encryption_sdk.identifiers.Algorithm :param bytes encoded_point: Raw verification key + :param encoding: Encoding used for key bytes + :type encoding: cryptography.hazmat.primitives.serialization.encoding :returns: Instance of Verifier generated from encoded point :rtype: aws_encryption_sdk.internal.crypto.Verifier """ + if encoding == serialization.Encoding.DER: + key = serialization.load_der_private_key(data=key_bytes, password=None, backend=default_backend()) + elif encoding == serialization.Encoding.PEM: + key = serialization.load_pem_private_key(data=key_bytes, password=None, backend=default_backend()) + else: + raise ValueError("Unsupported verification key encoding: {}".format(encoding)) return cls( algorithm=algorithm, key=serialization.load_pem_public_key(data=key_bytes, backend=default_backend()) ) - + def key_bytes(self): """Returns the raw verification key. From 6f5504741a4797ee1cce5986a5ba48718759b810 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 5 Feb 2024 13:30:49 -0800 Subject: [PATCH 012/376] fix pem/der --- src/aws_encryption_sdk/streaming_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index ec19b6dd5..f6b3529a9 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -172,7 +172,7 @@ def _has_mpl_attrs_post_init(self): assert isinstance(self.keyring, IKeyring) except AssertionError: raise ValueError(f"Argument provided to keyring MUST be a {IKeyring}. \ - Found {self.keyring.__class__.__name__=}") + Found {self.keyring.__class__.__name__}") mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( config=MaterialProvidersConfig() From 1b1b4e4bc61d7c74a3815d225e992c1cb7bce135 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 5 Feb 2024 16:11:08 -0800 Subject: [PATCH 013/376] debug --- src/aws_encryption_sdk/internal/crypto/authentication.py | 2 +- src/aws_encryption_sdk/streaming_client.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/aws_encryption_sdk/internal/crypto/authentication.py b/src/aws_encryption_sdk/internal/crypto/authentication.py index 8c1b9af31..5e8dcd10c 100644 --- a/src/aws_encryption_sdk/internal/crypto/authentication.py +++ b/src/aws_encryption_sdk/internal/crypto/authentication.py @@ -173,7 +173,7 @@ def from_key_bytes(cls, algorithm, key_bytes, encoding=serialization.Encoding.DE else: raise ValueError("Unsupported verification key encoding: {}".format(encoding)) return cls( - algorithm=algorithm, key=serialization.load_pem_public_key(data=key_bytes, backend=default_backend()) + algorithm=algorithm, key=key ) def key_bytes(self): diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index f6b3529a9..83e4a9ecd 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -165,12 +165,10 @@ def _has_mpl_attrs_post_init(self): # No CMM, provided legacy native `key_provider` => create legacy native DefaultCryptoMaterialsManager self.materials_manager = DefaultCryptoMaterialsManager( master_key_provider=self.key_provider - ) + ) elif self.keyring is not None: # No CMM, provided MPL keyring => create MPL's DefaultCryptographicMaterialsManager - try: - assert isinstance(self.keyring, IKeyring) - except AssertionError: + if not isinstance(self.keyring, IKeyring): raise ValueError(f"Argument provided to keyring MUST be a {IKeyring}. \ Found {self.keyring.__class__.__name__}") From 38a4cc9c6808d5da025822634e8ec9bb0b9f960f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 5 Feb 2024 16:14:15 -0800 Subject: [PATCH 014/376] debug --- src/aws_encryption_sdk/internal/crypto/authentication.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aws_encryption_sdk/internal/crypto/authentication.py b/src/aws_encryption_sdk/internal/crypto/authentication.py index 5e8dcd10c..80469ff1f 100644 --- a/src/aws_encryption_sdk/internal/crypto/authentication.py +++ b/src/aws_encryption_sdk/internal/crypto/authentication.py @@ -167,9 +167,9 @@ def from_key_bytes(cls, algorithm, key_bytes, encoding=serialization.Encoding.DE :rtype: aws_encryption_sdk.internal.crypto.Verifier """ if encoding == serialization.Encoding.DER: - key = serialization.load_der_private_key(data=key_bytes, password=None, backend=default_backend()) + key = serialization.load_der_private_key(data=key_bytes, backend=default_backend()) elif encoding == serialization.Encoding.PEM: - key = serialization.load_pem_private_key(data=key_bytes, password=None, backend=default_backend()) + key = serialization.load_pem_private_key(data=key_bytes, backend=default_backend()) else: raise ValueError("Unsupported verification key encoding: {}".format(encoding)) return cls( From 0cd0e2301fc7133e8023a825f128bc457af311aa Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 6 Feb 2024 15:47:48 -0800 Subject: [PATCH 015/376] fix --- .../internal/crypto/authentication.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/src/aws_encryption_sdk/internal/crypto/authentication.py b/src/aws_encryption_sdk/internal/crypto/authentication.py index 80469ff1f..88d21a2ef 100644 --- a/src/aws_encryption_sdk/internal/crypto/authentication.py +++ b/src/aws_encryption_sdk/internal/crypto/authentication.py @@ -78,12 +78,7 @@ def from_key_bytes(cls, algorithm, key_bytes, encoding=serialization.Encoding.DE :type encoding: cryptography.hazmat.primitives.serialization.encoding :rtype: aws_encryption_sdk.internal.crypto.Signer """ - if encoding == serialization.Encoding.DER: - key = serialization.load_der_private_key(data=key_bytes, password=None, backend=default_backend()) - elif encoding == serialization.Encoding.PEM: - key = serialization.load_pem_private_key(data=key_bytes, password=None, backend=default_backend()) - else: - raise ValueError("Unsupported signing key encoding: {}".format(encoding)) + key = serialization.load_der_private_key(data=key_bytes, password=None, backend=default_backend()) return cls(algorithm, key) def key_bytes(self, encoding=serialization.Encoding.DER): @@ -166,14 +161,8 @@ def from_key_bytes(cls, algorithm, key_bytes, encoding=serialization.Encoding.DE :returns: Instance of Verifier generated from encoded point :rtype: aws_encryption_sdk.internal.crypto.Verifier """ - if encoding == serialization.Encoding.DER: - key = serialization.load_der_private_key(data=key_bytes, backend=default_backend()) - elif encoding == serialization.Encoding.PEM: - key = serialization.load_pem_private_key(data=key_bytes, backend=default_backend()) - else: - raise ValueError("Unsupported verification key encoding: {}".format(encoding)) return cls( - algorithm=algorithm, key=key + algorithm=algorithm, key=serialization.load_der_public_key(data=key_bytes, backend=default_backend()) ) def key_bytes(self): From 44826a2568fd3fa86d3031d11837c707d17850e0 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 6 Feb 2024 16:09:14 -0800 Subject: [PATCH 016/376] fix --- .../internal/crypto/authentication.py | 14 +++++--------- src/aws_encryption_sdk/streaming_client.py | 3 ++- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/aws_encryption_sdk/internal/crypto/authentication.py b/src/aws_encryption_sdk/internal/crypto/authentication.py index 88d21a2ef..f90ac77e0 100644 --- a/src/aws_encryption_sdk/internal/crypto/authentication.py +++ b/src/aws_encryption_sdk/internal/crypto/authentication.py @@ -68,26 +68,24 @@ class Signer(_PrehashingAuthenticator): """ @classmethod - def from_key_bytes(cls, algorithm, key_bytes, encoding=serialization.Encoding.DER): + def from_key_bytes(cls, algorithm, key_bytes): """Builds a `Signer` from an algorithm suite and a raw signing key. :param algorithm: Algorithm on which to base signer :type algorithm: aws_encryption_sdk.identifiers.Algorithm :param bytes key_bytes: Raw signing key - :param encoding: Encoding used for key bytes - :type encoding: cryptography.hazmat.primitives.serialization.encoding :rtype: aws_encryption_sdk.internal.crypto.Signer """ key = serialization.load_der_private_key(data=key_bytes, password=None, backend=default_backend()) return cls(algorithm, key) - def key_bytes(self, encoding=serialization.Encoding.DER): + def key_bytes(self): """Returns the raw signing key. :rtype: bytes """ return self.key.private_bytes( - encoding=encoding, + encoding=serialization.Encoding.DER, format=serialization.PrivateFormat.PKCS8, encryption_algorithm=serialization.NoEncryption(), ) @@ -150,21 +148,19 @@ def from_encoded_point(cls, algorithm, encoded_point): ) @classmethod - def from_key_bytes(cls, algorithm, key_bytes, encoding=serialization.Encoding.DER): + def from_key_bytes(cls, algorithm, key_bytes): """Creates a `Verifier` object based on the supplied algorithm and raw verification key. :param algorithm: Algorithm on which to base verifier :type algorithm: aws_encryption_sdk.identifiers.Algorithm :param bytes encoded_point: Raw verification key - :param encoding: Encoding used for key bytes - :type encoding: cryptography.hazmat.primitives.serialization.encoding :returns: Instance of Verifier generated from encoded point :rtype: aws_encryption_sdk.internal.crypto.Verifier """ return cls( algorithm=algorithm, key=serialization.load_der_public_key(data=key_bytes, backend=default_backend()) ) - + def key_bytes(self): """Returns the raw verification key. diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 83e4a9ecd..582472025 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -919,7 +919,8 @@ def _read_header(self): # MPL verification key is NOT key bytes, it is bytes of the compressed point # TODO-MPL: clean this up, least-privilege violation import base64 - if hasattr(self.config.materials_manager, "mpl_cmm"): + if (isinstance(self.config.materials_manager, CMMHandler) + and hasattr(self.config.materials_manager, "mpl_cmm")): self.verifier = Verifier.from_encoded_point( algorithm=header.algorithm, encoded_point=base64.b64encode(decryption_materials.verification_key) From 02e9f843826506597ca03a3139da1b58a88da2f6 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 6 Feb 2024 16:27:51 -0800 Subject: [PATCH 017/376] fix --- src/aws_encryption_sdk/cmm_handler.py | 13 ++++++++----- src/aws_encryption_sdk/materials_handlers.py | 12 +++++++----- src/aws_encryption_sdk/streaming_client.py | 2 +- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/aws_encryption_sdk/cmm_handler.py b/src/aws_encryption_sdk/cmm_handler.py index 887d9d79e..fa1786837 100644 --- a/src/aws_encryption_sdk/cmm_handler.py +++ b/src/aws_encryption_sdk/cmm_handler.py @@ -17,9 +17,12 @@ CommitmentPolicyESDK, AlgorithmSuiteIdESDK, ) + except ImportError: pass +from typing import List + from aws_encryption_sdk.exceptions import ( AWSEncryptionSDKClientError, ) @@ -72,7 +75,7 @@ def __init__( elif isinstance(cmm, ICryptographicMaterialsManager): self.mpl_cmm = cmm else: - raise ValueError(f"Invalid CMM passed to CMMHandler: {cmm=}") + raise ValueError(f"Invalid CMM passed to CMMHandler. cmm: {cmm}") def get_encryption_materials( self, @@ -115,7 +118,7 @@ def _create_mpl_get_encryption_materials_input_from_request( @staticmethod def _map_native_commitment_policy_to_mpl_commitment_policy( native_commitment_policy: CommitmentPolicy - ) -> CommitmentPolicyESDK: + ) -> 'CommitmentPolicyESDK': if native_commitment_policy == CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT: return CommitmentPolicyESDK(value="FORBID_ENCRYPT_ALLOW_DECRYPT") elif native_commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT: @@ -123,7 +126,7 @@ def _map_native_commitment_policy_to_mpl_commitment_policy( elif native_commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT: return CommitmentPolicyESDK(value="REQUIRE_ENCRYPT_REQUIRE_DECRYPT") else: - raise ValueError(f"Invalid {native_commitment_policy=}") + raise ValueError(f"Invalid native_commitment_policy: {native_commitment_policy}") def decrypt_materials( self, @@ -146,7 +149,7 @@ def decrypt_materials( raise AWSEncryptionSDKClientError(e) @staticmethod - def _native_algorithm_id_to_mpl_algorithm_id(native_algorithm_id: str) -> AlgorithmSuiteIdESDK: + def _native_algorithm_id_to_mpl_algorithm_id(native_algorithm_id: str) -> 'AlgorithmSuiteIdESDK': # MPL algorithm suite ID = hexstr(native_algorithm_id) padded to 4 digits post-`x`. return AlgorithmSuiteIdESDK(f"{native_algorithm_id:#0{6}x}") @@ -154,7 +157,7 @@ def _native_algorithm_id_to_mpl_algorithm_id(native_algorithm_id: str) -> Algori def _create_mpl_decrypt_materials_input_from_request( request: DecryptionMaterialsRequest ) -> 'DecryptMaterialsInput': - key_blob_list: list[Native_EncryptedDataKey] = request.encrypted_data_keys + key_blob_list: List[Native_EncryptedDataKey] = request.encrypted_data_keys list_edks = [MPL_EncryptedDataKey( key_provider_id=key_blob.key_provider.provider_id, key_provider_info=key_blob.key_provider.key_info, diff --git a/src/aws_encryption_sdk/materials_handlers.py b/src/aws_encryption_sdk/materials_handlers.py index d54e4517b..970963e10 100644 --- a/src/aws_encryption_sdk/materials_handlers.py +++ b/src/aws_encryption_sdk/materials_handlers.py @@ -9,6 +9,8 @@ except ImportError: pass +from typing import Dict, List + from aws_encryption_sdk.materials_managers import ( DecryptionMaterials as Native_DecryptionMaterials, EncryptionMaterials as Native_EncryptionMaterials, @@ -53,7 +55,7 @@ def __init__( elif isinstance(materials, MPL_EncryptionMaterials): self.mpl_materials = materials else: - raise ValueError(f"Invalid EncryptionMaterials passed to EncryptionMaterialsHandler: {materials=}") + raise ValueError(f"Invalid EncryptionMaterials passed to EncryptionMaterialsHandler. materials: {materials}") @property def algorithm(self) -> Algorithm: @@ -68,7 +70,7 @@ def algorithm(self) -> Algorithm: ) @property - def encryption_context(self) -> dict[str, str]: + def encryption_context(self) -> Dict[str, str]: """Materials' encryption context.""" if hasattr(self, "native_materials"): return self.native_materials.encryption_context @@ -76,12 +78,12 @@ def encryption_context(self) -> dict[str, str]: return self.mpl_materials.encryption_context @property - def encrypted_data_keys(self) -> list[Native_EncryptedDataKey]: + def encrypted_data_keys(self) -> List[Native_EncryptedDataKey]: """Materials' encrypted data keys.""" if hasattr(self, "native_materials"): return self.native_materials.encrypted_data_keys else: - mpl_edk_list: list[MPL_EncryptedDataKey] = self.mpl_materials.encrypted_data_keys + mpl_edk_list: List[MPL_EncryptedDataKey] = self.mpl_materials.encrypted_data_keys key_blob_list: set[Native_EncryptedDataKey] = {Native_EncryptedDataKey( key_provider=MasterKeyInfo( provider_id=mpl_edk.key_provider_id, @@ -144,7 +146,7 @@ def __init__( elif isinstance(materials, MPL_DecryptionMaterials): self.mpl_materials = materials else: - raise ValueError(f"Invalid DecryptionMaterials passed to DecryptionMaterialsHandler: {materials=}") + raise ValueError(f"Invalid DecryptionMaterials passed to DecryptionMaterialsHandler. materials: {materials}") @property def data_key(self) -> DataKey: diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 582472025..6a2dc1d27 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -66,6 +66,7 @@ from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager from aws_encryption_sdk.materials_managers.default import DefaultCryptoMaterialsManager from aws_encryption_sdk.structures import MessageHeader +from aws_encryption_sdk.cmm_handler import CMMHandler try: from aws_cryptographic_materialproviders.mpl.client import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig @@ -75,7 +76,6 @@ from aws_cryptographic_materialproviders.mpl.references import ( IKeyring, ) - from aws_encryption_sdk.cmm_handler import CMMHandler _has_mpl = True except ImportError: From a3babfd2ef936cf4595bdaab041fc16e7a005868 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 6 Feb 2024 17:55:40 -0800 Subject: [PATCH 018/376] linter --- src/aws_encryption_sdk/cmm_handler.py | 36 +++++++++----------- src/aws_encryption_sdk/materials_handlers.py | 10 +++--- src/aws_encryption_sdk/streaming_client.py | 18 +++++----- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/aws_encryption_sdk/cmm_handler.py b/src/aws_encryption_sdk/cmm_handler.py index fa1786837..20298f801 100644 --- a/src/aws_encryption_sdk/cmm_handler.py +++ b/src/aws_encryption_sdk/cmm_handler.py @@ -85,38 +85,35 @@ def get_encryption_materials( Returns an EncryptionMaterialsHandler for the configured CMM. :param request: Request for encryption materials """ - if (self._is_using_native_cmm()): + if self._is_using_native_cmm(): return EncryptionMaterialsHandler(self.native_cmm.get_encryption_materials(request)) else: try: - input: GetEncryptionMaterialsInput = CMMHandler._create_mpl_get_encryption_materials_input_from_request( + mpl_input: GetEncryptionMaterialsInput = CMMHandler._native_to_mpl_get_encryption_materials( request ) - output: GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(input) - return EncryptionMaterialsHandler(output.encryption_materials) - except AwsCryptographicMaterialProvidersException as e: + mpl_output: GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) + return EncryptionMaterialsHandler(mpl_output.encryption_materials) + except AwsCryptographicMaterialProvidersException as mpl_exception: # Wrap MPL error into the ESDK error type # so customers only have to catch ESDK error types. - raise AWSEncryptionSDKClientError(e) + raise AWSEncryptionSDKClientError(mpl_exception) @staticmethod - def _create_mpl_get_encryption_materials_input_from_request( + def _native_to_mpl_get_encryption_materials( request: EncryptionMaterialsRequest ) -> 'GetEncryptionMaterialsInput': output: GetEncryptionMaterialsInput = GetEncryptionMaterialsInput( encryption_context=request.encryption_context, - commitment_policy=CMMHandler._map_native_commitment_policy_to_mpl_commitment_policy( + commitment_policy=CMMHandler._native_to_mpl_commmitment_policy( request.commitment_policy ), - # TODO double check this - # optional... maybe this needs to be kwargs?? - # algorithm_suite_id=request.algorithm.algorithm_id, max_plaintext_length=request.plaintext_length, ) return output @staticmethod - def _map_native_commitment_policy_to_mpl_commitment_policy( + def _native_to_mpl_commmitment_policy( native_commitment_policy: CommitmentPolicy ) -> 'CommitmentPolicyESDK': if native_commitment_policy == CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT: @@ -136,17 +133,18 @@ def decrypt_materials( Returns a DecryptionMaterialsHandler for the configured CMM. :param request: Request for decryption materials """ - if (self._is_using_native_cmm()): + if self._is_using_native_cmm(): return DecryptionMaterialsHandler(self.native_cmm.decrypt_materials(request)) else: try: - input: 'DecryptMaterialsInput' = CMMHandler._create_mpl_decrypt_materials_input_from_request(request) - output: 'DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(input) - return DecryptionMaterialsHandler(output.decryption_materials) - except AwsCryptographicMaterialProvidersException as e: + mpl_input: 'DecryptMaterialsInput' = \ + CMMHandler._create_mpl_decrypt_materials_input_from_request(request) + mpl_output: 'DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(mpl_input) + return DecryptionMaterialsHandler(mpl_output.decryption_materials) + except AwsCryptographicMaterialProvidersException as mpl_exception: # Wrap MPL error into the ESDK error type # so customers only have to catch ESDK error types. - raise AWSEncryptionSDKClientError(e) + raise AWSEncryptionSDKClientError(mpl_exception) @staticmethod def _native_algorithm_id_to_mpl_algorithm_id(native_algorithm_id: str) -> 'AlgorithmSuiteIdESDK': @@ -167,7 +165,7 @@ def _create_mpl_decrypt_materials_input_from_request( algorithm_suite_id=CMMHandler._native_algorithm_id_to_mpl_algorithm_id( request.algorithm.algorithm_id ), - commitment_policy=CMMHandler._map_native_commitment_policy_to_mpl_commitment_policy( + commitment_policy=CMMHandler._native_to_mpl_commmitment_policy( request.commitment_policy ), encrypted_data_keys=list_edks, diff --git a/src/aws_encryption_sdk/materials_handlers.py b/src/aws_encryption_sdk/materials_handlers.py index 970963e10..00d67ed71 100644 --- a/src/aws_encryption_sdk/materials_handlers.py +++ b/src/aws_encryption_sdk/materials_handlers.py @@ -9,7 +9,7 @@ except ImportError: pass -from typing import Dict, List +from typing import Dict, List, Set from aws_encryption_sdk.materials_managers import ( DecryptionMaterials as Native_DecryptionMaterials, @@ -55,7 +55,8 @@ def __init__( elif isinstance(materials, MPL_EncryptionMaterials): self.mpl_materials = materials else: - raise ValueError(f"Invalid EncryptionMaterials passed to EncryptionMaterialsHandler. materials: {materials}") + raise ValueError(f"Invalid EncryptionMaterials passed to EncryptionMaterialsHandler.\ + materials: {materials}") @property def algorithm(self) -> Algorithm: @@ -84,7 +85,7 @@ def encrypted_data_keys(self) -> List[Native_EncryptedDataKey]: return self.native_materials.encrypted_data_keys else: mpl_edk_list: List[MPL_EncryptedDataKey] = self.mpl_materials.encrypted_data_keys - key_blob_list: set[Native_EncryptedDataKey] = {Native_EncryptedDataKey( + key_blob_list: Set[Native_EncryptedDataKey] = {Native_EncryptedDataKey( key_provider=MasterKeyInfo( provider_id=mpl_edk.key_provider_id, key_info=mpl_edk.key_provider_info, @@ -146,7 +147,8 @@ def __init__( elif isinstance(materials, MPL_DecryptionMaterials): self.mpl_materials = materials else: - raise ValueError(f"Invalid DecryptionMaterials passed to DecryptionMaterialsHandler. materials: {materials}") + raise ValueError(f"Invalid DecryptionMaterials passed to DecryptionMaterialsHandler.\ + materials: {materials}") @property def data_key(self) -> DataKey: diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 6a2dc1d27..6b977e6e4 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -18,6 +18,7 @@ import io import logging import math +import base64 import attr import six @@ -77,14 +78,14 @@ IKeyring, ) - _has_mpl = True + HAS_MPL = True except ImportError: - _has_mpl = False + HAS_MPL = False _LOGGER = logging.getLogger(__name__) -def _exactly_one_arg_is_not_None(*args): +def _exactly_one_arg_is_not_none(*args): """ Private helper function. Returns `True` if exactly one item in the list is not `None`. @@ -146,7 +147,7 @@ class _ClientConfig(object): # pylint: disable=too-many-instance-attributes key_provider = attr.ib( hash=True, default=None, validator=attr.validators.optional(attr.validators.instance_of(MasterKeyProvider)) ) - if _has_mpl: + if HAS_MPL: keyring = attr.ib( hash=True, default=None, validator=attr.validators.optional(attr.validators.instance_of(IKeyring)) ) @@ -158,14 +159,14 @@ class _ClientConfig(object): # pylint: disable=too-many-instance-attributes ) # DEPRECATED: Value is no longer configurable here. Parameter left here to avoid breaking consumers. def _has_mpl_attrs_post_init(self): - if not _exactly_one_arg_is_not_None(self.materials_manager, self.key_provider, self.keyring): + if not _exactly_one_arg_is_not_none(self.materials_manager, self.key_provider, self.keyring): raise TypeError("Exactly one of keyring, materials_manager, or key_provider must be provided") if self.materials_manager is None: if self.key_provider is not None: # No CMM, provided legacy native `key_provider` => create legacy native DefaultCryptoMaterialsManager self.materials_manager = DefaultCryptoMaterialsManager( master_key_provider=self.key_provider - ) + ) elif self.keyring is not None: # No CMM, provided MPL keyring => create MPL's DefaultCryptographicMaterialsManager if not isinstance(self.keyring, IKeyring): @@ -194,9 +195,9 @@ def _no_mpl_attrs_post_init(self): def __attrs_post_init__(self): """Normalize inputs to crypto material manager.""" - if _has_mpl: + if HAS_MPL: self._has_mpl_attrs_post_init() - elif not _has_mpl: + elif not HAS_MPL: self._no_mpl_attrs_post_init() @@ -918,7 +919,6 @@ def _read_header(self): else: # MPL verification key is NOT key bytes, it is bytes of the compressed point # TODO-MPL: clean this up, least-privilege violation - import base64 if (isinstance(self.config.materials_manager, CMMHandler) and hasattr(self.config.materials_manager, "mpl_cmm")): self.verifier = Verifier.from_encoded_point( From d2c974afc11a0f11a3ab32ad44e17061d640c5d1 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 6 Feb 2024 17:59:21 -0800 Subject: [PATCH 019/376] linter --- setup.py | 1 + src/aws_encryption_sdk/cmm_handler.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/setup.py b/setup.py index 4cd8027cd..084edc09a 100644 --- a/setup.py +++ b/setup.py @@ -39,6 +39,7 @@ def get_requirements(): keywords="aws-encryption-sdk aws kms encryption", license="Apache License 2.0", install_requires=get_requirements(), + # pylint: disable=fixme # TODO: Point at MPL main branch once Python MPL is merged into main. extras_require={ "MPL": ["aws-cryptographic-material-providers @" \ diff --git a/src/aws_encryption_sdk/cmm_handler.py b/src/aws_encryption_sdk/cmm_handler.py index 20298f801..2479038a1 100644 --- a/src/aws_encryption_sdk/cmm_handler.py +++ b/src/aws_encryption_sdk/cmm_handler.py @@ -2,6 +2,8 @@ # These dependencies are only loaded if you install the MPL. try: + # pylint seems to struggle with this condition import + # pylint: disable=unused-import from aws_cryptographic_materialproviders.mpl.errors import ( AwsCryptographicMaterialProvidersException ) From 55b24a83580f880278d0689f57fbfe1bdca285f6 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 09:44:02 -0800 Subject: [PATCH 020/376] isort --- examples/src/basic_encryption.py | 5 --- examples/src/keyrings/hierarchical_keyring.py | 32 +++++--------- src/aws_encryption_sdk/cmm_handler.py | 44 ++++++------------- src/aws_encryption_sdk/materials_handlers.py | 13 ++---- src/aws_encryption_sdk/streaming_client.py | 13 +++--- 5 files changed, 32 insertions(+), 75 deletions(-) diff --git a/examples/src/basic_encryption.py b/examples/src/basic_encryption.py index 7b729feab..cfe8ac791 100644 --- a/examples/src/basic_encryption.py +++ b/examples/src/basic_encryption.py @@ -51,8 +51,3 @@ def cycle_string(key_arn, source_plaintext, botocore_session=None): assert all( pair in decrypted_header.encryption_context.items() for pair in encryptor_header.encryption_context.items() ) - -cycle_string( - "arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f", - "abcdefg", -) \ No newline at end of file diff --git a/examples/src/keyrings/hierarchical_keyring.py b/examples/src/keyrings/hierarchical_keyring.py index e8f662b73..20647bed6 100644 --- a/examples/src/keyrings/hierarchical_keyring.py +++ b/examples/src/keyrings/hierarchical_keyring.py @@ -4,43 +4,32 @@ """Example showing basic encryption and decryption of a value already in memory.""" -import aws_encryption_sdk -from aws_encryption_sdk import CommitmentPolicy -import boto3 - import sys -from aws_encryption_sdk.exceptions import ( - AWSEncryptionSDKClientError, - SerializationError, -) +import boto3 + +import aws_encryption_sdk +from aws_encryption_sdk import CommitmentPolicy +from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError, SerializationError module_root_dir = '/'.join(__file__.split("/")[:-1]) sys.path.append(module_root_dir) import aws_cryptographic_materialproviders - +from aws_cryptographic_materialproviders.keystore.client import KeyStore +from aws_cryptographic_materialproviders.keystore.config import KeyStoreConfig +from aws_cryptographic_materialproviders.keystore.models import CreateKeyInput, KMSConfigurationKmsKeyArn from aws_cryptographic_materialproviders.mpl.client import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig from aws_cryptographic_materialproviders.mpl.models import ( - CreateAwsKmsHierarchicalKeyringInput, CacheTypeDefault, + CreateAwsKmsHierarchicalKeyringInput, DefaultCache, GetBranchKeyIdInput, GetBranchKeyIdOutput, ) -from aws_cryptographic_materialproviders.mpl.references import ( - IKeyring, - IBranchKeyIdSupplier, -) - -from aws_cryptographic_materialproviders.keystore.client import KeyStore -from aws_cryptographic_materialproviders.keystore.config import KeyStoreConfig -from aws_cryptographic_materialproviders.keystore.models import ( - CreateKeyInput, - KMSConfigurationKmsKeyArn, -) +from aws_cryptographic_materialproviders.mpl.references import IBranchKeyIdSupplier, IKeyring EXAMPLE_DATA: bytes = b"Hello World" @@ -241,6 +230,7 @@ def get_branch_key_id( # hack in a test import botocore + encrypt_and_decrypt_with_keyring( "KeyStoreDdbTable", "KeyStoreDdbTable", diff --git a/src/aws_encryption_sdk/cmm_handler.py b/src/aws_encryption_sdk/cmm_handler.py index 2479038a1..5bac15b87 100644 --- a/src/aws_encryption_sdk/cmm_handler.py +++ b/src/aws_encryption_sdk/cmm_handler.py @@ -2,49 +2,31 @@ # These dependencies are only loaded if you install the MPL. try: - # pylint seems to struggle with this condition import + # pylint seems to struggle with this conditional import # pylint: disable=unused-import - from aws_cryptographic_materialproviders.mpl.errors import ( - AwsCryptographicMaterialProvidersException - ) - from aws_cryptographic_materialproviders.mpl.references import ( - ICryptographicMaterialsManager, - ) + from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException from aws_cryptographic_materialproviders.mpl.models import ( - GetEncryptionMaterialsInput, - GetEncryptionMaterialsOutput, + AlgorithmSuiteIdESDK, + CommitmentPolicyESDK, DecryptMaterialsInput, DecryptMaterialsOutput, EncryptedDataKey as MPL_EncryptedDataKey, - CommitmentPolicyESDK, - AlgorithmSuiteIdESDK, + GetEncryptionMaterialsInput, + GetEncryptionMaterialsOutput, ) + from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager except ImportError: pass from typing import List -from aws_encryption_sdk.exceptions import ( - AWSEncryptionSDKClientError, -) -from aws_encryption_sdk.materials_managers import ( - DecryptionMaterialsRequest, - EncryptionMaterialsRequest, -) -from aws_encryption_sdk.materials_managers.base import ( - CryptoMaterialsManager, -) -from aws_encryption_sdk.materials_handlers import ( - EncryptionMaterialsHandler, - DecryptionMaterialsHandler, -) -from aws_encryption_sdk.structures import ( - EncryptedDataKey as Native_EncryptedDataKey, -) -from aws_encryption_sdk.identifiers import ( - CommitmentPolicy, -) +from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError +from aws_encryption_sdk.identifiers import CommitmentPolicy +from aws_encryption_sdk.materials_handlers import DecryptionMaterialsHandler, EncryptionMaterialsHandler +from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest +from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager +from aws_encryption_sdk.structures import EncryptedDataKey as Native_EncryptedDataKey # TODO-MPL Should this implement interface..? seems like yes since it implements all of interface methods diff --git a/src/aws_encryption_sdk/materials_handlers.py b/src/aws_encryption_sdk/materials_handlers.py index 00d67ed71..57f54144e 100644 --- a/src/aws_encryption_sdk/materials_handlers.py +++ b/src/aws_encryption_sdk/materials_handlers.py @@ -3,27 +3,20 @@ try: from aws_cryptographic_materialproviders.mpl.models import ( DecryptionMaterials as MPL_DecryptionMaterials, - EncryptionMaterials as MPL_EncryptionMaterials, EncryptedDataKey as MPL_EncryptedDataKey, + EncryptionMaterials as MPL_EncryptionMaterials, ) except ImportError: pass from typing import Dict, List, Set +from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite from aws_encryption_sdk.materials_managers import ( DecryptionMaterials as Native_DecryptionMaterials, EncryptionMaterials as Native_EncryptionMaterials, ) -from aws_encryption_sdk.identifiers import ( - Algorithm, - AlgorithmSuite, -) -from aws_encryption_sdk.structures import ( - DataKey, - EncryptedDataKey as Native_EncryptedDataKey, - MasterKeyInfo, -) +from aws_encryption_sdk.structures import DataKey, EncryptedDataKey as Native_EncryptedDataKey, MasterKeyInfo def _mpl_algorithm_id_to_native_algorithm_id(mpl_algorithm_id: str): diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 6b977e6e4..afe9987ff 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -14,16 +14,17 @@ from __future__ import division import abc +import base64 import hmac import io import logging import math -import base64 import attr import six import aws_encryption_sdk.internal.utils +from aws_encryption_sdk.cmm_handler import CMMHandler from aws_encryption_sdk.exceptions import ( ActionNotAllowedError, AWSEncryptionSDKClientError, @@ -67,16 +68,12 @@ from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager from aws_encryption_sdk.materials_managers.default import DefaultCryptoMaterialsManager from aws_encryption_sdk.structures import MessageHeader -from aws_encryption_sdk.cmm_handler import CMMHandler + try: from aws_cryptographic_materialproviders.mpl.client import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig - from aws_cryptographic_materialproviders.mpl.models import ( - CreateDefaultCryptographicMaterialsManagerInput - ) - from aws_cryptographic_materialproviders.mpl.references import ( - IKeyring, - ) + from aws_cryptographic_materialproviders.mpl.models import CreateDefaultCryptographicMaterialsManagerInput + from aws_cryptographic_materialproviders.mpl.references import IKeyring HAS_MPL = True except ImportError: From 7e5fa4837f252f8b038efc51567d04f90d6510d8 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 09:54:14 -0800 Subject: [PATCH 021/376] flake8 examples --- examples/src/keyrings/hierarchical_keyring.py | 34 ++++++------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/examples/src/keyrings/hierarchical_keyring.py b/examples/src/keyrings/hierarchical_keyring.py index 20647bed6..81d02f786 100644 --- a/examples/src/keyrings/hierarchical_keyring.py +++ b/examples/src/keyrings/hierarchical_keyring.py @@ -1,8 +1,5 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 - - - """Example showing basic encryption and decryption of a value already in memory.""" import sys @@ -10,13 +7,8 @@ import aws_encryption_sdk from aws_encryption_sdk import CommitmentPolicy -from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError, SerializationError - -module_root_dir = '/'.join(__file__.split("/")[:-1]) - -sys.path.append(module_root_dir) +from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError -import aws_cryptographic_materialproviders from aws_cryptographic_materialproviders.keystore.client import KeyStore from aws_cryptographic_materialproviders.keystore.config import KeyStoreConfig from aws_cryptographic_materialproviders.keystore.models import CreateKeyInput, KMSConfigurationKmsKeyArn @@ -31,13 +23,18 @@ ) from aws_cryptographic_materialproviders.mpl.references import IBranchKeyIdSupplier, IKeyring +module_root_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(module_root_dir) + EXAMPLE_DATA: bytes = b"Hello World" + def encrypt_and_decrypt_with_keyring( - key_store_table_name: str, - logical_key_store_name: str, - kms_key_id: str - ): + key_store_table_name: str, + logical_key_store_name: str, + kms_key_id: str +): # 1. Instantiate the encryption SDK client. # This builds the client with the REQUIRE_ENCRYPT_REQUIRE_DECRYPT commitment policy, @@ -90,7 +87,7 @@ def get_branch_key_id( if b"tenant" not in encryption_context: raise ValueError("EncryptionContext invalid, does not contain expected tenant key value pair.") - + tenant_key_id: str = encryption_context.get(b"tenant") branch_key_id: str @@ -227,12 +224,3 @@ def get_branch_key_id( assert plaintext_bytes_B == EXAMPLE_DATA # Also, a thread-safe example ig - -# hack in a test -import botocore - -encrypt_and_decrypt_with_keyring( - "KeyStoreDdbTable", - "KeyStoreDdbTable", - "arn:aws:kms:us-west-2:370957321024:key/9d989aa2-2f9c-438c-a745-cc57d3ad0126" -) \ No newline at end of file From 055deabd332af255f40df7fb52b80063df268f06 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 09:58:05 -0800 Subject: [PATCH 022/376] isort + flake8 --- examples/src/keyrings/hierarchical_keyring.py | 10 +++++----- examples/src/keyrings/module_.py | 1 + examples/src/module_.py | 1 + examples/test/examples_test_utils.py | 2 +- examples/test/test_i_basic_encryption.py | 1 - ..._i_basic_file_encryption_with_multiple_providers.py | 4 +--- ...st_i_basic_file_encryption_with_raw_key_provider.py | 1 - examples/test/test_i_data_key_caching_basic.py | 1 - examples/test/test_i_discovery_kms_provider.py | 4 +--- examples/test/test_i_mrk_aware_kms_provider.py | 4 +--- examples/test/test_i_multiple_kms_cmk.py | 4 +--- examples/test/test_i_one_kms_cmk.py | 4 +--- examples/test/test_i_one_kms_cmk_streaming_data.py | 1 - examples/test/test_i_one_kms_cmk_unsigned.py | 4 +--- examples/test/test_i_set_commitment.py | 4 +--- 15 files changed, 15 insertions(+), 31 deletions(-) diff --git a/examples/src/keyrings/hierarchical_keyring.py b/examples/src/keyrings/hierarchical_keyring.py index 81d02f786..acc594cc3 100644 --- a/examples/src/keyrings/hierarchical_keyring.py +++ b/examples/src/keyrings/hierarchical_keyring.py @@ -4,11 +4,6 @@ import sys import boto3 - -import aws_encryption_sdk -from aws_encryption_sdk import CommitmentPolicy -from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError - from aws_cryptographic_materialproviders.keystore.client import KeyStore from aws_cryptographic_materialproviders.keystore.config import KeyStoreConfig from aws_cryptographic_materialproviders.keystore.models import CreateKeyInput, KMSConfigurationKmsKeyArn @@ -23,6 +18,10 @@ ) from aws_cryptographic_materialproviders.mpl.references import IBranchKeyIdSupplier, IKeyring +import aws_encryption_sdk +from aws_encryption_sdk import CommitmentPolicy +from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError + module_root_dir = '/'.join(__file__.split("/")[:-1]) sys.path.append(module_root_dir) @@ -35,6 +34,7 @@ def encrypt_and_decrypt_with_keyring( logical_key_store_name: str, kms_key_id: str ): + """Creates a hierarchical keyring using the provided resources, then encrypts and decrypts a string with it.""" # 1. Instantiate the encryption SDK client. # This builds the client with the REQUIRE_ENCRYPT_REQUIRE_DECRYPT commitment policy, diff --git a/examples/src/keyrings/module_.py b/examples/src/keyrings/module_.py index e69de29bb..2f64c8e0f 100644 --- a/examples/src/keyrings/module_.py +++ b/examples/src/keyrings/module_.py @@ -0,0 +1 @@ +"""Should remove this.""" \ No newline at end of file diff --git a/examples/src/module_.py b/examples/src/module_.py index e69de29bb..2f64c8e0f 100644 --- a/examples/src/module_.py +++ b/examples/src/module_.py @@ -0,0 +1 @@ +"""Should remove this.""" \ No newline at end of file diff --git a/examples/test/examples_test_utils.py b/examples/test/examples_test_utils.py index 8a51f21c8..08e8cf2f5 100644 --- a/examples/test/examples_test_utils.py +++ b/examples/test/examples_test_utils.py @@ -49,7 +49,7 @@ from integration_test_utils import ( # noqa pylint: disable=unused-import,import-error get_cmk_arn, - get_second_cmk_arn, get_mrk_arn, + get_second_cmk_arn, get_second_mrk_arn, ) diff --git a/examples/test/test_i_basic_encryption.py b/examples/test/test_i_basic_encryption.py index f2a4fab51..aa32d61fa 100644 --- a/examples/test/test_i_basic_encryption.py +++ b/examples/test/test_i_basic_encryption.py @@ -17,7 +17,6 @@ from ..src.basic_encryption import cycle_string from .examples_test_utils import get_cmk_arn, static_plaintext - pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_basic_file_encryption_with_multiple_providers.py b/examples/test/test_i_basic_file_encryption_with_multiple_providers.py index 282a272ab..0792f4958 100644 --- a/examples/test/test_i_basic_file_encryption_with_multiple_providers.py +++ b/examples/test/test_i_basic_file_encryption_with_multiple_providers.py @@ -18,9 +18,7 @@ import pytest from ..src.basic_file_encryption_with_multiple_providers import cycle_file -from .examples_test_utils import get_cmk_arn -from .examples_test_utils import static_plaintext - +from .examples_test_utils import get_cmk_arn, static_plaintext pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_basic_file_encryption_with_raw_key_provider.py b/examples/test/test_i_basic_file_encryption_with_raw_key_provider.py index 710c0ccac..046b7f964 100644 --- a/examples/test/test_i_basic_file_encryption_with_raw_key_provider.py +++ b/examples/test/test_i_basic_file_encryption_with_raw_key_provider.py @@ -19,7 +19,6 @@ from ..src.basic_file_encryption_with_raw_key_provider import cycle_file from .examples_test_utils import static_plaintext - pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_data_key_caching_basic.py b/examples/test/test_i_data_key_caching_basic.py index 734c35692..7a30f4e53 100644 --- a/examples/test/test_i_data_key_caching_basic.py +++ b/examples/test/test_i_data_key_caching_basic.py @@ -16,7 +16,6 @@ from ..src.data_key_caching_basic import encrypt_with_caching from .examples_test_utils import get_cmk_arn - pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_discovery_kms_provider.py b/examples/test/test_i_discovery_kms_provider.py index e9a1c6e71..0f64cbf59 100644 --- a/examples/test/test_i_discovery_kms_provider.py +++ b/examples/test/test_i_discovery_kms_provider.py @@ -16,9 +16,7 @@ import pytest from ..src.discovery_kms_provider import encrypt_decrypt -from .examples_test_utils import get_cmk_arn -from .examples_test_utils import static_plaintext - +from .examples_test_utils import get_cmk_arn, static_plaintext pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_mrk_aware_kms_provider.py b/examples/test/test_i_mrk_aware_kms_provider.py index 8e7a003f8..a90101fa8 100644 --- a/examples/test/test_i_mrk_aware_kms_provider.py +++ b/examples/test/test_i_mrk_aware_kms_provider.py @@ -15,9 +15,7 @@ import pytest from ..src.mrk_aware_kms_provider import encrypt_decrypt -from .examples_test_utils import get_mrk_arn, get_second_mrk_arn -from .examples_test_utils import static_plaintext - +from .examples_test_utils import get_mrk_arn, get_second_mrk_arn, static_plaintext pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_multiple_kms_cmk.py b/examples/test/test_i_multiple_kms_cmk.py index 39369cbc6..2915a0fd7 100644 --- a/examples/test/test_i_multiple_kms_cmk.py +++ b/examples/test/test_i_multiple_kms_cmk.py @@ -16,9 +16,7 @@ import pytest from ..src.multiple_kms_cmk import encrypt_decrypt -from .examples_test_utils import get_cmk_arn, get_second_cmk_arn -from .examples_test_utils import static_plaintext - +from .examples_test_utils import get_cmk_arn, get_second_cmk_arn, static_plaintext pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_one_kms_cmk.py b/examples/test/test_i_one_kms_cmk.py index 71ce74d3d..96dd48dae 100644 --- a/examples/test/test_i_one_kms_cmk.py +++ b/examples/test/test_i_one_kms_cmk.py @@ -16,9 +16,7 @@ import pytest from ..src.one_kms_cmk import encrypt_decrypt -from .examples_test_utils import get_cmk_arn -from .examples_test_utils import static_plaintext - +from .examples_test_utils import get_cmk_arn, static_plaintext pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_one_kms_cmk_streaming_data.py b/examples/test/test_i_one_kms_cmk_streaming_data.py index b22fa4232..f0a3094d0 100644 --- a/examples/test/test_i_one_kms_cmk_streaming_data.py +++ b/examples/test/test_i_one_kms_cmk_streaming_data.py @@ -20,7 +20,6 @@ from ..src.one_kms_cmk_streaming_data import encrypt_decrypt_stream from .examples_test_utils import get_cmk_arn, static_plaintext - pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_one_kms_cmk_unsigned.py b/examples/test/test_i_one_kms_cmk_unsigned.py index 8a2758c96..41f16473d 100644 --- a/examples/test/test_i_one_kms_cmk_unsigned.py +++ b/examples/test/test_i_one_kms_cmk_unsigned.py @@ -16,9 +16,7 @@ import pytest from ..src.one_kms_cmk_unsigned import encrypt_decrypt -from .examples_test_utils import get_cmk_arn -from .examples_test_utils import static_plaintext - +from .examples_test_utils import get_cmk_arn, static_plaintext pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_set_commitment.py b/examples/test/test_i_set_commitment.py index 96247334b..c14a379bf 100644 --- a/examples/test/test_i_set_commitment.py +++ b/examples/test/test_i_set_commitment.py @@ -16,9 +16,7 @@ import pytest from ..src.set_commitment import encrypt_decrypt -from .examples_test_utils import get_cmk_arn -from .examples_test_utils import static_plaintext - +from .examples_test_utils import get_cmk_arn, static_plaintext pytestmark = [pytest.mark.examples] From 6cf01d4c6cfd0b67656a1faba6af4894675caaba Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 10:08:03 -0800 Subject: [PATCH 023/376] flake8/pylint examples --- examples/src/keyrings/hierarchical_keyring.py | 1 - examples/src/keyrings/module_.py | 2 +- examples/src/module_.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/src/keyrings/hierarchical_keyring.py b/examples/src/keyrings/hierarchical_keyring.py index acc594cc3..76aef25e0 100644 --- a/examples/src/keyrings/hierarchical_keyring.py +++ b/examples/src/keyrings/hierarchical_keyring.py @@ -35,7 +35,6 @@ def encrypt_and_decrypt_with_keyring( kms_key_id: str ): """Creates a hierarchical keyring using the provided resources, then encrypts and decrypts a string with it.""" - # 1. Instantiate the encryption SDK client. # This builds the client with the REQUIRE_ENCRYPT_REQUIRE_DECRYPT commitment policy, # which enforces that this client only encrypts using committing algorithm suites and enforces diff --git a/examples/src/keyrings/module_.py b/examples/src/keyrings/module_.py index 2f64c8e0f..d9a8c058f 100644 --- a/examples/src/keyrings/module_.py +++ b/examples/src/keyrings/module_.py @@ -1 +1 @@ -"""Should remove this.""" \ No newline at end of file +"""Should remove this.""" diff --git a/examples/src/module_.py b/examples/src/module_.py index 2f64c8e0f..d9a8c058f 100644 --- a/examples/src/module_.py +++ b/examples/src/module_.py @@ -1 +1 @@ -"""Should remove this.""" \ No newline at end of file +"""Should remove this.""" From 00cfed1f368752b872ebb25631331e04e4660893 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 10:12:24 -0800 Subject: [PATCH 024/376] reset tests --- examples/test/examples_test_utils.py | 2 +- examples/test/test_i_basic_encryption.py | 1 + .../test_i_basic_file_encryption_with_multiple_providers.py | 4 +++- .../test_i_basic_file_encryption_with_raw_key_provider.py | 1 + examples/test/test_i_data_key_caching_basic.py | 1 + examples/test/test_i_discovery_kms_provider.py | 4 +++- examples/test/test_i_mrk_aware_kms_provider.py | 4 +++- examples/test/test_i_multiple_kms_cmk.py | 4 +++- examples/test/test_i_one_kms_cmk.py | 4 +++- examples/test/test_i_one_kms_cmk_streaming_data.py | 1 + examples/test/test_i_one_kms_cmk_unsigned.py | 4 +++- examples/test/test_i_set_commitment.py | 4 +++- 12 files changed, 26 insertions(+), 8 deletions(-) diff --git a/examples/test/examples_test_utils.py b/examples/test/examples_test_utils.py index 08e8cf2f5..8a51f21c8 100644 --- a/examples/test/examples_test_utils.py +++ b/examples/test/examples_test_utils.py @@ -49,7 +49,7 @@ from integration_test_utils import ( # noqa pylint: disable=unused-import,import-error get_cmk_arn, - get_mrk_arn, get_second_cmk_arn, + get_mrk_arn, get_second_mrk_arn, ) diff --git a/examples/test/test_i_basic_encryption.py b/examples/test/test_i_basic_encryption.py index aa32d61fa..f2a4fab51 100644 --- a/examples/test/test_i_basic_encryption.py +++ b/examples/test/test_i_basic_encryption.py @@ -17,6 +17,7 @@ from ..src.basic_encryption import cycle_string from .examples_test_utils import get_cmk_arn, static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_basic_file_encryption_with_multiple_providers.py b/examples/test/test_i_basic_file_encryption_with_multiple_providers.py index 0792f4958..282a272ab 100644 --- a/examples/test/test_i_basic_file_encryption_with_multiple_providers.py +++ b/examples/test/test_i_basic_file_encryption_with_multiple_providers.py @@ -18,7 +18,9 @@ import pytest from ..src.basic_file_encryption_with_multiple_providers import cycle_file -from .examples_test_utils import get_cmk_arn, static_plaintext +from .examples_test_utils import get_cmk_arn +from .examples_test_utils import static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_basic_file_encryption_with_raw_key_provider.py b/examples/test/test_i_basic_file_encryption_with_raw_key_provider.py index 046b7f964..710c0ccac 100644 --- a/examples/test/test_i_basic_file_encryption_with_raw_key_provider.py +++ b/examples/test/test_i_basic_file_encryption_with_raw_key_provider.py @@ -19,6 +19,7 @@ from ..src.basic_file_encryption_with_raw_key_provider import cycle_file from .examples_test_utils import static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_data_key_caching_basic.py b/examples/test/test_i_data_key_caching_basic.py index 7a30f4e53..734c35692 100644 --- a/examples/test/test_i_data_key_caching_basic.py +++ b/examples/test/test_i_data_key_caching_basic.py @@ -16,6 +16,7 @@ from ..src.data_key_caching_basic import encrypt_with_caching from .examples_test_utils import get_cmk_arn + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_discovery_kms_provider.py b/examples/test/test_i_discovery_kms_provider.py index 0f64cbf59..e9a1c6e71 100644 --- a/examples/test/test_i_discovery_kms_provider.py +++ b/examples/test/test_i_discovery_kms_provider.py @@ -16,7 +16,9 @@ import pytest from ..src.discovery_kms_provider import encrypt_decrypt -from .examples_test_utils import get_cmk_arn, static_plaintext +from .examples_test_utils import get_cmk_arn +from .examples_test_utils import static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_mrk_aware_kms_provider.py b/examples/test/test_i_mrk_aware_kms_provider.py index a90101fa8..8e7a003f8 100644 --- a/examples/test/test_i_mrk_aware_kms_provider.py +++ b/examples/test/test_i_mrk_aware_kms_provider.py @@ -15,7 +15,9 @@ import pytest from ..src.mrk_aware_kms_provider import encrypt_decrypt -from .examples_test_utils import get_mrk_arn, get_second_mrk_arn, static_plaintext +from .examples_test_utils import get_mrk_arn, get_second_mrk_arn +from .examples_test_utils import static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_multiple_kms_cmk.py b/examples/test/test_i_multiple_kms_cmk.py index 2915a0fd7..39369cbc6 100644 --- a/examples/test/test_i_multiple_kms_cmk.py +++ b/examples/test/test_i_multiple_kms_cmk.py @@ -16,7 +16,9 @@ import pytest from ..src.multiple_kms_cmk import encrypt_decrypt -from .examples_test_utils import get_cmk_arn, get_second_cmk_arn, static_plaintext +from .examples_test_utils import get_cmk_arn, get_second_cmk_arn +from .examples_test_utils import static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_one_kms_cmk.py b/examples/test/test_i_one_kms_cmk.py index 96dd48dae..71ce74d3d 100644 --- a/examples/test/test_i_one_kms_cmk.py +++ b/examples/test/test_i_one_kms_cmk.py @@ -16,7 +16,9 @@ import pytest from ..src.one_kms_cmk import encrypt_decrypt -from .examples_test_utils import get_cmk_arn, static_plaintext +from .examples_test_utils import get_cmk_arn +from .examples_test_utils import static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_one_kms_cmk_streaming_data.py b/examples/test/test_i_one_kms_cmk_streaming_data.py index f0a3094d0..b22fa4232 100644 --- a/examples/test/test_i_one_kms_cmk_streaming_data.py +++ b/examples/test/test_i_one_kms_cmk_streaming_data.py @@ -20,6 +20,7 @@ from ..src.one_kms_cmk_streaming_data import encrypt_decrypt_stream from .examples_test_utils import get_cmk_arn, static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_one_kms_cmk_unsigned.py b/examples/test/test_i_one_kms_cmk_unsigned.py index 41f16473d..8a2758c96 100644 --- a/examples/test/test_i_one_kms_cmk_unsigned.py +++ b/examples/test/test_i_one_kms_cmk_unsigned.py @@ -16,7 +16,9 @@ import pytest from ..src.one_kms_cmk_unsigned import encrypt_decrypt -from .examples_test_utils import get_cmk_arn, static_plaintext +from .examples_test_utils import get_cmk_arn +from .examples_test_utils import static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_set_commitment.py b/examples/test/test_i_set_commitment.py index c14a379bf..96247334b 100644 --- a/examples/test/test_i_set_commitment.py +++ b/examples/test/test_i_set_commitment.py @@ -16,7 +16,9 @@ import pytest from ..src.set_commitment import encrypt_decrypt -from .examples_test_utils import get_cmk_arn, static_plaintext +from .examples_test_utils import get_cmk_arn +from .examples_test_utils import static_plaintext + pytestmark = [pytest.mark.examples] From 61bbb3b474bbff6b360cbea867245a2c28405659 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 11:05:51 -0800 Subject: [PATCH 025/376] extend mpl --- .github/workflows/ci_tests.yaml | 12 ++++++++++ .../keyrings/test_i_hierarchical_keyring.py | 12 ++++++++++ tox.ini | 24 +++++++++++++++++-- 3 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 examples/test/keyrings/test_i_hierarchical_keyring.py diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 9d491203c..f1701de76 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -45,12 +45,24 @@ jobs: # Enable them once we sort how to provide them. # - integ # - examples + optional_dependency: + - "" + - mpl exclude: # x86 builds are only meaningful for Windows - os: ubuntu-latest architecture: x86 - os: macos-latest architecture: x86 + # MPL is not supported on <3.11 + - python: 3.7 + optional_dependency: mpl + - python: 3.8 + optional_dependency: mpl + - python: 3.9 + optional_dependency: mpl + - python: 3.10 + optional_dependency: mpl steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 diff --git a/examples/test/keyrings/test_i_hierarchical_keyring.py b/examples/test/keyrings/test_i_hierarchical_keyring.py new file mode 100644 index 000000000..5df72383f --- /dev/null +++ b/examples/test/keyrings/test_i_hierarchical_keyring.py @@ -0,0 +1,12 @@ +"""Unit test suite for the hierarchical keyring example.""" +import pytest + +from ..src.keyrings.hierarchical_keyring import encrypt_and_decrypt_with_keyring + +pytestmark = [pytest.mark.examples] + + +def test_encrypt_and_decrypt_with_keyring(): + key_store_table_name = "KeyStoreDdbTable" + key_arn = "arn:aws:kms:us-west-2:370957321024:key/9d989aa2-2f9c-438c-a745-cc57d3ad0126" + encrypt_and_decrypt_with_keyring(key_store_table_name, key_store_table_name, key_arn) diff --git a/tox.ini b/tox.ini index 9ca7a0cd6..c90a6fcd6 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,10 @@ [tox] envlist = - py{37,38,39,310,311,312}-{local,integ,accept,examples}, nocmk, + # <3.11: run all non-MPL tests + py{37,38,39,310}-{local,integ,accept,examples}, + # >=3.11: run all MPL tests and non-MPL tests + py{311,312}-{local,integ,accept,examples}{,-mpl}, + nocmk, bandit, doc8, readme, docs, {flake8,pylint}{,-tests,-examples}, isort-check, black-check, @@ -61,12 +65,17 @@ passenv = # Pass through custom pip config file settings PIP_CONFIG_FILE sitepackages = False -deps = -rdev_requirements/test-requirements.txt +deps = + -rdev_requirements/test-requirements.txt + # install the MPL if in environment + mpl: aws-cryptographic-material-providers>=0.0.1 commands = local: {[testenv:base-command]commands} test/ -m local integ: {[testenv:base-command]commands} test/ -m integ accept: {[testenv:base-command]commands} test/ -m accept examples: {[testenv:base-command]commands} examples/test/ -m examples + # append MPL examples to base examples command + examples-mpl: {[testenv:examples]commands} examples/mpl/test/ all: {[testenv:base-command]commands} test/ examples/test/ manual: {[testenv:base-command]commands} @@ -134,6 +143,17 @@ sitepackages = {[testenv:test-upstream-requirements-base]sitepackages} recreate = {[testenv:test-upstream-requirements-base]recreate} commands = {[testenv:test-upstream-requirements-base]commands} +# Test MPL +[testenv:py311-local-mpl] +basepython = {[testenv:pylint]basepython} +deps = {[testenv:pylint]deps} +commands = + pylint \ + --rcfile=test/pylintrc \ + test/unit/ \ + test/functional/ \ + test/integration/ + # Linters [testenv:flake8] basepython = python3 From 4d53ad695908384c6e3705fc7ea5982ed7be9d8f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 11:15:27 -0800 Subject: [PATCH 026/376] mpl gha --- .github/workflows/ci_tests.yaml | 16 ++++++++-------- tox.ini | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index f1701de76..3d22ab05f 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -45,9 +45,9 @@ jobs: # Enable them once we sort how to provide them. # - integ # - examples - optional_dependency: + optional_mpl_dependency: - "" - - mpl + - -mpl exclude: # x86 builds are only meaningful for Windows - os: ubuntu-latest @@ -56,13 +56,13 @@ jobs: architecture: x86 # MPL is not supported on <3.11 - python: 3.7 - optional_dependency: mpl + optional_mpl_dependency: mpl - python: 3.8 - optional_dependency: mpl + optional_mpl_dependency: mpl - python: 3.9 - optional_dependency: mpl + optional_mpl_dependency: mpl - python: 3.10 - optional_dependency: mpl + optional_mpl_dependency: mpl steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 @@ -74,7 +74,7 @@ jobs: pip install --upgrade -r dev_requirements/ci-requirements.txt - name: run test env: - TOXENV: ${{ matrix.category }} + TOXENV: ${{ matrix.category }}${{ matrix.optional_mpl_dependency }} run: tox -- -vv upstream-py37: runs-on: ubuntu-latest @@ -114,5 +114,5 @@ jobs: pip install --upgrade -r dev_requirements/ci-requirements.txt - name: run test env: - TOXENV: ${{ matrix.category }} + TOXENV: ${{ matrix.category }}${{ matrix.optional_mpl_dependency }} run: tox -- -vv diff --git a/tox.ini b/tox.ini index c90a6fcd6..8f12ab8e7 100644 --- a/tox.ini +++ b/tox.ini @@ -75,7 +75,7 @@ commands = accept: {[testenv:base-command]commands} test/ -m accept examples: {[testenv:base-command]commands} examples/test/ -m examples # append MPL examples to base examples command - examples-mpl: {[testenv:examples]commands} examples/mpl/test/ + examples-mpl: {[testenv:base-command]commands} examples/test/ examples/mpl/test -m examples all: {[testenv:base-command]commands} test/ examples/test/ manual: {[testenv:base-command]commands} From c1736d3e91d3d78cd3412ab5c9095eccd41340d6 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 11:26:21 -0800 Subject: [PATCH 027/376] debug --- .github/workflows/ci_tests.yaml | 8 ++++---- tox.ini | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 3d22ab05f..603f54371 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -56,13 +56,13 @@ jobs: architecture: x86 # MPL is not supported on <3.11 - python: 3.7 - optional_mpl_dependency: mpl + optional_mpl_dependency: -mpl - python: 3.8 - optional_mpl_dependency: mpl + optional_mpl_dependency: -mpl - python: 3.9 - optional_mpl_dependency: mpl + optional_mpl_dependency: -mpl - python: 3.10 - optional_mpl_dependency: mpl + optional_mpl_dependency: -mpl steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 diff --git a/tox.ini b/tox.ini index 8f12ab8e7..37fbbae51 100644 --- a/tox.ini +++ b/tox.ini @@ -68,7 +68,9 @@ sitepackages = False deps = -rdev_requirements/test-requirements.txt # install the MPL if in environment - mpl: aws-cryptographic-material-providers>=0.0.1 + mpl: "aws-cryptographic-material-providers @" \ + "git+https://github.com/aws/aws-cryptographic-material-providers-library.git@" \ + "lucmcdon/python-mpl#subdirectory=AwsCryptographicMaterialProviders/runtimes/python" commands = local: {[testenv:base-command]commands} test/ -m local integ: {[testenv:base-command]commands} test/ -m integ From 9991789b79842aa0812c84bcc3c33a6b36c2f182 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 12:40:53 -0800 Subject: [PATCH 028/376] debug --- tox.ini | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 37fbbae51..ef0f6fa29 100644 --- a/tox.ini +++ b/tox.ini @@ -68,9 +68,7 @@ sitepackages = False deps = -rdev_requirements/test-requirements.txt # install the MPL if in environment - mpl: "aws-cryptographic-material-providers @" \ - "git+https://github.com/aws/aws-cryptographic-material-providers-library.git@" \ - "lucmcdon/python-mpl#subdirectory=AwsCryptographicMaterialProviders/runtimes/python" + mpl: "aws-cryptographic-material-providers @git+https://github.com/aws/aws-cryptographic-material-providers-library.git@lucmcdon/python-mpl#subdirectory=AwsCryptographicMaterialProviders/runtimes/python" commands = local: {[testenv:base-command]commands} test/ -m local integ: {[testenv:base-command]commands} test/ -m integ From a501e8f07d1bd125fd77e6b5c28710eb19402e9a Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 12:53:41 -0800 Subject: [PATCH 029/376] debug --- .../keyrings/test_i_hierarchical_keyring.py | 2 +- tox.ini | 38 ++++++++++--------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/examples/test/keyrings/test_i_hierarchical_keyring.py b/examples/test/keyrings/test_i_hierarchical_keyring.py index 5df72383f..d80bb565d 100644 --- a/examples/test/keyrings/test_i_hierarchical_keyring.py +++ b/examples/test/keyrings/test_i_hierarchical_keyring.py @@ -1,7 +1,7 @@ """Unit test suite for the hierarchical keyring example.""" import pytest -from ..src.keyrings.hierarchical_keyring import encrypt_and_decrypt_with_keyring +from ...src.keyrings.hierarchical_keyring import encrypt_and_decrypt_with_keyring pytestmark = [pytest.mark.examples] diff --git a/tox.ini b/tox.ini index ef0f6fa29..d06cbab2e 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,7 @@ envlist = isort-check, black-check, # prone to false positives vulture +ignore_base_python_conflict = true # Additional test environments: # @@ -47,28 +48,29 @@ envlist = commands = pytest --basetemp={envtmpdir} -l {posargs} [testenv] -passenv = - # Identifies AWS KMS key id to use in integration tests - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID \ - # Identifies a second AWS KMS key id to use in integration tests - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2 \ - # Identifies AWS KMS MRK key id to use in integration tests - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1 \ - # Identifies a related AWS KMS MRK key id to use in integration tests - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2 \ - # Pass through AWS credentials - AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN \ - # AWS Role access in CodeBuild is via the contaner URI - AWS_CONTAINER_CREDENTIALS_RELATIVE_URI \ - # Pass through AWS profile name (useful for local testing) - AWS_PROFILE \ - # Pass through custom pip config file settings - PIP_CONFIG_FILE +passenv = AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2,AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_SESSION_TOKEN,AWS_CONTAINER_CREDENTIALS_RELATIVE_URI,AWS_PROFILE,PIP_CONFIG_FILE +; passenv = +; # Identifies AWS KMS key id to use in integration tests +; AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID \ +; # Identifies a second AWS KMS key id to use in integration tests +; AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2 \ +; # Identifies AWS KMS MRK key id to use in integration tests +; AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1 \ +; # Identifies a related AWS KMS MRK key id to use in integration tests +; AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2 \ +; # Pass through AWS credentials +; AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN \ +; # AWS Role access in CodeBuild is via the contaner URI +; AWS_CONTAINER_CREDENTIALS_RELATIVE_URI \ +; # Pass through AWS profile name (useful for local testing) +; AWS_PROFILE \ +; # Pass through custom pip config file settings +; PIP_CONFIG_FILE sitepackages = False deps = -rdev_requirements/test-requirements.txt # install the MPL if in environment - mpl: "aws-cryptographic-material-providers @git+https://github.com/aws/aws-cryptographic-material-providers-library.git@lucmcdon/python-mpl#subdirectory=AwsCryptographicMaterialProviders/runtimes/python" + mpl: -rrequirements_mpl.txt commands = local: {[testenv:base-command]commands} test/ -m local integ: {[testenv:base-command]commands} test/ -m integ From 6eb8f82edb6139e6df62ec21957ad89ef5efa58c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 12:53:55 -0800 Subject: [PATCH 030/376] debug --- examples/src/keyrings/__init__.py | 13 +++++++++++++ examples/test/keyrings/__init__.py | 13 +++++++++++++ requirements_mpl.txt | 1 + 3 files changed, 27 insertions(+) create mode 100644 examples/src/keyrings/__init__.py create mode 100644 examples/test/keyrings/__init__.py create mode 100644 requirements_mpl.txt diff --git a/examples/src/keyrings/__init__.py b/examples/src/keyrings/__init__.py new file mode 100644 index 000000000..e8fd618b1 --- /dev/null +++ b/examples/src/keyrings/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Stub module indicator to make linter configuration simpler.""" diff --git a/examples/test/keyrings/__init__.py b/examples/test/keyrings/__init__.py new file mode 100644 index 000000000..e8fd618b1 --- /dev/null +++ b/examples/test/keyrings/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Stub module indicator to make linter configuration simpler.""" diff --git a/requirements_mpl.txt b/requirements_mpl.txt new file mode 100644 index 000000000..209e10f2c --- /dev/null +++ b/requirements_mpl.txt @@ -0,0 +1 @@ +aws-cryptographic-material-providers @ git+https://github.com/aws/aws-cryptographic-material-providers-library.git@lucmcdon/python-mpl#subdirectory=AwsCryptographicMaterialProviders/runtimes/python \ No newline at end of file From 5ccfa0cce6afdde5e598ceb35719feb109da1d48 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 13:14:29 -0800 Subject: [PATCH 031/376] codebuild mpl --- tox.ini | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tox.ini b/tox.ini index d06cbab2e..8eb141821 100644 --- a/tox.ini +++ b/tox.ini @@ -145,17 +145,6 @@ sitepackages = {[testenv:test-upstream-requirements-base]sitepackages} recreate = {[testenv:test-upstream-requirements-base]recreate} commands = {[testenv:test-upstream-requirements-base]commands} -# Test MPL -[testenv:py311-local-mpl] -basepython = {[testenv:pylint]basepython} -deps = {[testenv:pylint]deps} -commands = - pylint \ - --rcfile=test/pylintrc \ - test/unit/ \ - test/functional/ \ - test/integration/ - # Linters [testenv:flake8] basepython = python3 From 5e7ec9b94694ed4ace20aabd0ea87edd9e74f479 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 13:16:13 -0800 Subject: [PATCH 032/376] codebuild mpl --- codebuild/py311/examples_mpl.yml | 22 ++++++++++++++++++++++ codebuild/py311/integ_mpl.yml | 22 ++++++++++++++++++++++ codebuild/py312/examples_mpl.yml | 27 +++++++++++++++++++++++++++ codebuild/py312/integ_mpl.yml | 27 +++++++++++++++++++++++++++ 4 files changed, 98 insertions(+) create mode 100644 codebuild/py311/examples_mpl.yml create mode 100644 codebuild/py311/integ_mpl.yml create mode 100644 codebuild/py312/examples_mpl.yml create mode 100644 codebuild/py312/integ_mpl.yml diff --git a/codebuild/py311/examples_mpl.yml b/codebuild/py311/examples_mpl.yml new file mode 100644 index 000000000..abea2ad8c --- /dev/null +++ b/codebuild/py311/examples_mpl.yml @@ -0,0 +1,22 @@ +version: 0.2 + +env: + variables: + TOXENV: "py311-examples-mpl" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.11 + build: + commands: + - pip install "tox < 4.0" + - tox diff --git a/codebuild/py311/integ_mpl.yml b/codebuild/py311/integ_mpl.yml new file mode 100644 index 000000000..ad969c621 --- /dev/null +++ b/codebuild/py311/integ_mpl.yml @@ -0,0 +1,22 @@ +version: 0.2 + +env: + variables: + TOXENV: "py311-integ-mpl" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.11 + build: + commands: + - pip install "tox < 4.0" + - tox diff --git a/codebuild/py312/examples_mpl.yml b/codebuild/py312/examples_mpl.yml new file mode 100644 index 000000000..8ffd24964 --- /dev/null +++ b/codebuild/py312/examples_mpl.yml @@ -0,0 +1,27 @@ +version: 0.2 + +env: + variables: + TOXENV: "py312-examples-mpl" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: latest + build: + commands: + - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - + - pyenv install 3.12.0 + - pyenv local 3.12.0 + - pip install --upgrade pip + - pip install setuptools + - pip install "tox < 4.0" + - tox diff --git a/codebuild/py312/integ_mpl.yml b/codebuild/py312/integ_mpl.yml new file mode 100644 index 000000000..085cb4660 --- /dev/null +++ b/codebuild/py312/integ_mpl.yml @@ -0,0 +1,27 @@ +version: 0.2 + +env: + variables: + TOXENV: "py312-integ-mpl" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: latest + build: + commands: + - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - + - pyenv install 3.12.0 + - pyenv local 3.12.0 + - pip install --upgrade pip + - pip install setuptools + - pip install "tox < 4.0" + - tox From cc48697824accd00df18e71877a4f888d0f32125 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 13:20:54 -0800 Subject: [PATCH 033/376] codebuild mpl --- buildspec.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/buildspec.yml b/buildspec.yml index f92d203a0..c718c3df5 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -58,27 +58,51 @@ batch: buildspec: codebuild/py311/integ.yml env: image: aws/codebuild/standard:7.0 + - identifier: py311_integ_mpl + buildspec: codebuild/py311/integ_mpl.yml + env: + image: aws/codebuild/standard:7.0 - identifier: py311_examples buildspec: codebuild/py311/examples.yml env: image: aws/codebuild/standard:7.0 + - identifier: py311_examples_mpl + buildspec: codebuild/py311/examples_mpl.yml + env: + image: aws/codebuild/standard:7.0 - identifier: py311_awses_latest buildspec: codebuild/py311/awses_local.yml env: image: aws/codebuild/standard:7.0 + - identifier: py311_awses_latest_mpl + buildspec: codebuild/py311/awses_local_mpl.yml + env: + image: aws/codebuild/standard:7.0 - identifier: py312_integ buildspec: codebuild/py312/integ.yml env: image: aws/codebuild/standard:7.0 + - identifier: py312_integ_mpl + buildspec: codebuild/py312/integ_mpl.yml + env: + image: aws/codebuild/standard:7.0 - identifier: py312_examples buildspec: codebuild/py312/examples.yml env: image: aws/codebuild/standard:7.0 + - identifier: py312_examples_mpl + buildspec: codebuild/py312/examples_mpl.yml + env: + image: aws/codebuild/standard:7.0 - identifier: py312_awses_latest buildspec: codebuild/py312/awses_local.yml env: image: aws/codebuild/standard:7.0 + - identifier: py312_awses_latest_mpl + buildspec: codebuild/py312/awses_local_mpl.yml + env: + image: aws/codebuild/standard:7.0 - identifier: code_coverage buildspec: codebuild/coverage/coverage.yml From fae43d14db29780616356893d3ae7da9ce996dab Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 13:21:05 -0800 Subject: [PATCH 034/376] codebuild mpl --- codebuild/py311/awses_local_mpl.yml | 25 ++++++++++++++++++++++++ codebuild/py312/awses_local_mpl.yml | 30 +++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 codebuild/py311/awses_local_mpl.yml create mode 100644 codebuild/py312/awses_local_mpl.yml diff --git a/codebuild/py311/awses_local_mpl.yml b/codebuild/py311/awses_local_mpl.yml new file mode 100644 index 000000000..f98859b40 --- /dev/null +++ b/codebuild/py311/awses_local_mpl.yml @@ -0,0 +1,25 @@ +version: 0.2 + +env: + variables: + TOXENV: "py311-awses_local-mpl" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + python: 3.11 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - tox diff --git a/codebuild/py312/awses_local_mpl.yml b/codebuild/py312/awses_local_mpl.yml new file mode 100644 index 000000000..689d40da8 --- /dev/null +++ b/codebuild/py312/awses_local_mpl.yml @@ -0,0 +1,30 @@ +version: 0.2 + +env: + variables: + TOXENV: "py312-awses_local-mpl" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + python: latest + build: + commands: + - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - + - pyenv install 3.12.0 + - pyenv local 3.12.0 + - pip install --upgrade pip + - pip install setuptools + - pip install "tox < 4.0" + - cd test_vector_handlers + - tox From 263761678b201618c488874c040cfd7d50d0db7f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 13:32:27 -0800 Subject: [PATCH 035/376] debug --- tox.ini | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tox.ini b/tox.ini index 8eb141821..8e6cf3f34 100644 --- a/tox.ini +++ b/tox.ini @@ -48,24 +48,24 @@ ignore_base_python_conflict = true commands = pytest --basetemp={envtmpdir} -l {posargs} [testenv] -passenv = AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2,AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_SESSION_TOKEN,AWS_CONTAINER_CREDENTIALS_RELATIVE_URI,AWS_PROFILE,PIP_CONFIG_FILE -; passenv = -; # Identifies AWS KMS key id to use in integration tests -; AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID \ -; # Identifies a second AWS KMS key id to use in integration tests -; AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2 \ -; # Identifies AWS KMS MRK key id to use in integration tests -; AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1 \ -; # Identifies a related AWS KMS MRK key id to use in integration tests -; AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2 \ -; # Pass through AWS credentials -; AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN \ -; # AWS Role access in CodeBuild is via the contaner URI -; AWS_CONTAINER_CREDENTIALS_RELATIVE_URI \ -; # Pass through AWS profile name (useful for local testing) -; AWS_PROFILE \ -; # Pass through custom pip config file settings -; PIP_CONFIG_FILE +; passenv = AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2,AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_SESSION_TOKEN,AWS_CONTAINER_CREDENTIALS_RELATIVE_URI,AWS_PROFILE,PIP_CONFIG_FILE +passenv = + # Identifies AWS KMS key id to use in integration tests + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID \ + # Identifies a second AWS KMS key id to use in integration tests + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2 \ + # Identifies AWS KMS MRK key id to use in integration tests + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1 \ + # Identifies a related AWS KMS MRK key id to use in integration tests + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2 \ + # Pass through AWS credentials + AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN \ + # AWS Role access in CodeBuild is via the contaner URI + AWS_CONTAINER_CREDENTIALS_RELATIVE_URI \ + # Pass through AWS profile name (useful for local testing) + AWS_PROFILE \ + # Pass through custom pip config file settings + PIP_CONFIG_FILE sitepackages = False deps = -rdev_requirements/test-requirements.txt From 2694932f5090404eaacf9c5d442b6acac98c0246 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 13:37:16 -0800 Subject: [PATCH 036/376] debug --- tox.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 8e6cf3f34..59a1dde74 100644 --- a/tox.ini +++ b/tox.ini @@ -65,7 +65,9 @@ passenv = # Pass through AWS profile name (useful for local testing) AWS_PROFILE \ # Pass through custom pip config file settings - PIP_CONFIG_FILE + PIP_CONFIG_FILE \ + # Pass through any configured AWS region + REGION sitepackages = False deps = -rdev_requirements/test-requirements.txt From f674d3e27a2f21f3a340c9c95b3ef60fc786dd3d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 13:40:29 -0800 Subject: [PATCH 037/376] debug --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 59a1dde74..26e0e5433 100644 --- a/tox.ini +++ b/tox.ini @@ -67,7 +67,7 @@ passenv = # Pass through custom pip config file settings PIP_CONFIG_FILE \ # Pass through any configured AWS region - REGION + AWS_REGION sitepackages = False deps = -rdev_requirements/test-requirements.txt From 0b5e655b1100f4b236a14dd47ab7e7451a90eed1 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 13:44:55 -0800 Subject: [PATCH 038/376] debug --- codebuild/py311/awses_local_mpl.yml | 1 + codebuild/py311/examples_mpl.yml | 1 + codebuild/py311/integ_mpl.yml | 1 + codebuild/py312/awses_local_mpl.yml | 1 + codebuild/py312/examples_mpl.yml | 1 + codebuild/py312/integ_mpl.yml | 1 + tox.ini | 4 ++-- 7 files changed, 8 insertions(+), 2 deletions(-) diff --git a/codebuild/py311/awses_local_mpl.yml b/codebuild/py311/awses_local_mpl.yml index f98859b40..04d268d5a 100644 --- a/codebuild/py311/awses_local_mpl.yml +++ b/codebuild/py311/awses_local_mpl.yml @@ -3,6 +3,7 @@ version: 0.2 env: variables: TOXENV: "py311-awses_local-mpl" + AWS_REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/codebuild/py311/examples_mpl.yml b/codebuild/py311/examples_mpl.yml index abea2ad8c..05bdc07c0 100644 --- a/codebuild/py311/examples_mpl.yml +++ b/codebuild/py311/examples_mpl.yml @@ -3,6 +3,7 @@ version: 0.2 env: variables: TOXENV: "py311-examples-mpl" + AWS_REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/codebuild/py311/integ_mpl.yml b/codebuild/py311/integ_mpl.yml index ad969c621..e6766619c 100644 --- a/codebuild/py311/integ_mpl.yml +++ b/codebuild/py311/integ_mpl.yml @@ -3,6 +3,7 @@ version: 0.2 env: variables: TOXENV: "py311-integ-mpl" + AWS_REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/codebuild/py312/awses_local_mpl.yml b/codebuild/py312/awses_local_mpl.yml index 689d40da8..a504696ec 100644 --- a/codebuild/py312/awses_local_mpl.yml +++ b/codebuild/py312/awses_local_mpl.yml @@ -3,6 +3,7 @@ version: 0.2 env: variables: TOXENV: "py312-awses_local-mpl" + AWS_REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/codebuild/py312/examples_mpl.yml b/codebuild/py312/examples_mpl.yml index 8ffd24964..a947c67b3 100644 --- a/codebuild/py312/examples_mpl.yml +++ b/codebuild/py312/examples_mpl.yml @@ -3,6 +3,7 @@ version: 0.2 env: variables: TOXENV: "py312-examples-mpl" + AWS_REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/codebuild/py312/integ_mpl.yml b/codebuild/py312/integ_mpl.yml index 085cb4660..3cf473d08 100644 --- a/codebuild/py312/integ_mpl.yml +++ b/codebuild/py312/integ_mpl.yml @@ -3,6 +3,7 @@ version: 0.2 env: variables: TOXENV: "py312-integ-mpl" + AWS_REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/tox.ini b/tox.ini index 26e0e5433..8b50e6b01 100644 --- a/tox.ini +++ b/tox.ini @@ -77,9 +77,9 @@ commands = local: {[testenv:base-command]commands} test/ -m local integ: {[testenv:base-command]commands} test/ -m integ accept: {[testenv:base-command]commands} test/ -m accept - examples: {[testenv:base-command]commands} examples/test/ -m examples + examples: {[testenv:base-command]commands} examples/test/ -m examples --ignore examples/test/keyrings/ # append MPL examples to base examples command - examples-mpl: {[testenv:base-command]commands} examples/test/ examples/mpl/test -m examples + examples-mpl: {[testenv:base-command]commands} examples/test/ -m examples all: {[testenv:base-command]commands} test/ examples/test/ manual: {[testenv:base-command]commands} From 831df1713823dd185883c87dfcab0e521b5fefcd Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 13:52:00 -0800 Subject: [PATCH 039/376] debug --- codebuild/py311/awses_local_mpl.yml | 2 +- codebuild/py311/examples_mpl.yml | 2 +- codebuild/py311/integ_mpl.yml | 2 +- codebuild/py312/awses_local_mpl.yml | 2 +- codebuild/py312/examples_mpl.yml | 2 +- codebuild/py312/integ_mpl.yml | 2 +- test_vector_handlers/tox.ini | 4 +++- tox.ini | 2 -- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/codebuild/py311/awses_local_mpl.yml b/codebuild/py311/awses_local_mpl.yml index 04d268d5a..859931aa3 100644 --- a/codebuild/py311/awses_local_mpl.yml +++ b/codebuild/py311/awses_local_mpl.yml @@ -3,7 +3,7 @@ version: 0.2 env: variables: TOXENV: "py311-awses_local-mpl" - AWS_REGION: "us-west-2" + REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/codebuild/py311/examples_mpl.yml b/codebuild/py311/examples_mpl.yml index 05bdc07c0..e29472507 100644 --- a/codebuild/py311/examples_mpl.yml +++ b/codebuild/py311/examples_mpl.yml @@ -3,7 +3,7 @@ version: 0.2 env: variables: TOXENV: "py311-examples-mpl" - AWS_REGION: "us-west-2" + REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/codebuild/py311/integ_mpl.yml b/codebuild/py311/integ_mpl.yml index e6766619c..694bc0850 100644 --- a/codebuild/py311/integ_mpl.yml +++ b/codebuild/py311/integ_mpl.yml @@ -3,7 +3,7 @@ version: 0.2 env: variables: TOXENV: "py311-integ-mpl" - AWS_REGION: "us-west-2" + REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/codebuild/py312/awses_local_mpl.yml b/codebuild/py312/awses_local_mpl.yml index a504696ec..f39bf8760 100644 --- a/codebuild/py312/awses_local_mpl.yml +++ b/codebuild/py312/awses_local_mpl.yml @@ -3,7 +3,7 @@ version: 0.2 env: variables: TOXENV: "py312-awses_local-mpl" - AWS_REGION: "us-west-2" + REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/codebuild/py312/examples_mpl.yml b/codebuild/py312/examples_mpl.yml index a947c67b3..d6bc3f440 100644 --- a/codebuild/py312/examples_mpl.yml +++ b/codebuild/py312/examples_mpl.yml @@ -3,7 +3,7 @@ version: 0.2 env: variables: TOXENV: "py312-examples-mpl" - AWS_REGION: "us-west-2" + REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/codebuild/py312/integ_mpl.yml b/codebuild/py312/integ_mpl.yml index 3cf473d08..8ffda4bd0 100644 --- a/codebuild/py312/integ_mpl.yml +++ b/codebuild/py312/integ_mpl.yml @@ -3,7 +3,7 @@ version: 0.2 env: variables: TOXENV: "py312-integ-mpl" - AWS_REGION: "us-west-2" + REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index 643750cd2..b6711361e 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -2,7 +2,7 @@ envlist = # The test vectors depend on new features now, # so until release we can only effectively test the local version of the ESDK. - py{37,38,39,310}-awses_local, + py{37,38,39,310}-awses_local{,-mpl}, # 1.2.0 and 1.2.max are being difficult because of attrs bandit, doc8, readme, {flake8,pylint}{,-tests}, @@ -48,6 +48,8 @@ passenv = sitepackages = False deps = -rtest/requirements.txt + # install the MPL if in environment + mpl: -rrequirements_mpl.txt .. commands = {[testenv:base-command]commands} diff --git a/tox.ini b/tox.ini index 8b50e6b01..903ea5170 100644 --- a/tox.ini +++ b/tox.ini @@ -66,8 +66,6 @@ passenv = AWS_PROFILE \ # Pass through custom pip config file settings PIP_CONFIG_FILE \ - # Pass through any configured AWS region - AWS_REGION sitepackages = False deps = -rdev_requirements/test-requirements.txt From 477e3a097da42fddfded9894d48fad6923d96144 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 13:56:57 -0800 Subject: [PATCH 040/376] debug --- examples/src/keyrings/hierarchical_keyring.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/src/keyrings/hierarchical_keyring.py b/examples/src/keyrings/hierarchical_keyring.py index 76aef25e0..a99728b6e 100644 --- a/examples/src/keyrings/hierarchical_keyring.py +++ b/examples/src/keyrings/hierarchical_keyring.py @@ -48,8 +48,8 @@ def encrypt_and_decrypt_with_keyring( ) # 2. Create boto3 clients for DynamoDB and KMS. - ddb_client = boto3.client('dynamodb') - kms_client = boto3.client('kms') + ddb_client = boto3.client('dynamodb', region_name="us-west-2") + kms_client = boto3.client('kms', region_name="us-west-2") # 3. Configure your KeyStore resource. # This SHOULD be the same configuration that you used From 166c5ab6ff339f8fd6dafa6ffea071704eb484a5 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 14:24:13 -0800 Subject: [PATCH 041/376] debug --- test_vector_handlers/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index b6711361e..7004080e3 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -49,7 +49,7 @@ sitepackages = False deps = -rtest/requirements.txt # install the MPL if in environment - mpl: -rrequirements_mpl.txt + mpl: -r../requirements_mpl.txt .. commands = {[testenv:base-command]commands} From 7ac88805b863add77147cabedc08c5968480ad55 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 14:35:06 -0800 Subject: [PATCH 042/376] debug --- tox.ini | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 903ea5170..4202979a4 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,6 @@ envlist = isort-check, black-check, # prone to false positives vulture -ignore_base_python_conflict = true # Additional test environments: # @@ -65,7 +64,7 @@ passenv = # Pass through AWS profile name (useful for local testing) AWS_PROFILE \ # Pass through custom pip config file settings - PIP_CONFIG_FILE \ + PIP_CONFIG_FILE sitepackages = False deps = -rdev_requirements/test-requirements.txt From 7e3ca151e85eb44d4d125f172f32ed70d3b6523b Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 14:41:54 -0800 Subject: [PATCH 043/376] fix --- codebuild/py312/awses_local_mpl.yml | 2 +- codebuild/py312/examples_mpl.yml | 2 +- codebuild/py312/integ_mpl.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/codebuild/py312/awses_local_mpl.yml b/codebuild/py312/awses_local_mpl.yml index f39bf8760..db25f4f57 100644 --- a/codebuild/py312/awses_local_mpl.yml +++ b/codebuild/py312/awses_local_mpl.yml @@ -22,7 +22,7 @@ phases: build: commands: - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - - - pyenv install 3.12.0 + - pyenv install --skip-existing 3.12.0 - pyenv local 3.12.0 - pip install --upgrade pip - pip install setuptools diff --git a/codebuild/py312/examples_mpl.yml b/codebuild/py312/examples_mpl.yml index d6bc3f440..ff2168cd5 100644 --- a/codebuild/py312/examples_mpl.yml +++ b/codebuild/py312/examples_mpl.yml @@ -20,7 +20,7 @@ phases: build: commands: - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - - - pyenv install 3.12.0 + - pyenv install --skip-existing 3.12.0 - pyenv local 3.12.0 - pip install --upgrade pip - pip install setuptools diff --git a/codebuild/py312/integ_mpl.yml b/codebuild/py312/integ_mpl.yml index 8ffda4bd0..553f41e8a 100644 --- a/codebuild/py312/integ_mpl.yml +++ b/codebuild/py312/integ_mpl.yml @@ -20,7 +20,7 @@ phases: build: commands: - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - - - pyenv install 3.12.0 + - pyenv install --skip-existing 3.12.0 - pyenv local 3.12.0 - pip install --upgrade pip - pip install setuptools From 4c6a1d00711352236d63b2adc44081dded3a1a65 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 15:08:53 -0800 Subject: [PATCH 044/376] fix --- src/aws_encryption_sdk/cmm_handler.py | 158 ----------------- src/aws_encryption_sdk/materials_handlers.py | 170 ------------------- 2 files changed, 328 deletions(-) delete mode 100644 src/aws_encryption_sdk/cmm_handler.py delete mode 100644 src/aws_encryption_sdk/materials_handlers.py diff --git a/src/aws_encryption_sdk/cmm_handler.py b/src/aws_encryption_sdk/cmm_handler.py deleted file mode 100644 index 5bac15b87..000000000 --- a/src/aws_encryption_sdk/cmm_handler.py +++ /dev/null @@ -1,158 +0,0 @@ -"""Retrieves encryption/decryption materials from an underlying materials provider.""" - -# These dependencies are only loaded if you install the MPL. -try: - # pylint seems to struggle with this conditional import - # pylint: disable=unused-import - from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException - from aws_cryptographic_materialproviders.mpl.models import ( - AlgorithmSuiteIdESDK, - CommitmentPolicyESDK, - DecryptMaterialsInput, - DecryptMaterialsOutput, - EncryptedDataKey as MPL_EncryptedDataKey, - GetEncryptionMaterialsInput, - GetEncryptionMaterialsOutput, - ) - from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager - -except ImportError: - pass - -from typing import List - -from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError -from aws_encryption_sdk.identifiers import CommitmentPolicy -from aws_encryption_sdk.materials_handlers import DecryptionMaterialsHandler, EncryptionMaterialsHandler -from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest -from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager -from aws_encryption_sdk.structures import EncryptedDataKey as Native_EncryptedDataKey - - -# TODO-MPL Should this implement interface..? seems like yes since it implements all of interface methods -class CMMHandler(CryptoMaterialsManager): - """ - In instances where encryption materials may be provided by either - an implementation of the native - `aws_encryption_sdk.materials_managers.base.CryptoMaterialsManager` - or an implementation of the MPL's - `aws_cryptographic_materialproviders.mpl.references.ICryptographicMaterialsManager`, - this provides the correct materials based on the underlying materials manager. - """ - - native_cmm: CryptoMaterialsManager - mpl_cmm: 'ICryptographicMaterialsManager' - - def _is_using_native_cmm(self): - return hasattr(self, "native_cmm") and not hasattr(self, "mpl_cmm") - - def __init__( - self, - cmm: 'CryptoMaterialsManager | ICryptographicMaterialsManager' - ): - """ - Create DecryptionMaterialsHandler. - :param cmm: Underlying cryptographic materials manager - """ - if isinstance(cmm, CryptoMaterialsManager): - self.native_cmm = cmm - elif isinstance(cmm, ICryptographicMaterialsManager): - self.mpl_cmm = cmm - else: - raise ValueError(f"Invalid CMM passed to CMMHandler. cmm: {cmm}") - - def get_encryption_materials( - self, - request: EncryptionMaterialsRequest - ) -> EncryptionMaterialsHandler: - """ - Returns an EncryptionMaterialsHandler for the configured CMM. - :param request: Request for encryption materials - """ - if self._is_using_native_cmm(): - return EncryptionMaterialsHandler(self.native_cmm.get_encryption_materials(request)) - else: - try: - mpl_input: GetEncryptionMaterialsInput = CMMHandler._native_to_mpl_get_encryption_materials( - request - ) - mpl_output: GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) - return EncryptionMaterialsHandler(mpl_output.encryption_materials) - except AwsCryptographicMaterialProvidersException as mpl_exception: - # Wrap MPL error into the ESDK error type - # so customers only have to catch ESDK error types. - raise AWSEncryptionSDKClientError(mpl_exception) - - @staticmethod - def _native_to_mpl_get_encryption_materials( - request: EncryptionMaterialsRequest - ) -> 'GetEncryptionMaterialsInput': - output: GetEncryptionMaterialsInput = GetEncryptionMaterialsInput( - encryption_context=request.encryption_context, - commitment_policy=CMMHandler._native_to_mpl_commmitment_policy( - request.commitment_policy - ), - max_plaintext_length=request.plaintext_length, - ) - return output - - @staticmethod - def _native_to_mpl_commmitment_policy( - native_commitment_policy: CommitmentPolicy - ) -> 'CommitmentPolicyESDK': - if native_commitment_policy == CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT: - return CommitmentPolicyESDK(value="FORBID_ENCRYPT_ALLOW_DECRYPT") - elif native_commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT: - return CommitmentPolicyESDK(value="REQUIRE_ENCRYPT_ALLOW_DECRYPT") - elif native_commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT: - return CommitmentPolicyESDK(value="REQUIRE_ENCRYPT_REQUIRE_DECRYPT") - else: - raise ValueError(f"Invalid native_commitment_policy: {native_commitment_policy}") - - def decrypt_materials( - self, - request: DecryptionMaterialsRequest - ) -> DecryptionMaterialsHandler: - """ - Returns a DecryptionMaterialsHandler for the configured CMM. - :param request: Request for decryption materials - """ - if self._is_using_native_cmm(): - return DecryptionMaterialsHandler(self.native_cmm.decrypt_materials(request)) - else: - try: - mpl_input: 'DecryptMaterialsInput' = \ - CMMHandler._create_mpl_decrypt_materials_input_from_request(request) - mpl_output: 'DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(mpl_input) - return DecryptionMaterialsHandler(mpl_output.decryption_materials) - except AwsCryptographicMaterialProvidersException as mpl_exception: - # Wrap MPL error into the ESDK error type - # so customers only have to catch ESDK error types. - raise AWSEncryptionSDKClientError(mpl_exception) - - @staticmethod - def _native_algorithm_id_to_mpl_algorithm_id(native_algorithm_id: str) -> 'AlgorithmSuiteIdESDK': - # MPL algorithm suite ID = hexstr(native_algorithm_id) padded to 4 digits post-`x`. - return AlgorithmSuiteIdESDK(f"{native_algorithm_id:#0{6}x}") - - @staticmethod - def _create_mpl_decrypt_materials_input_from_request( - request: DecryptionMaterialsRequest - ) -> 'DecryptMaterialsInput': - key_blob_list: List[Native_EncryptedDataKey] = request.encrypted_data_keys - list_edks = [MPL_EncryptedDataKey( - key_provider_id=key_blob.key_provider.provider_id, - key_provider_info=key_blob.key_provider.key_info, - ciphertext=key_blob.encrypted_data_key, - ) for key_blob in key_blob_list] - output: DecryptMaterialsInput = DecryptMaterialsInput( - algorithm_suite_id=CMMHandler._native_algorithm_id_to_mpl_algorithm_id( - request.algorithm.algorithm_id - ), - commitment_policy=CMMHandler._native_to_mpl_commmitment_policy( - request.commitment_policy - ), - encrypted_data_keys=list_edks, - encryption_context=request.encryption_context, - ) - return output diff --git a/src/aws_encryption_sdk/materials_handlers.py b/src/aws_encryption_sdk/materials_handlers.py deleted file mode 100644 index 57f54144e..000000000 --- a/src/aws_encryption_sdk/materials_handlers.py +++ /dev/null @@ -1,170 +0,0 @@ -"""Provides encryption/decryption materials from an underlying materials provider.""" -# These dependencies are only loaded if you install the MPL. -try: - from aws_cryptographic_materialproviders.mpl.models import ( - DecryptionMaterials as MPL_DecryptionMaterials, - EncryptedDataKey as MPL_EncryptedDataKey, - EncryptionMaterials as MPL_EncryptionMaterials, - ) -except ImportError: - pass - -from typing import Dict, List, Set - -from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite -from aws_encryption_sdk.materials_managers import ( - DecryptionMaterials as Native_DecryptionMaterials, - EncryptionMaterials as Native_EncryptionMaterials, -) -from aws_encryption_sdk.structures import DataKey, EncryptedDataKey as Native_EncryptedDataKey, MasterKeyInfo - - -def _mpl_algorithm_id_to_native_algorithm_id(mpl_algorithm_id: str): - # MPL algorithm suite ID == hex(native algorithm suite ID) - return int(mpl_algorithm_id, 16) - - -class EncryptionMaterialsHandler: - """ - In instances where encryption materials may be provided by either - the native `aws_encryption_sdk.materials_managers.EncryptionMaterials` - or the MPL's `aws_cryptographic_materialproviders.mpl.models.EncryptionMaterials`, - this provides the correct materials based on the configured materials provider. - """ - - native_materials: Native_EncryptionMaterials - mpl_materials: 'MPL_EncryptionMaterials' - - def __init__( - self, - materials: 'Native_EncryptionMaterials | MPL_EncryptionMaterials' - ): - """ - Create EncryptionMaterialsHandler. - :param materials: Underlying encryption materials - """ - if isinstance(materials, Native_EncryptionMaterials): - self.native_materials = materials - elif isinstance(materials, MPL_EncryptionMaterials): - self.mpl_materials = materials - else: - raise ValueError(f"Invalid EncryptionMaterials passed to EncryptionMaterialsHandler.\ - materials: {materials}") - - @property - def algorithm(self) -> Algorithm: - """Materials' native Algorithm.""" - if hasattr(self, "native_materials"): - return self.native_materials.algorithm - else: - return AlgorithmSuite.get_by_id( - _mpl_algorithm_id_to_native_algorithm_id( - self.mpl_materials.algorithm_suite.id.value - ) - ) - - @property - def encryption_context(self) -> Dict[str, str]: - """Materials' encryption context.""" - if hasattr(self, "native_materials"): - return self.native_materials.encryption_context - else: - return self.mpl_materials.encryption_context - - @property - def encrypted_data_keys(self) -> List[Native_EncryptedDataKey]: - """Materials' encrypted data keys.""" - if hasattr(self, "native_materials"): - return self.native_materials.encrypted_data_keys - else: - mpl_edk_list: List[MPL_EncryptedDataKey] = self.mpl_materials.encrypted_data_keys - key_blob_list: Set[Native_EncryptedDataKey] = {Native_EncryptedDataKey( - key_provider=MasterKeyInfo( - provider_id=mpl_edk.key_provider_id, - key_info=mpl_edk.key_provider_info, - ), - encrypted_data_key=mpl_edk.ciphertext, - ) for mpl_edk in mpl_edk_list} - return key_blob_list - - @property - def data_encryption_key(self) -> DataKey: - """Materials' data encryption key.""" - if hasattr(self, "native_materials"): - return self.native_materials.data_encryption_key - else: - # TODO-MPL This impl is probably wrong, but works for for now - # If this works for all features, great! Remove this comment before launch. - # Otherwise, fix the implementation. - mpl_dek = self.mpl_materials.plaintext_data_key - return DataKey( - # key_provider is unused, but the return type is DataKey - key_provider=MasterKeyInfo( - provider_id="", - key_info=b'' - ), - data_key=mpl_dek, - encrypted_data_key=b'', # No encrypted DEK - ) - - @property - def signing_key(self) -> bytes: - """Materials' signing key.""" - if hasattr(self, "native_materials"): - return self.native_materials.signing_key - else: - return self.mpl_materials.signing_key - - -class DecryptionMaterialsHandler: - """ - In instances where decryption materials may be provided by either - the native `aws_encryption_sdk.materials_managers.DecryptionMaterials` - or the MPL's `aws_cryptographic_materialproviders.mpl.models.DecryptionMaterials`, - this provides the correct materials based on the configured materials provider. - """ - - native_materials: Native_DecryptionMaterials - mpl_materials: 'MPL_DecryptionMaterials' - - def __init__( - self, - materials: 'Native_DecryptionMaterials | MPL_DecryptionMaterials' - ): - """ - Create DecryptionMaterialsHandler. - :param materials: Underlying decryption materials - """ - if isinstance(materials, Native_DecryptionMaterials): - self.native_materials = materials - elif isinstance(materials, MPL_DecryptionMaterials): - self.mpl_materials = materials - else: - raise ValueError(f"Invalid DecryptionMaterials passed to DecryptionMaterialsHandler.\ - materials: {materials}") - - @property - def data_key(self) -> DataKey: - """Materials' data key.""" - if hasattr(self, "native_materials"): - return self.native_materials.data_key - else: - # TODO-MPL This impl is probably wrong, but works for for now - # If this works for all features, great! Remove this comment before launch. - # Otherwise, fix the implementation. - return DataKey( - key_provider=MasterKeyInfo( - provider_id="", - key_info=b'' - ), - data_key=self.mpl_materials.plaintext_data_key, - encrypted_data_key=b'', - ) - - @property - def verification_key(self) -> bytes: - """Materials' verification key.""" - if hasattr(self, "native_materials"): - return self.native_materials.verification_key - else: - return self.mpl_materials.verification_key From e2e185844e0ac83f07dce346156ed9a7ab693275 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 15:09:10 -0800 Subject: [PATCH 045/376] fix --- src/aws_encryption_sdk/mpl/cmm_handler.py | 158 ++++++++++++++++ .../mpl/materials_handlers.py | 170 ++++++++++++++++++ 2 files changed, 328 insertions(+) create mode 100644 src/aws_encryption_sdk/mpl/cmm_handler.py create mode 100644 src/aws_encryption_sdk/mpl/materials_handlers.py diff --git a/src/aws_encryption_sdk/mpl/cmm_handler.py b/src/aws_encryption_sdk/mpl/cmm_handler.py new file mode 100644 index 000000000..5dfaab973 --- /dev/null +++ b/src/aws_encryption_sdk/mpl/cmm_handler.py @@ -0,0 +1,158 @@ +"""Retrieves encryption/decryption materials from an underlying materials provider.""" + +# These dependencies are only loaded if you install the MPL. +try: + # pylint seems to struggle with this conditional import + # pylint: disable=unused-import + from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException + from aws_cryptographic_materialproviders.mpl.models import ( + AlgorithmSuiteIdESDK, + CommitmentPolicyESDK, + DecryptMaterialsInput, + DecryptMaterialsOutput, + EncryptedDataKey as MPL_EncryptedDataKey, + GetEncryptionMaterialsInput, + GetEncryptionMaterialsOutput, + ) + from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager + +except ImportError: + pass + +from typing import List + +from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError +from aws_encryption_sdk.identifiers import CommitmentPolicy +from aws_encryption_sdk.mpl.materials_handlers import DecryptionMaterialsHandler, EncryptionMaterialsHandler +from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest +from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager +from aws_encryption_sdk.structures import EncryptedDataKey as Native_EncryptedDataKey + + +# TODO-MPL Should this implement interface...? seems like yes since it implements all of interface methods +class CMMHandler(CryptoMaterialsManager): + """ + In instances where encryption materials may be provided by either + an implementation of the native + `aws_encryption_sdk.materials_managers.base.CryptoMaterialsManager` + or an implementation of the MPL's + `aws_cryptographic_materialproviders.mpl.references.ICryptographicMaterialsManager`, + this provides the correct materials based on the underlying materials manager. + """ + + native_cmm: CryptoMaterialsManager + mpl_cmm: 'ICryptographicMaterialsManager' + + def _is_using_native_cmm(self): + return hasattr(self, "native_cmm") and not hasattr(self, "mpl_cmm") + + def __init__( + self, + cmm: 'CryptoMaterialsManager | ICryptographicMaterialsManager' + ): + """ + Create DecryptionMaterialsHandler. + :param cmm: Underlying cryptographic materials manager + """ + if isinstance(cmm, CryptoMaterialsManager): + self.native_cmm = cmm + elif isinstance(cmm, ICryptographicMaterialsManager): + self.mpl_cmm = cmm + else: + raise ValueError(f"Invalid CMM passed to CMMHandler. cmm: {cmm}") + + def get_encryption_materials( + self, + request: EncryptionMaterialsRequest + ) -> EncryptionMaterialsHandler: + """ + Returns an EncryptionMaterialsHandler for the configured CMM. + :param request: Request for encryption materials + """ + if self._is_using_native_cmm(): + return EncryptionMaterialsHandler(self.native_cmm.get_encryption_materials(request)) + else: + try: + mpl_input: GetEncryptionMaterialsInput = CMMHandler._native_to_mpl_get_encryption_materials( + request + ) + mpl_output: GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) + return EncryptionMaterialsHandler(mpl_output.encryption_materials) + except AwsCryptographicMaterialProvidersException as mpl_exception: + # Wrap MPL error into the ESDK error type + # so customers only have to catch ESDK error types. + raise AWSEncryptionSDKClientError(mpl_exception) + + @staticmethod + def _native_to_mpl_get_encryption_materials( + request: EncryptionMaterialsRequest + ) -> 'GetEncryptionMaterialsInput': + output: GetEncryptionMaterialsInput = GetEncryptionMaterialsInput( + encryption_context=request.encryption_context, + commitment_policy=CMMHandler._native_to_mpl_commmitment_policy( + request.commitment_policy + ), + max_plaintext_length=request.plaintext_length, + ) + return output + + @staticmethod + def _native_to_mpl_commmitment_policy( + native_commitment_policy: CommitmentPolicy + ) -> 'CommitmentPolicyESDK': + if native_commitment_policy == CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT: + return CommitmentPolicyESDK(value="FORBID_ENCRYPT_ALLOW_DECRYPT") + elif native_commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT: + return CommitmentPolicyESDK(value="REQUIRE_ENCRYPT_ALLOW_DECRYPT") + elif native_commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT: + return CommitmentPolicyESDK(value="REQUIRE_ENCRYPT_REQUIRE_DECRYPT") + else: + raise ValueError(f"Invalid native_commitment_policy: {native_commitment_policy}") + + def decrypt_materials( + self, + request: DecryptionMaterialsRequest + ) -> DecryptionMaterialsHandler: + """ + Returns a DecryptionMaterialsHandler for the configured CMM. + :param request: Request for decryption materials + """ + if self._is_using_native_cmm(): + return DecryptionMaterialsHandler(self.native_cmm.decrypt_materials(request)) + else: + try: + mpl_input: 'DecryptMaterialsInput' = \ + CMMHandler._create_mpl_decrypt_materials_input_from_request(request) + mpl_output: 'DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(mpl_input) + return DecryptionMaterialsHandler(mpl_output.decryption_materials) + except AwsCryptographicMaterialProvidersException as mpl_exception: + # Wrap MPL error into the ESDK error type + # so customers only have to catch ESDK error types. + raise AWSEncryptionSDKClientError(mpl_exception) + + @staticmethod + def _native_algorithm_id_to_mpl_algorithm_id(native_algorithm_id: str) -> 'AlgorithmSuiteIdESDK': + # MPL algorithm suite ID = hexstr(native_algorithm_id) padded to 4 digits post-`x`. + return AlgorithmSuiteIdESDK(f"{native_algorithm_id:#0{6}x}") + + @staticmethod + def _create_mpl_decrypt_materials_input_from_request( + request: DecryptionMaterialsRequest + ) -> 'DecryptMaterialsInput': + key_blob_list: List[Native_EncryptedDataKey] = request.encrypted_data_keys + list_edks = [MPL_EncryptedDataKey( + key_provider_id=key_blob.key_provider.provider_id, + key_provider_info=key_blob.key_provider.key_info, + ciphertext=key_blob.encrypted_data_key, + ) for key_blob in key_blob_list] + output: DecryptMaterialsInput = DecryptMaterialsInput( + algorithm_suite_id=CMMHandler._native_algorithm_id_to_mpl_algorithm_id( + request.algorithm.algorithm_id + ), + commitment_policy=CMMHandler._native_to_mpl_commmitment_policy( + request.commitment_policy + ), + encrypted_data_keys=list_edks, + encryption_context=request.encryption_context, + ) + return output diff --git a/src/aws_encryption_sdk/mpl/materials_handlers.py b/src/aws_encryption_sdk/mpl/materials_handlers.py new file mode 100644 index 000000000..57f54144e --- /dev/null +++ b/src/aws_encryption_sdk/mpl/materials_handlers.py @@ -0,0 +1,170 @@ +"""Provides encryption/decryption materials from an underlying materials provider.""" +# These dependencies are only loaded if you install the MPL. +try: + from aws_cryptographic_materialproviders.mpl.models import ( + DecryptionMaterials as MPL_DecryptionMaterials, + EncryptedDataKey as MPL_EncryptedDataKey, + EncryptionMaterials as MPL_EncryptionMaterials, + ) +except ImportError: + pass + +from typing import Dict, List, Set + +from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite +from aws_encryption_sdk.materials_managers import ( + DecryptionMaterials as Native_DecryptionMaterials, + EncryptionMaterials as Native_EncryptionMaterials, +) +from aws_encryption_sdk.structures import DataKey, EncryptedDataKey as Native_EncryptedDataKey, MasterKeyInfo + + +def _mpl_algorithm_id_to_native_algorithm_id(mpl_algorithm_id: str): + # MPL algorithm suite ID == hex(native algorithm suite ID) + return int(mpl_algorithm_id, 16) + + +class EncryptionMaterialsHandler: + """ + In instances where encryption materials may be provided by either + the native `aws_encryption_sdk.materials_managers.EncryptionMaterials` + or the MPL's `aws_cryptographic_materialproviders.mpl.models.EncryptionMaterials`, + this provides the correct materials based on the configured materials provider. + """ + + native_materials: Native_EncryptionMaterials + mpl_materials: 'MPL_EncryptionMaterials' + + def __init__( + self, + materials: 'Native_EncryptionMaterials | MPL_EncryptionMaterials' + ): + """ + Create EncryptionMaterialsHandler. + :param materials: Underlying encryption materials + """ + if isinstance(materials, Native_EncryptionMaterials): + self.native_materials = materials + elif isinstance(materials, MPL_EncryptionMaterials): + self.mpl_materials = materials + else: + raise ValueError(f"Invalid EncryptionMaterials passed to EncryptionMaterialsHandler.\ + materials: {materials}") + + @property + def algorithm(self) -> Algorithm: + """Materials' native Algorithm.""" + if hasattr(self, "native_materials"): + return self.native_materials.algorithm + else: + return AlgorithmSuite.get_by_id( + _mpl_algorithm_id_to_native_algorithm_id( + self.mpl_materials.algorithm_suite.id.value + ) + ) + + @property + def encryption_context(self) -> Dict[str, str]: + """Materials' encryption context.""" + if hasattr(self, "native_materials"): + return self.native_materials.encryption_context + else: + return self.mpl_materials.encryption_context + + @property + def encrypted_data_keys(self) -> List[Native_EncryptedDataKey]: + """Materials' encrypted data keys.""" + if hasattr(self, "native_materials"): + return self.native_materials.encrypted_data_keys + else: + mpl_edk_list: List[MPL_EncryptedDataKey] = self.mpl_materials.encrypted_data_keys + key_blob_list: Set[Native_EncryptedDataKey] = {Native_EncryptedDataKey( + key_provider=MasterKeyInfo( + provider_id=mpl_edk.key_provider_id, + key_info=mpl_edk.key_provider_info, + ), + encrypted_data_key=mpl_edk.ciphertext, + ) for mpl_edk in mpl_edk_list} + return key_blob_list + + @property + def data_encryption_key(self) -> DataKey: + """Materials' data encryption key.""" + if hasattr(self, "native_materials"): + return self.native_materials.data_encryption_key + else: + # TODO-MPL This impl is probably wrong, but works for for now + # If this works for all features, great! Remove this comment before launch. + # Otherwise, fix the implementation. + mpl_dek = self.mpl_materials.plaintext_data_key + return DataKey( + # key_provider is unused, but the return type is DataKey + key_provider=MasterKeyInfo( + provider_id="", + key_info=b'' + ), + data_key=mpl_dek, + encrypted_data_key=b'', # No encrypted DEK + ) + + @property + def signing_key(self) -> bytes: + """Materials' signing key.""" + if hasattr(self, "native_materials"): + return self.native_materials.signing_key + else: + return self.mpl_materials.signing_key + + +class DecryptionMaterialsHandler: + """ + In instances where decryption materials may be provided by either + the native `aws_encryption_sdk.materials_managers.DecryptionMaterials` + or the MPL's `aws_cryptographic_materialproviders.mpl.models.DecryptionMaterials`, + this provides the correct materials based on the configured materials provider. + """ + + native_materials: Native_DecryptionMaterials + mpl_materials: 'MPL_DecryptionMaterials' + + def __init__( + self, + materials: 'Native_DecryptionMaterials | MPL_DecryptionMaterials' + ): + """ + Create DecryptionMaterialsHandler. + :param materials: Underlying decryption materials + """ + if isinstance(materials, Native_DecryptionMaterials): + self.native_materials = materials + elif isinstance(materials, MPL_DecryptionMaterials): + self.mpl_materials = materials + else: + raise ValueError(f"Invalid DecryptionMaterials passed to DecryptionMaterialsHandler.\ + materials: {materials}") + + @property + def data_key(self) -> DataKey: + """Materials' data key.""" + if hasattr(self, "native_materials"): + return self.native_materials.data_key + else: + # TODO-MPL This impl is probably wrong, but works for for now + # If this works for all features, great! Remove this comment before launch. + # Otherwise, fix the implementation. + return DataKey( + key_provider=MasterKeyInfo( + provider_id="", + key_info=b'' + ), + data_key=self.mpl_materials.plaintext_data_key, + encrypted_data_key=b'', + ) + + @property + def verification_key(self) -> bytes: + """Materials' verification key.""" + if hasattr(self, "native_materials"): + return self.native_materials.verification_key + else: + return self.mpl_materials.verification_key From c790011ed67a16cfb6790ad56941c12099552bbe Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 15:12:52 -0800 Subject: [PATCH 046/376] mpl --- src/aws_encryption_sdk/streaming_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index afe9987ff..680784b1a 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -24,7 +24,7 @@ import six import aws_encryption_sdk.internal.utils -from aws_encryption_sdk.cmm_handler import CMMHandler +from aws_encryption_sdk.mpl.cmm_handler import CMMHandler from aws_encryption_sdk.exceptions import ( ActionNotAllowedError, AWSEncryptionSDKClientError, From 33ace5897bb532fbc15942c3317bee8e90215adc Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 15:18:27 -0800 Subject: [PATCH 047/376] fix --- src/aws_encryption_sdk/mpl/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 src/aws_encryption_sdk/mpl/__init__.py diff --git a/src/aws_encryption_sdk/mpl/__init__.py b/src/aws_encryption_sdk/mpl/__init__.py new file mode 100644 index 000000000..41497cc20 --- /dev/null +++ b/src/aws_encryption_sdk/mpl/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Modules related to the MPL.""" From cbf2cdf79d107bb371ed76646307f9e3ec038d9e Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 15:32:30 -0800 Subject: [PATCH 048/376] fix --- src/aws_encryption_sdk/mpl/cmm_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/mpl/cmm_handler.py b/src/aws_encryption_sdk/mpl/cmm_handler.py index 5dfaab973..2ccbcb5f3 100644 --- a/src/aws_encryption_sdk/mpl/cmm_handler.py +++ b/src/aws_encryption_sdk/mpl/cmm_handler.py @@ -29,7 +29,7 @@ from aws_encryption_sdk.structures import EncryptedDataKey as Native_EncryptedDataKey -# TODO-MPL Should this implement interface...? seems like yes since it implements all of interface methods +# TODO-MPL Should this implement interface..? seems like yes since it implements all of interface methods class CMMHandler(CryptoMaterialsManager): """ In instances where encryption materials may be provided by either From b2594771818c265c6f6ce31a56a4339b86840f38 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 15:43:03 -0800 Subject: [PATCH 049/376] fix --- src/aws_encryption_sdk/mpl/cmm_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/mpl/cmm_handler.py b/src/aws_encryption_sdk/mpl/cmm_handler.py index 2ccbcb5f3..5dfaab973 100644 --- a/src/aws_encryption_sdk/mpl/cmm_handler.py +++ b/src/aws_encryption_sdk/mpl/cmm_handler.py @@ -29,7 +29,7 @@ from aws_encryption_sdk.structures import EncryptedDataKey as Native_EncryptedDataKey -# TODO-MPL Should this implement interface..? seems like yes since it implements all of interface methods +# TODO-MPL Should this implement interface...? seems like yes since it implements all of interface methods class CMMHandler(CryptoMaterialsManager): """ In instances where encryption materials may be provided by either From 9d52cf2ad37bf69fcc42bee1375365ea4e19bf15 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 8 Feb 2024 14:18:52 -0800 Subject: [PATCH 050/376] . --- src/aws_encryption_sdk/streaming_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 680784b1a..04d44334a 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -915,7 +915,7 @@ def _read_header(self): self.verifier = None else: # MPL verification key is NOT key bytes, it is bytes of the compressed point - # TODO-MPL: clean this up, least-privilege violation + # TODO-MPL: clean this up, least-privilege violation. if (isinstance(self.config.materials_manager, CMMHandler) and hasattr(self.config.materials_manager, "mpl_cmm")): self.verifier = Verifier.from_encoded_point( From 31b761616f7741293c8e6a33e2e9174c758f7c70 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 9 Feb 2024 09:45:18 -0800 Subject: [PATCH 051/376] debug tox mpl keystore env --- codebuild/py311/examples_mpl.yml | 6 ++++-- codebuild/py312/examples_mpl.yml | 5 +++-- tox.ini | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/codebuild/py311/examples_mpl.yml b/codebuild/py311/examples_mpl.yml index e29472507..b1afa5016 100644 --- a/codebuild/py311/examples_mpl.yml +++ b/codebuild/py311/examples_mpl.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - TOXENV: "py311-examples-mpl" + # No TOXENV; examples using the MPL switch envs REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f @@ -20,4 +20,6 @@ phases: build: commands: - pip install "tox < 4.0" - - tox + - tox -e py311-examples-mpl + - tox -e py311-examples-mpl-keystore + diff --git a/codebuild/py312/examples_mpl.yml b/codebuild/py312/examples_mpl.yml index ff2168cd5..cf53585b4 100644 --- a/codebuild/py312/examples_mpl.yml +++ b/codebuild/py312/examples_mpl.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - TOXENV: "py312-examples-mpl" + # No TOXENV; examples using the MPL switch envs REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f @@ -25,4 +25,5 @@ phases: - pip install --upgrade pip - pip install setuptools - pip install "tox < 4.0" - - tox + - tox -e py312-examples-mpl + - tox -e py312-examples-mpl-keystore diff --git a/tox.ini b/tox.ini index 4202979a4..d618cb030 100644 --- a/tox.ini +++ b/tox.ini @@ -75,8 +75,8 @@ commands = integ: {[testenv:base-command]commands} test/ -m integ accept: {[testenv:base-command]commands} test/ -m accept examples: {[testenv:base-command]commands} examples/test/ -m examples --ignore examples/test/keyrings/ - # append MPL examples to base examples command - examples-mpl: {[testenv:base-command]commands} examples/test/ -m examples + # MPL keyring examples require a special IAM role; run these separately under a separate set of permissions + examples-mpl-keyring: {[testenv:base-command]commands} examples/test/keyrings -m examples all: {[testenv:base-command]commands} test/ examples/test/ manual: {[testenv:base-command]commands} From 353b8cfc944b437bfb86f24390c0019693b5b76f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 9 Feb 2024 09:51:54 -0800 Subject: [PATCH 052/376] debug tox mpl keystore env --- codebuild/py311/examples_mpl.yml | 2 +- codebuild/py312/examples_mpl.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/codebuild/py311/examples_mpl.yml b/codebuild/py311/examples_mpl.yml index b1afa5016..c5b1e1872 100644 --- a/codebuild/py311/examples_mpl.yml +++ b/codebuild/py311/examples_mpl.yml @@ -21,5 +21,5 @@ phases: commands: - pip install "tox < 4.0" - tox -e py311-examples-mpl - - tox -e py311-examples-mpl-keystore + - tox -e py311-examples-mpl-keyring diff --git a/codebuild/py312/examples_mpl.yml b/codebuild/py312/examples_mpl.yml index cf53585b4..97a19ad50 100644 --- a/codebuild/py312/examples_mpl.yml +++ b/codebuild/py312/examples_mpl.yml @@ -26,4 +26,4 @@ phases: - pip install setuptools - pip install "tox < 4.0" - tox -e py312-examples-mpl - - tox -e py312-examples-mpl-keystore + - tox -e py312-examples-mpl-keyring From fb64d950a31a82169cbedaf289367dc3001a0c68 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 9 Feb 2024 09:57:15 -0800 Subject: [PATCH 053/376] debug tox mpl keystore env --- codebuild/py311/examples_mpl.yml | 8 ++++++++ codebuild/py312/examples_mpl.yml | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/codebuild/py311/examples_mpl.yml b/codebuild/py311/examples_mpl.yml index c5b1e1872..f7d705923 100644 --- a/codebuild/py311/examples_mpl.yml +++ b/codebuild/py311/examples_mpl.yml @@ -21,5 +21,13 @@ phases: commands: - pip install "tox < 4.0" - tox -e py311-examples-mpl + # Assume special role + - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-Py311ExamplesMpl") + - export TMP_ROLE + - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') + - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') + - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') + - aws sts get-caller-identity + # Run special role-specific examples - tox -e py311-examples-mpl-keyring diff --git a/codebuild/py312/examples_mpl.yml b/codebuild/py312/examples_mpl.yml index 97a19ad50..c95f606e6 100644 --- a/codebuild/py312/examples_mpl.yml +++ b/codebuild/py312/examples_mpl.yml @@ -26,4 +26,12 @@ phases: - pip install setuptools - pip install "tox < 4.0" - tox -e py312-examples-mpl + # Assume special role + - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-Py311ExamplesMpl") + - export TMP_ROLE + - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') + - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') + - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') + - aws sts get-caller-identity + # Run special role-specific examples - tox -e py312-examples-mpl-keyring From 916ae8e00b6195ed91dcbfa0448aa0e2c23c49bc Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 9 Feb 2024 10:19:26 -0800 Subject: [PATCH 054/376] debug tox mpl keystore env --- .../internal/crypto/authentication.py | 9 +++++++-- src/aws_encryption_sdk/streaming_client.py | 17 ++++++++++++++--- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/aws_encryption_sdk/internal/crypto/authentication.py b/src/aws_encryption_sdk/internal/crypto/authentication.py index f90ac77e0..a6446981e 100644 --- a/src/aws_encryption_sdk/internal/crypto/authentication.py +++ b/src/aws_encryption_sdk/internal/crypto/authentication.py @@ -68,7 +68,7 @@ class Signer(_PrehashingAuthenticator): """ @classmethod - def from_key_bytes(cls, algorithm, key_bytes): + def from_key_bytes(cls, algorithm, key_bytes, encoding=serialization.Encoding.DER): """Builds a `Signer` from an algorithm suite and a raw signing key. :param algorithm: Algorithm on which to base signer @@ -76,7 +76,12 @@ def from_key_bytes(cls, algorithm, key_bytes): :param bytes key_bytes: Raw signing key :rtype: aws_encryption_sdk.internal.crypto.Signer """ - key = serialization.load_der_private_key(data=key_bytes, password=None, backend=default_backend()) + if encoding == serialization.Encoding.DER: + key = serialization.load_der_private_key(data=key_bytes, password=None, backend=default_backend()) + elif serialization.Encoding.PEM: + key = serialization.load_pem_private_key(data=key_bytes, password=None, backend=default_backend()) + else: + raise ValueError(f"Unsupported encoding for Signer: {encoding}") return cls(algorithm, key) def key_bytes(self): diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 04d44334a..e514337f5 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -23,6 +23,8 @@ import attr import six +from cryptography.hazmat.primitives import serialization + import aws_encryption_sdk.internal.utils from aws_encryption_sdk.mpl.cmm_handler import CMMHandler from aws_encryption_sdk.exceptions import ( @@ -555,9 +557,18 @@ def _prep_message(self): if self._encryption_materials.signing_key is None: self.signer = None else: - self.signer = Signer.from_key_bytes( - algorithm=self._encryption_materials.algorithm, key_bytes=self._encryption_materials.signing_key - ) + # MPL verification key is NOT key bytes, it is bytes of the compressed point + # TODO-MPL: clean this up, least-privilege violation. + if (isinstance(self.config.materials_manager, CMMHandler) + and hasattr(self.config.materials_manager, "mpl_cmm")): + self.signer = Signer.from_key_bytes( + algorithm=self._encryption_materials.algorithm, key_bytes=self._encryption_materials.signing_key, + encoding=serialization.Encoding.PEM, + ) + else: + self.signer = Signer.from_key_bytes( + algorithm=self._encryption_materials.algorithm, key_bytes=self._encryption_materials.signing_key + ) aws_encryption_sdk.internal.utils.validate_frame_length( frame_length=self.config.frame_length, algorithm=self._encryption_materials.algorithm ) From 222b13549febbbfe144e7f865a89203b4f3789fd Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 9 Feb 2024 10:38:14 -0800 Subject: [PATCH 055/376] debug tox mpl keystore env --- codebuild/py311/examples_mpl.yml | 2 +- codebuild/py312/examples_mpl.yml | 2 +- tox.ini | 7 +++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/codebuild/py311/examples_mpl.yml b/codebuild/py311/examples_mpl.yml index f7d705923..f8f2a6a01 100644 --- a/codebuild/py311/examples_mpl.yml +++ b/codebuild/py311/examples_mpl.yml @@ -29,5 +29,5 @@ phases: - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') - aws sts get-caller-identity # Run special role-specific examples - - tox -e py311-examples-mpl-keyring + - tox -e py311-mplexamples-mpl diff --git a/codebuild/py312/examples_mpl.yml b/codebuild/py312/examples_mpl.yml index c95f606e6..ba0660024 100644 --- a/codebuild/py312/examples_mpl.yml +++ b/codebuild/py312/examples_mpl.yml @@ -34,4 +34,4 @@ phases: - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') - aws sts get-caller-identity # Run special role-specific examples - - tox -e py312-examples-mpl-keyring + - tox -e py312-mplexamples-mpl diff --git a/tox.ini b/tox.ini index d618cb030..61d65b11d 100644 --- a/tox.ini +++ b/tox.ini @@ -2,8 +2,11 @@ envlist = # <3.11: run all non-MPL tests py{37,38,39,310}-{local,integ,accept,examples}, - # >=3.11: run all MPL tests and non-MPL tests + # >=3.11: run all tests with MPL installed and without MPL installed + # The `-mpl` suffix tells tox to install the MPL py{311,312}-{local,integ,accept,examples}{,-mpl}, + # >=3.11: run ONLY the MPL-specific tests (requires a special IAM role) + py{311,312}-{mplexamples}-mpl nocmk, bandit, doc8, readme, docs, {flake8,pylint}{,-tests,-examples}, @@ -76,7 +79,7 @@ commands = accept: {[testenv:base-command]commands} test/ -m accept examples: {[testenv:base-command]commands} examples/test/ -m examples --ignore examples/test/keyrings/ # MPL keyring examples require a special IAM role; run these separately under a separate set of permissions - examples-mpl-keyring: {[testenv:base-command]commands} examples/test/keyrings -m examples + mplexamples: {[testenv:base-command]commands} examples/test/keyrings -m examples all: {[testenv:base-command]commands} test/ examples/test/ manual: {[testenv:base-command]commands} From cab60167b76eb142258b97f2493879ad0028e818 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 9 Feb 2024 16:03:12 -0800 Subject: [PATCH 056/376] some unit tests --- src/aws_encryption_sdk/mpl/cmm_handler.py | 2 + .../mpl/materials_handlers.py | 4 +- .../mpl/mpl_import_handler.py | 14 +++ src/aws_encryption_sdk/streaming_client.py | 16 +-- test/unit/mpl/README.md | 1 + test/unit/mpl/test_cmm_handler.py | 111 ++++++++++++++++++ test/unit/test_mpl_import_handler.py | 34 ++++++ test/unit/test_streaming_client_configs.py | 3 + test/unit/test_streaming_client_mpl_import.py | 42 +++++++ tox.ini | 6 +- 10 files changed, 221 insertions(+), 12 deletions(-) create mode 100644 src/aws_encryption_sdk/mpl/mpl_import_handler.py create mode 100644 test/unit/mpl/README.md create mode 100644 test/unit/mpl/test_cmm_handler.py create mode 100644 test/unit/test_mpl_import_handler.py create mode 100644 test/unit/test_streaming_client_mpl_import.py diff --git a/src/aws_encryption_sdk/mpl/cmm_handler.py b/src/aws_encryption_sdk/mpl/cmm_handler.py index 5dfaab973..1f6c9ff41 100644 --- a/src/aws_encryption_sdk/mpl/cmm_handler.py +++ b/src/aws_encryption_sdk/mpl/cmm_handler.py @@ -76,7 +76,9 @@ def get_encryption_materials( mpl_input: GetEncryptionMaterialsInput = CMMHandler._native_to_mpl_get_encryption_materials( request ) + print(f"mpl_input: {mpl_input}") mpl_output: GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) + print(f"mpl_output: {mpl_output}") return EncryptionMaterialsHandler(mpl_output.encryption_materials) except AwsCryptographicMaterialProvidersException as mpl_exception: # Wrap MPL error into the ESDK error type diff --git a/src/aws_encryption_sdk/mpl/materials_handlers.py b/src/aws_encryption_sdk/mpl/materials_handlers.py index 57f54144e..df5b57d53 100644 --- a/src/aws_encryption_sdk/mpl/materials_handlers.py +++ b/src/aws_encryption_sdk/mpl/materials_handlers.py @@ -48,8 +48,8 @@ def __init__( elif isinstance(materials, MPL_EncryptionMaterials): self.mpl_materials = materials else: - raise ValueError(f"Invalid EncryptionMaterials passed to EncryptionMaterialsHandler.\ - materials: {materials}") + raise ValueError("Invalid EncryptionMaterials passed to EncryptionMaterialsHandler. " \ + f"materials: {materials}") @property def algorithm(self) -> Algorithm: diff --git a/src/aws_encryption_sdk/mpl/mpl_import_handler.py b/src/aws_encryption_sdk/mpl/mpl_import_handler.py new file mode 100644 index 000000000..40669da1e --- /dev/null +++ b/src/aws_encryption_sdk/mpl/mpl_import_handler.py @@ -0,0 +1,14 @@ +def has_mpl(): + """Returns True if the aws_cryptographic_materialproviders library is installed, False otherwise.""" + try: + _import_mpl() + return True + except ImportError: + return False + +def _import_mpl(): + """Private wrapper for import to help with unit test coverage. + + This is not directly tested. + """ + import aws_cryptographic_materialproviders \ No newline at end of file diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index e514337f5..106121377 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -71,15 +71,15 @@ from aws_encryption_sdk.materials_managers.default import DefaultCryptoMaterialsManager from aws_encryption_sdk.structures import MessageHeader -try: +from aws_encryption_sdk.mpl import mpl_import_handler +if mpl_import_handler.has_mpl(): from aws_cryptographic_materialproviders.mpl.client import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig from aws_cryptographic_materialproviders.mpl.models import CreateDefaultCryptographicMaterialsManagerInput from aws_cryptographic_materialproviders.mpl.references import IKeyring - - HAS_MPL = True -except ImportError: - HAS_MPL = False + _HAS_MPL = True +else: + _HAS_MPL = False _LOGGER = logging.getLogger(__name__) @@ -146,7 +146,7 @@ class _ClientConfig(object): # pylint: disable=too-many-instance-attributes key_provider = attr.ib( hash=True, default=None, validator=attr.validators.optional(attr.validators.instance_of(MasterKeyProvider)) ) - if HAS_MPL: + if _HAS_MPL: keyring = attr.ib( hash=True, default=None, validator=attr.validators.optional(attr.validators.instance_of(IKeyring)) ) @@ -194,9 +194,9 @@ def _no_mpl_attrs_post_init(self): def __attrs_post_init__(self): """Normalize inputs to crypto material manager.""" - if HAS_MPL: + if _HAS_MPL: self._has_mpl_attrs_post_init() - elif not HAS_MPL: + else: self._no_mpl_attrs_post_init() diff --git a/test/unit/mpl/README.md b/test/unit/mpl/README.md new file mode 100644 index 000000000..839feb7a2 --- /dev/null +++ b/test/unit/mpl/README.md @@ -0,0 +1 @@ +Tests in this file REQUIRE the aws-cryptographic-material-providers module to be installed in order to run. \ No newline at end of file diff --git a/test/unit/mpl/test_cmm_handler.py b/test/unit/mpl/test_cmm_handler.py new file mode 100644 index 000000000..45b49ed91 --- /dev/null +++ b/test/unit/mpl/test_cmm_handler.py @@ -0,0 +1,111 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Test suite to verify the mpl_import_handler module handles importing the MPL correctly.""" +import pytest +from mock import MagicMock, patch + +from aws_encryption_sdk.mpl.cmm_handler import CMMHandler + +from aws_encryption_sdk.mpl.materials_handlers import DecryptionMaterialsHandler, EncryptionMaterialsHandler +from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest +from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager +from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager + +from aws_encryption_sdk.materials_managers import ( + DecryptionMaterials as Native_DecryptionMaterials, + EncryptionMaterials as Native_EncryptionMaterials, +) + +from aws_cryptographic_materialproviders.mpl.models import ( + AlgorithmSuiteIdESDK, + CommitmentPolicyESDK, + DecryptMaterialsInput, + DecryptMaterialsOutput, + EncryptedDataKey as MPL_EncryptedDataKey, + GetEncryptionMaterialsInput, + GetEncryptionMaterialsOutput, + ) + +from aws_cryptographic_materialproviders.mpl.models import ( + DecryptionMaterials as MPL_DecryptionMaterials, + EncryptedDataKey as MPL_EncryptedDataKey, + EncryptionMaterials as MPL_EncryptionMaterials, + ) + +mock_native_cmm = MagicMock(__class__=CryptoMaterialsManager) +mock_mpl_cmm = MagicMock(__class__=ICryptographicMaterialsManager) +mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) +mock_encryption_materials_handler = MagicMock(__class__=EncryptionMaterialsHandler) +mock_native_encryption_materials = MagicMock(__class__=Native_EncryptionMaterials) +mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) + +pytestmark = [pytest.mark.unit, pytest.mark.local] + + +def test_GIVEN_native_CMM_WHEN_create_CMMHandler_THEN_is_using_native_cmm_returns_True(): + cmm_handler = CMMHandler(cmm=mock_native_cmm) + assert cmm_handler._is_using_native_cmm() + + +def test_GIVEN_mpl_CMM_WHEN_create_CMMHandler_THEN_is_using_native_cmm_returns_False(): + cmm_handler = CMMHandler(cmm=mock_mpl_cmm) + assert not cmm_handler._is_using_native_cmm() + + +def test_GIVEN_unknown_CMM_WHEN_create_CMMHandler_THEN_raise_ValueError(): + with pytest.raises(ValueError): + CMMHandler(cmm="not a CMM") + + +@patch.object(mock_native_cmm, "get_encryption_materials") +def test_GIVEN_native_CMM_WHEN_get_encryption_materials_THEN_return_native_encryption_materials(mock_get_encryption_materials): + # Mock: native_cmm.get_encryption_materials returns mock native encryption materials + mock_get_encryption_materials.return_value = mock_native_encryption_materials + + cmm_handler = CMMHandler(cmm=mock_native_cmm) + test = cmm_handler.get_encryption_materials(mock_encryption_materials_request) + + # Verify cmm_handler returns EncryptionMaterialsHandler + assert isinstance(test, EncryptionMaterialsHandler) + # Verify returned EncryptionMaterialsHandler uses the output of `get_encryption_materials` + assert test.native_materials == mock_native_encryption_materials + # Verify we actually called `get_encryption_materials` + mock_native_cmm.get_encryption_materials.assert_called_once_with(mock_encryption_materials_request) + + +@patch.object(mock_mpl_cmm, "get_encryption_materials") +@patch("aws_encryption_sdk.mpl.cmm_handler.CMMHandler._native_to_mpl_get_encryption_materials") +def test_GIVEN_mpl_CMM_WHEN_get_encryption_materials_THEN_return_mpl_encryption_materials( + mock_native_to_mpl_get_encryption_materials, + mock_get_encryption_materials, + +): + # Mock: mpl_cmm.get_encryption_materials returns mock MPL encryption materials + mock_get_encryption_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) + mock_get_encryption_materials_output.encryption_materials = mock_mpl_encryption_materials + mock_get_encryption_materials.return_value = mock_get_encryption_materials_output + + # Mock: CMMHandler._native_to_mpl_get_encryption_materials creates a GetEncryptionMaterialsInput + mock_get_encryption_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) + mock_native_to_mpl_get_encryption_materials.return_value = mock_get_encryption_materials_input + + cmm_handler = CMMHandler(cmm=mock_mpl_cmm) + test = cmm_handler.get_encryption_materials(mock_encryption_materials_request) + + # Verify cmm_handler returns EncryptionMaterialsHandler + assert isinstance(test, EncryptionMaterialsHandler) + # Verify returned EncryptionMaterialsHandler uses the output of `get_encryption_materials` + assert test.mpl_materials == mock_mpl_encryption_materials + # Verify we actually called `get_encryption_materials` + mock_mpl_cmm.get_encryption_materials.assert_called_once_with(mock_get_encryption_materials_input) + diff --git a/test/unit/test_mpl_import_handler.py b/test/unit/test_mpl_import_handler.py new file mode 100644 index 000000000..c17c358b4 --- /dev/null +++ b/test/unit/test_mpl_import_handler.py @@ -0,0 +1,34 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Test suite to verify the mpl_import_handler module handles importing the MPL correctly.""" +import pytest +from mock import patch + +from aws_encryption_sdk.mpl import mpl_import_handler + +pytestmark = [pytest.mark.unit, pytest.mark.local] + +@patch("aws_encryption_sdk.mpl.mpl_import_handler._import_mpl") +def test_GIVEN_import_mpl_succeeds_WHEN_call_has_mpl_THEN_return_True(import_mock): + # Mock a successful import of `aws_cryptographic_material_providers` + import_mock.return_value = None # No exception means successful import + + assert mpl_import_handler.has_mpl() is True + +@patch("aws_encryption_sdk.mpl.mpl_import_handler._import_mpl") +def test_GIVEN_import_mpl_fails_WHEN_call_has_mpl_THEN_return_False(import_mock): + # Mock not having a `aws_cryptographic_material_providers` module, + # even if it is installed in the Python environment + import_mock.side_effect = ImportError() + + assert not mpl_import_handler.has_mpl() \ No newline at end of file diff --git a/test/unit/test_streaming_client_configs.py b/test/unit/test_streaming_client_configs.py index 426f8f85f..80b7fdb28 100644 --- a/test/unit/test_streaming_client_configs.py +++ b/test/unit/test_streaming_client_configs.py @@ -154,3 +154,6 @@ def test_client_config_converts(kwargs, stream_type): assert isinstance(test.source, stream_type) if test.key_provider is not None: assert isinstance(test.materials_manager, DefaultCryptoMaterialsManager) + +def test_GIVEN_has_mpl_WHEN_import_THEN_imports_mpl_modules(): + \ No newline at end of file diff --git a/test/unit/test_streaming_client_mpl_import.py b/test/unit/test_streaming_client_mpl_import.py new file mode 100644 index 000000000..8ce016caf --- /dev/null +++ b/test/unit/test_streaming_client_mpl_import.py @@ -0,0 +1,42 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Unit test suite to validate aws_encryption_sdk.streaming_client MPL import logic.""" +import io + +import pytest +from mock import patch +from importlib import reload + +import aws_encryption_sdk.streaming_client + +pytestmark = [pytest.mark.unit, pytest.mark.local] + +@patch.object(aws_encryption_sdk.streaming_client.mpl_import_handler, "has_mpl") +def test_GIVEN_has_mpl_returns_True_WHEN_import_streaming_client_THEN_imports_mpl_modules(has_mpl_mock): + has_mpl_mock.return_value = True + + # Reload module given the mock + reload(aws_encryption_sdk.streaming_client) + + assert hasattr(aws_encryption_sdk.streaming_client, "_HAS_MPL") + assert aws_encryption_sdk.streaming_client._HAS_MPL is True + +@patch.object(aws_encryption_sdk.streaming_client.mpl_import_handler, "has_mpl") +def test_GIVEN_has_mpl_returns_False_WHEN_import_streaming_client_THEN_does_not_import_mpl_modules(has_mpl_mock): + has_mpl_mock.return_value = False + + # Reload module given the mock + reload(aws_encryption_sdk.streaming_client) + + assert hasattr(aws_encryption_sdk.streaming_client, "_HAS_MPL") + assert aws_encryption_sdk.streaming_client._HAS_MPL is False \ No newline at end of file diff --git a/tox.ini b/tox.ini index 61d65b11d..3cc7017e1 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,8 @@ envlist = # The `-mpl` suffix tells tox to install the MPL py{311,312}-{local,integ,accept,examples}{,-mpl}, # >=3.11: run ONLY the MPL-specific tests (requires a special IAM role) - py{311,312}-{mplexamples}-mpl + # the extra `-mpl` suffix tells tox to install the MPL + py{311,312}-mpl{local,examples}-mpl nocmk, bandit, doc8, readme, docs, {flake8,pylint}{,-tests,-examples}, @@ -74,7 +75,8 @@ deps = # install the MPL if in environment mpl: -rrequirements_mpl.txt commands = - local: {[testenv:base-command]commands} test/ -m local + local: {[testenv:base-command]commands} test/ -m local --ignore test/mpl/ + local: {[testenv:base-command]commands} test/mpl/ -m local integ: {[testenv:base-command]commands} test/ -m integ accept: {[testenv:base-command]commands} test/ -m accept examples: {[testenv:base-command]commands} examples/test/ -m examples --ignore examples/test/keyrings/ From a7416b18f9afa367e5053f0bd936735da7ec6e01 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 13 Feb 2024 09:26:04 -0800 Subject: [PATCH 057/376] add mpl coverage --- buildspec.yml | 2 ++ codebuild/coverage/coverage_mpl.yml | 14 ++++++++++++++ tox.ini | 6 +++++- 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 codebuild/coverage/coverage_mpl.yml diff --git a/buildspec.yml b/buildspec.yml index c718c3df5..3d70c144d 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -106,6 +106,8 @@ batch: - identifier: code_coverage buildspec: codebuild/coverage/coverage.yml + - identifier: code_coverage_mpl + buildspec: codebuild/coverage/coverage_mpl.yml - identifier: compliance buildspec: codebuild/compliance/compliance.yml diff --git a/codebuild/coverage/coverage_mpl.yml b/codebuild/coverage/coverage_mpl.yml new file mode 100644 index 000000000..5dcc65382 --- /dev/null +++ b/codebuild/coverage/coverage_mpl.yml @@ -0,0 +1,14 @@ +version: 0.2 + +env: + variables: + TOXENV: "mplcoverage-mpl" + +phases: + install: + runtime-versions: + python: latest + build: + commands: + - pip install "tox < 4.0" + - tox diff --git a/tox.ini b/tox.ini index 3cc7017e1..7aacb047a 100644 --- a/tox.ini +++ b/tox.ini @@ -6,6 +6,8 @@ envlist = # The `-mpl` suffix tells tox to install the MPL py{311,312}-{local,integ,accept,examples}{,-mpl}, # >=3.11: run ONLY the MPL-specific tests (requires a special IAM role) + # the `mpl` prefix runs only MPL-specific tests + # (non-MPL-specific tests are run from the line above) # the extra `-mpl` suffix tells tox to install the MPL py{311,312}-mpl{local,examples}-mpl nocmk, @@ -87,7 +89,9 @@ commands = # Run code coverage on the unit tests [testenv:coverage] -commands = {[testenv:base-command]commands} --cov aws_encryption_sdk test/ -m local +commands = {[testenv:base-command]commands} --cov aws_encryption_sdk test/ -m local --ignore test/unit/mpl/ +[testenv:mplcoverage] +commands = {[testenv:base-command]commands} --cov aws_encryption_sdk test/unit/mpl/ -m local # Verify that local tests work without environment variables present [testenv:nocmk] From 7b3dc5fc303afad5b0d3519b12de833135e3d326 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 13 Feb 2024 09:30:19 -0800 Subject: [PATCH 058/376] . --- tox.ini | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index 7aacb047a..194b4d412 100644 --- a/tox.ini +++ b/tox.ini @@ -77,14 +77,14 @@ deps = # install the MPL if in environment mpl: -rrequirements_mpl.txt commands = - local: {[testenv:base-command]commands} test/ -m local --ignore test/mpl/ - local: {[testenv:base-command]commands} test/mpl/ -m local - integ: {[testenv:base-command]commands} test/ -m integ - accept: {[testenv:base-command]commands} test/ -m accept + local: {[testenv:base-command]commands} test/ -m local --ignore test/unit/mpl/ + mpllocal: {[testenv:base-command]commands} test/unit/mpl/ -m local + integ: {[testenv:base-command]commands} test/ -m integ --ignore test/unit/mpl/ + accept: {[testenv:base-command]commands} test/ -m accept --ignore test/unit/mpl/ examples: {[testenv:base-command]commands} examples/test/ -m examples --ignore examples/test/keyrings/ # MPL keyring examples require a special IAM role; run these separately under a separate set of permissions mplexamples: {[testenv:base-command]commands} examples/test/keyrings -m examples - all: {[testenv:base-command]commands} test/ examples/test/ + all: {[testenv:base-command]commands} test/ examples/test/ --ignore test/unit/mpl/ manual: {[testenv:base-command]commands} # Run code coverage on the unit tests From 7a5e4eb9e6055576759d45950bd66ace921d65aa Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 13 Feb 2024 09:33:38 -0800 Subject: [PATCH 059/376] . --- test/unit/test_streaming_client_configs.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/unit/test_streaming_client_configs.py b/test/unit/test_streaming_client_configs.py index 80b7fdb28..426f8f85f 100644 --- a/test/unit/test_streaming_client_configs.py +++ b/test/unit/test_streaming_client_configs.py @@ -154,6 +154,3 @@ def test_client_config_converts(kwargs, stream_type): assert isinstance(test.source, stream_type) if test.key_provider is not None: assert isinstance(test.materials_manager, DefaultCryptoMaterialsManager) - -def test_GIVEN_has_mpl_WHEN_import_THEN_imports_mpl_modules(): - \ No newline at end of file From 0649995f59e6b3de7ad68d5ff7497a44dc021c31 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 13 Feb 2024 15:46:47 -0800 Subject: [PATCH 060/376] mock imports --- src/aws_encryption_sdk/streaming_client.py | 2 +- test/unit/test_streaming_client_mpl_import.py | 11 +++++++++++ test/unit/test_streaming_client_stream_decryptor.py | 2 +- tox.ini | 2 +- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 106121377..044626c7f 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -1095,7 +1095,7 @@ def close(self): """Closes out the stream.""" _LOGGER.debug("Closing stream") if not hasattr(self, "footer"): - raise SerializationError("Footer not read, message may be corrupted or data key may be incorrect") + raise SerializationError("Footer not read") super(StreamDecryptor, self).close() diff --git a/test/unit/test_streaming_client_mpl_import.py b/test/unit/test_streaming_client_mpl_import.py index 8ce016caf..f71f337b7 100644 --- a/test/unit/test_streaming_client_mpl_import.py +++ b/test/unit/test_streaming_client_mpl_import.py @@ -17,6 +17,8 @@ from mock import patch from importlib import reload +from mock import Mock + import aws_encryption_sdk.streaming_client pytestmark = [pytest.mark.unit, pytest.mark.local] @@ -25,6 +27,15 @@ def test_GIVEN_has_mpl_returns_True_WHEN_import_streaming_client_THEN_imports_mpl_modules(has_mpl_mock): has_mpl_mock.return_value = True + # Mock any imports used in the try/catch block + # If more imports are added there, then this needs to be expanded + # This unit test should pass even if the MPL is not installed + import sys + sys.modules['aws_cryptographic_materialproviders.mpl.client'] = Mock() + sys.modules['aws_cryptographic_materialproviders.mpl.config'] = Mock() + sys.modules['aws_cryptographic_materialproviders.mpl.models'] = Mock() + sys.modules['aws_cryptographic_materialproviders.mpl.references'] = Mock() + # Reload module given the mock reload(aws_encryption_sdk.streaming_client) diff --git a/test/unit/test_streaming_client_stream_decryptor.py b/test/unit/test_streaming_client_stream_decryptor.py index 94b22b092..157755094 100644 --- a/test/unit/test_streaming_client_stream_decryptor.py +++ b/test/unit/test_streaming_client_stream_decryptor.py @@ -767,4 +767,4 @@ def test_close_no_footer(self, mock_close): ) with pytest.raises(SerializationError) as excinfo: test_decryptor.close() - excinfo.match("Footer not read, message may be corrupted or data key may be incorrect") + excinfo.match("Footer not read") diff --git a/tox.ini b/tox.ini index 194b4d412..9e2d95477 100644 --- a/tox.ini +++ b/tox.ini @@ -90,7 +90,7 @@ commands = # Run code coverage on the unit tests [testenv:coverage] commands = {[testenv:base-command]commands} --cov aws_encryption_sdk test/ -m local --ignore test/unit/mpl/ -[testenv:mplcoverage] +[testenv:mplcoverage-mpl] commands = {[testenv:base-command]commands} --cov aws_encryption_sdk test/unit/mpl/ -m local # Verify that local tests work without environment variables present From 6691fa2d81c211e7a97be9450f028ca89b47bcb6 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 15:19:17 -0800 Subject: [PATCH 061/376] refactor, fix --- .github/workflows/ci_static-analysis.yaml | 14 +++++- examples/src/keyrings/hierarchical_keyring.py | 50 ++++++++++--------- .../{ => internal}/mpl/__init__.py | 0 .../{ => internal}/mpl/cmm_handler.py | 2 +- .../{ => internal}/mpl/materials_handlers.py | 4 +- .../internal/mpl/mpl_import_handler.py | 21 ++++++++ .../mpl/mpl_import_handler.py | 14 ------ src/aws_encryption_sdk/streaming_client.py | 6 +-- test/unit/mpl/test_cmm_handler.py | 44 ++++++---------- test/unit/test_mpl_import_handler.py | 10 ++-- test/unit/test_streaming_client_mpl_import.py | 12 ++--- tox.ini | 22 +++++--- 12 files changed, 108 insertions(+), 91 deletions(-) rename src/aws_encryption_sdk/{ => internal}/mpl/__init__.py (100%) rename src/aws_encryption_sdk/{ => internal}/mpl/cmm_handler.py (98%) rename src/aws_encryption_sdk/{ => internal}/mpl/materials_handlers.py (98%) create mode 100644 src/aws_encryption_sdk/internal/mpl/mpl_import_handler.py delete mode 100644 src/aws_encryption_sdk/mpl/mpl_import_handler.py diff --git a/.github/workflows/ci_static-analysis.yaml b/.github/workflows/ci_static-analysis.yaml index 0093ae9a9..85d7f4a62 100644 --- a/.github/workflows/ci_static-analysis.yaml +++ b/.github/workflows/ci_static-analysis.yaml @@ -13,6 +13,9 @@ jobs: strategy: fail-fast: false matrix: + python: + - 3.8 + - 3.11 category: - bandit - doc8 @@ -26,15 +29,22 @@ jobs: - pylint-examples - black-check - isort-check + optional_mpl_dependency: + - "" + - -mpl + exclude: + # MPL is not supported on <3.11 + - python: 3.8 + optional_mpl_dependency: -mpl steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: ${{ matrix.python }} - run: | python -m pip install --upgrade pip pip install --upgrade -r dev_requirements/ci-requirements.txt - name: run test env: - TOXENV: ${{ matrix.category }} + TOXENV: ${{ matrix.category }}${{ matrix.optional_mpl_dependency }} run: tox -- -vv diff --git a/examples/src/keyrings/hierarchical_keyring.py b/examples/src/keyrings/hierarchical_keyring.py index a99728b6e..21108d9a0 100644 --- a/examples/src/keyrings/hierarchical_keyring.py +++ b/examples/src/keyrings/hierarchical_keyring.py @@ -4,23 +4,28 @@ import sys import boto3 -from aws_cryptographic_materialproviders.keystore.client import KeyStore -from aws_cryptographic_materialproviders.keystore.config import KeyStoreConfig -from aws_cryptographic_materialproviders.keystore.models import CreateKeyInput, KMSConfigurationKmsKeyArn -from aws_cryptographic_materialproviders.mpl.client import AwsCryptographicMaterialProviders -from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig -from aws_cryptographic_materialproviders.mpl.models import ( - CacheTypeDefault, - CreateAwsKmsHierarchicalKeyringInput, - DefaultCache, - GetBranchKeyIdInput, - GetBranchKeyIdOutput, -) -from aws_cryptographic_materialproviders.mpl.references import IBranchKeyIdSupplier, IKeyring +from typing import Dict import aws_encryption_sdk from aws_encryption_sdk import CommitmentPolicy from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError +from aws_encryption_sdk.internal.mpl import mpl_import_handler + +if mpl_import_handler.has_mpl(): + # noqa pylint: disable=import-error + from aws_cryptographic_materialproviders.keystore.client import KeyStore + from aws_cryptographic_materialproviders.keystore.config import KeyStoreConfig + from aws_cryptographic_materialproviders.keystore.models import CreateKeyInput, KMSConfigurationKmsKeyArn + from aws_cryptographic_materialproviders.mpl.client import AwsCryptographicMaterialProviders + from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig + from aws_cryptographic_materialproviders.mpl.models import ( + CacheTypeDefault, + CreateAwsKmsHierarchicalKeyringInput, + DefaultCache, + GetBranchKeyIdInput, + GetBranchKeyIdOutput, + ) + from aws_cryptographic_materialproviders.mpl.references import IBranchKeyIdSupplier, IKeyring module_root_dir = '/'.join(__file__.split("/")[:-1]) @@ -71,6 +76,7 @@ def encrypt_and_decrypt_with_keyring( branch_key_id_B: str = keystore.create_key(input=CreateKeyInput()).branch_key_identifier class ExampleBranchKeyIdSupplier(IBranchKeyIdSupplier): + """Example implementation of a branch key ID supplier.""" branch_key_id_for_tenant_A: str branch_key_id_for_tenant_B: str @@ -80,9 +86,11 @@ def __init__(self, tenant_1_id, tenant_2_id): def get_branch_key_id( self, - input: GetBranchKeyIdInput + # Change this to `native_input` + input: GetBranchKeyIdInput # noqa pylint: disable=redefined-builtin ) -> GetBranchKeyIdOutput: - encryption_context: dict[str, str] = input.encryption_context + """Returns branch key ID from the tenant ID in input's encryption context.""" + encryption_context: Dict[str, str] = input.encryption_context if b"tenant" not in encryption_context: raise ValueError("EncryptionContext invalid, does not contain expected tenant key value pair.") @@ -128,7 +136,7 @@ def get_branch_key_id( # The Branch Key Id supplier uses the encryption context to determine which branch key id will # be used to encrypt data. # Create encryption context for TenantA - encryption_context_A: dict[str, str] = { + encryption_context_A: Dict[str, str] = { "tenant": "TenantA", "encryption": "context", "is not": "secret", @@ -138,7 +146,7 @@ def get_branch_key_id( } # Create encryption context for TenantB - encryption_context_B: dict[str, str] = { + encryption_context_B: Dict[str, str] = { "tenant": "TenantB", "encryption": "context", "is not": "secret", @@ -191,8 +199,6 @@ def get_branch_key_id( input=keyring_input_B ) - # TODO: Run the decrypt, get expected exception type - # This should fail try: client.decrypt( source=ciphertext_A, @@ -201,7 +207,7 @@ def get_branch_key_id( except AWSEncryptionSDKClientError: pass - # # This should fail + # This should fail try: client.decrypt( source=ciphertext_B, @@ -220,6 +226,4 @@ def get_branch_key_id( source=ciphertext_B, keyring=hierarchical_keyring_B ) - assert plaintext_bytes_B == EXAMPLE_DATA - -# Also, a thread-safe example ig + assert plaintext_bytes_B == EXAMPLE_DATA \ No newline at end of file diff --git a/src/aws_encryption_sdk/mpl/__init__.py b/src/aws_encryption_sdk/internal/mpl/__init__.py similarity index 100% rename from src/aws_encryption_sdk/mpl/__init__.py rename to src/aws_encryption_sdk/internal/mpl/__init__.py diff --git a/src/aws_encryption_sdk/mpl/cmm_handler.py b/src/aws_encryption_sdk/internal/mpl/cmm_handler.py similarity index 98% rename from src/aws_encryption_sdk/mpl/cmm_handler.py rename to src/aws_encryption_sdk/internal/mpl/cmm_handler.py index 1f6c9ff41..c285afa04 100644 --- a/src/aws_encryption_sdk/mpl/cmm_handler.py +++ b/src/aws_encryption_sdk/internal/mpl/cmm_handler.py @@ -23,7 +23,7 @@ from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError from aws_encryption_sdk.identifiers import CommitmentPolicy -from aws_encryption_sdk.mpl.materials_handlers import DecryptionMaterialsHandler, EncryptionMaterialsHandler +from aws_encryption_sdk.internal.mpl.materials_handlers import DecryptionMaterialsHandler, EncryptionMaterialsHandler from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager from aws_encryption_sdk.structures import EncryptedDataKey as Native_EncryptedDataKey diff --git a/src/aws_encryption_sdk/mpl/materials_handlers.py b/src/aws_encryption_sdk/internal/mpl/materials_handlers.py similarity index 98% rename from src/aws_encryption_sdk/mpl/materials_handlers.py rename to src/aws_encryption_sdk/internal/mpl/materials_handlers.py index df5b57d53..bf32c2718 100644 --- a/src/aws_encryption_sdk/mpl/materials_handlers.py +++ b/src/aws_encryption_sdk/internal/mpl/materials_handlers.py @@ -48,8 +48,8 @@ def __init__( elif isinstance(materials, MPL_EncryptionMaterials): self.mpl_materials = materials else: - raise ValueError("Invalid EncryptionMaterials passed to EncryptionMaterialsHandler. " \ - f"materials: {materials}") + raise ValueError("Invalid EncryptionMaterials passed to EncryptionMaterialsHandler. " + f"materials: {materials}") @property def algorithm(self) -> Algorithm: diff --git a/src/aws_encryption_sdk/internal/mpl/mpl_import_handler.py b/src/aws_encryption_sdk/internal/mpl/mpl_import_handler.py new file mode 100644 index 000000000..55319bc43 --- /dev/null +++ b/src/aws_encryption_sdk/internal/mpl/mpl_import_handler.py @@ -0,0 +1,21 @@ +"""Detects whether the MPL is installed for use by internal ESDK code. +External customers should not need to interact with this. +""" + + +def has_mpl(): + """Returns True if the aws-cryptographic-material-providers library is installed, False otherwise.""" + try: + _import_mpl() + return True + except ImportError: + return False + + +def _import_mpl(): + """Private wrapper for import. + This only exists to help with unit test coverage. + This is not directly tested. + """ + # noqa pylint:disable=unused-import,import-outside-toplevel,import-error + import aws_cryptographic_materialproviders diff --git a/src/aws_encryption_sdk/mpl/mpl_import_handler.py b/src/aws_encryption_sdk/mpl/mpl_import_handler.py deleted file mode 100644 index 40669da1e..000000000 --- a/src/aws_encryption_sdk/mpl/mpl_import_handler.py +++ /dev/null @@ -1,14 +0,0 @@ -def has_mpl(): - """Returns True if the aws_cryptographic_materialproviders library is installed, False otherwise.""" - try: - _import_mpl() - return True - except ImportError: - return False - -def _import_mpl(): - """Private wrapper for import to help with unit test coverage. - - This is not directly tested. - """ - import aws_cryptographic_materialproviders \ No newline at end of file diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 044626c7f..c4c15559b 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -22,11 +22,9 @@ import attr import six - from cryptography.hazmat.primitives import serialization import aws_encryption_sdk.internal.utils -from aws_encryption_sdk.mpl.cmm_handler import CMMHandler from aws_encryption_sdk.exceptions import ( ActionNotAllowedError, AWSEncryptionSDKClientError, @@ -60,6 +58,8 @@ serialize_non_framed_close, serialize_non_framed_open, ) +from aws_encryption_sdk.internal.mpl import mpl_import_handler +from aws_encryption_sdk.internal.mpl.cmm_handler import CMMHandler from aws_encryption_sdk.internal.utils.commitment import ( validate_commitment_policy_on_decrypt, validate_commitment_policy_on_encrypt, @@ -71,8 +71,8 @@ from aws_encryption_sdk.materials_managers.default import DefaultCryptoMaterialsManager from aws_encryption_sdk.structures import MessageHeader -from aws_encryption_sdk.mpl import mpl_import_handler if mpl_import_handler.has_mpl(): + # noqa pylint: disable=import-error from aws_cryptographic_materialproviders.mpl.client import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig from aws_cryptographic_materialproviders.mpl.models import CreateDefaultCryptographicMaterialsManagerInput diff --git a/test/unit/mpl/test_cmm_handler.py b/test/unit/mpl/test_cmm_handler.py index 45b49ed91..343ac514b 100644 --- a/test/unit/mpl/test_cmm_handler.py +++ b/test/unit/mpl/test_cmm_handler.py @@ -12,35 +12,21 @@ # language governing permissions and limitations under the License. """Test suite to verify the mpl_import_handler module handles importing the MPL correctly.""" import pytest -from mock import MagicMock, patch - -from aws_encryption_sdk.mpl.cmm_handler import CMMHandler - -from aws_encryption_sdk.mpl.materials_handlers import DecryptionMaterialsHandler, EncryptionMaterialsHandler -from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest -from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager +from aws_cryptographic_materialproviders.mpl.models import ( + EncryptionMaterials as MPL_EncryptionMaterials, + GetEncryptionMaterialsInput, + GetEncryptionMaterialsOutput, +) from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager +from mock import MagicMock, patch +from aws_encryption_sdk.internal.mpl.cmm_handler import CMMHandler +from aws_encryption_sdk.internal.mpl.materials_handlers import EncryptionMaterialsHandler from aws_encryption_sdk.materials_managers import ( - DecryptionMaterials as Native_DecryptionMaterials, EncryptionMaterials as Native_EncryptionMaterials, + EncryptionMaterialsRequest, ) - -from aws_cryptographic_materialproviders.mpl.models import ( - AlgorithmSuiteIdESDK, - CommitmentPolicyESDK, - DecryptMaterialsInput, - DecryptMaterialsOutput, - EncryptedDataKey as MPL_EncryptedDataKey, - GetEncryptionMaterialsInput, - GetEncryptionMaterialsOutput, - ) - -from aws_cryptographic_materialproviders.mpl.models import ( - DecryptionMaterials as MPL_DecryptionMaterials, - EncryptedDataKey as MPL_EncryptedDataKey, - EncryptionMaterials as MPL_EncryptionMaterials, - ) +from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager mock_native_cmm = MagicMock(__class__=CryptoMaterialsManager) mock_mpl_cmm = MagicMock(__class__=ICryptographicMaterialsManager) @@ -68,7 +54,9 @@ def test_GIVEN_unknown_CMM_WHEN_create_CMMHandler_THEN_raise_ValueError(): @patch.object(mock_native_cmm, "get_encryption_materials") -def test_GIVEN_native_CMM_WHEN_get_encryption_materials_THEN_return_native_encryption_materials(mock_get_encryption_materials): +def test_GIVEN_native_CMM_WHEN_get_encryption_materials_THEN_return_native_encryption_materials( + mock_get_encryption_materials +): # Mock: native_cmm.get_encryption_materials returns mock native encryption materials mock_get_encryption_materials.return_value = mock_native_encryption_materials @@ -84,17 +72,16 @@ def test_GIVEN_native_CMM_WHEN_get_encryption_materials_THEN_return_native_encry @patch.object(mock_mpl_cmm, "get_encryption_materials") -@patch("aws_encryption_sdk.mpl.cmm_handler.CMMHandler._native_to_mpl_get_encryption_materials") +@patch("aws_encryption_sdk.internal.mpl.cmm_handler.CMMHandler._native_to_mpl_get_encryption_materials") def test_GIVEN_mpl_CMM_WHEN_get_encryption_materials_THEN_return_mpl_encryption_materials( mock_native_to_mpl_get_encryption_materials, mock_get_encryption_materials, - ): # Mock: mpl_cmm.get_encryption_materials returns mock MPL encryption materials mock_get_encryption_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) mock_get_encryption_materials_output.encryption_materials = mock_mpl_encryption_materials mock_get_encryption_materials.return_value = mock_get_encryption_materials_output - + # Mock: CMMHandler._native_to_mpl_get_encryption_materials creates a GetEncryptionMaterialsInput mock_get_encryption_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) mock_native_to_mpl_get_encryption_materials.return_value = mock_get_encryption_materials_input @@ -108,4 +95,3 @@ def test_GIVEN_mpl_CMM_WHEN_get_encryption_materials_THEN_return_mpl_encryption_ assert test.mpl_materials == mock_mpl_encryption_materials # Verify we actually called `get_encryption_materials` mock_mpl_cmm.get_encryption_materials.assert_called_once_with(mock_get_encryption_materials_input) - diff --git a/test/unit/test_mpl_import_handler.py b/test/unit/test_mpl_import_handler.py index c17c358b4..b82c3092b 100644 --- a/test/unit/test_mpl_import_handler.py +++ b/test/unit/test_mpl_import_handler.py @@ -14,21 +14,23 @@ import pytest from mock import patch -from aws_encryption_sdk.mpl import mpl_import_handler +from aws_encryption_sdk.internal.mpl import mpl_import_handler pytestmark = [pytest.mark.unit, pytest.mark.local] -@patch("aws_encryption_sdk.mpl.mpl_import_handler._import_mpl") + +@patch("aws_encryption_sdk.internal.mpl.mpl_import_handler._import_mpl") def test_GIVEN_import_mpl_succeeds_WHEN_call_has_mpl_THEN_return_True(import_mock): # Mock a successful import of `aws_cryptographic_material_providers` import_mock.return_value = None # No exception means successful import assert mpl_import_handler.has_mpl() is True -@patch("aws_encryption_sdk.mpl.mpl_import_handler._import_mpl") + +@patch("aws_encryption_sdk.internal.mpl.mpl_import_handler._import_mpl") def test_GIVEN_import_mpl_fails_WHEN_call_has_mpl_THEN_return_False(import_mock): # Mock not having a `aws_cryptographic_material_providers` module, # even if it is installed in the Python environment import_mock.side_effect = ImportError() - assert not mpl_import_handler.has_mpl() \ No newline at end of file + assert not mpl_import_handler.has_mpl() diff --git a/test/unit/test_streaming_client_mpl_import.py b/test/unit/test_streaming_client_mpl_import.py index f71f337b7..594ef3478 100644 --- a/test/unit/test_streaming_client_mpl_import.py +++ b/test/unit/test_streaming_client_mpl_import.py @@ -11,18 +11,18 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. """Unit test suite to validate aws_encryption_sdk.streaming_client MPL import logic.""" -import io -import pytest -from mock import patch +import sys from importlib import reload -from mock import Mock +import pytest +from mock import Mock, patch import aws_encryption_sdk.streaming_client pytestmark = [pytest.mark.unit, pytest.mark.local] + @patch.object(aws_encryption_sdk.streaming_client.mpl_import_handler, "has_mpl") def test_GIVEN_has_mpl_returns_True_WHEN_import_streaming_client_THEN_imports_mpl_modules(has_mpl_mock): has_mpl_mock.return_value = True @@ -30,7 +30,6 @@ def test_GIVEN_has_mpl_returns_True_WHEN_import_streaming_client_THEN_imports_mp # Mock any imports used in the try/catch block # If more imports are added there, then this needs to be expanded # This unit test should pass even if the MPL is not installed - import sys sys.modules['aws_cryptographic_materialproviders.mpl.client'] = Mock() sys.modules['aws_cryptographic_materialproviders.mpl.config'] = Mock() sys.modules['aws_cryptographic_materialproviders.mpl.models'] = Mock() @@ -42,6 +41,7 @@ def test_GIVEN_has_mpl_returns_True_WHEN_import_streaming_client_THEN_imports_mp assert hasattr(aws_encryption_sdk.streaming_client, "_HAS_MPL") assert aws_encryption_sdk.streaming_client._HAS_MPL is True + @patch.object(aws_encryption_sdk.streaming_client.mpl_import_handler, "has_mpl") def test_GIVEN_has_mpl_returns_False_WHEN_import_streaming_client_THEN_does_not_import_mpl_modules(has_mpl_mock): has_mpl_mock.return_value = False @@ -50,4 +50,4 @@ def test_GIVEN_has_mpl_returns_False_WHEN_import_streaming_client_THEN_does_not_ reload(aws_encryption_sdk.streaming_client) assert hasattr(aws_encryption_sdk.streaming_client, "_HAS_MPL") - assert aws_encryption_sdk.streaming_client._HAS_MPL is False \ No newline at end of file + assert aws_encryption_sdk.streaming_client._HAS_MPL is False diff --git a/tox.ini b/tox.ini index 9e2d95477..20b4d9426 100644 --- a/tox.ini +++ b/tox.ini @@ -3,12 +3,18 @@ envlist = # <3.11: run all non-MPL tests py{37,38,39,310}-{local,integ,accept,examples}, # >=3.11: run all tests with MPL installed and without MPL installed - # The `-mpl` suffix tells tox to install the MPL + # The `-mpl` suffix tells tox to install the MPL. + # In the case where the suffix IS NOT appended, + # this runs tests for the target version WITHOUT the MPL installed. + # In the case where the suffix IS appended, + # this runs tests for the target version WITH the MPL installed. + # This does not run any MPL-specific tests; it only runs non-MPL-specific + # tests in a test environment that also has the MPL. py{311,312}-{local,integ,accept,examples}{,-mpl}, - # >=3.11: run ONLY the MPL-specific tests (requires a special IAM role) - # the `mpl` prefix runs only MPL-specific tests - # (non-MPL-specific tests are run from the line above) - # the extra `-mpl` suffix tells tox to install the MPL + # >=3.11: Run ONLY the MPL-specific tests. + # These must be separate from the above target. + # These require the `-mpl` suffix so tox installs the MPL. + # The `mpl` prefix runs only MPL-specific tests py{311,312}-mpl{local,examples}-mpl nocmk, bandit, doc8, readme, docs, @@ -78,6 +84,7 @@ deps = mpl: -rrequirements_mpl.txt commands = local: {[testenv:base-command]commands} test/ -m local --ignore test/unit/mpl/ + # MPL unit tests require the MPL to be installed mpllocal: {[testenv:base-command]commands} test/unit/mpl/ -m local integ: {[testenv:base-command]commands} test/ -m integ --ignore test/unit/mpl/ accept: {[testenv:base-command]commands} test/ -m accept --ignore test/unit/mpl/ @@ -194,13 +201,14 @@ commands = --max-module-lines=1500 \ src/aws_encryption_sdk/ \ setup.py + --ignore-paths=src/aws_encryption_sdk/internal/mpl/ [testenv:pylint-examples] basepython = {[testenv:pylint]basepython} deps = {[testenv:pylint]deps} commands = - pylint --rcfile=examples/src/pylintrc examples/src/ - pylint --rcfile=examples/test/pylintrc --disable R0801 examples/test/ + pylint --rcfile=examples/src/pylintrc examples/src/ --ignore-paths=examples/src/keyrings + pylint --rcfile=examples/test/pylintrc --disable R0801 examples/test/ --ignore-paths=examples/test/keyrings [testenv:pylint-tests] basepython = {[testenv:pylint]basepython} From 3ae1e069df8b2ea75762ce61ba4ea5f3798c24a0 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 15:22:21 -0800 Subject: [PATCH 062/376] refactor, fix --- .github/workflows/ci_static-analysis.yaml | 8 -------- examples/src/keyrings/hierarchical_keyring.py | 3 ++- src/aws_encryption_sdk/internal/mpl/cmm_handler.py | 2 -- 3 files changed, 2 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci_static-analysis.yaml b/.github/workflows/ci_static-analysis.yaml index 85d7f4a62..802bad2bc 100644 --- a/.github/workflows/ci_static-analysis.yaml +++ b/.github/workflows/ci_static-analysis.yaml @@ -15,7 +15,6 @@ jobs: matrix: python: - 3.8 - - 3.11 category: - bandit - doc8 @@ -29,13 +28,6 @@ jobs: - pylint-examples - black-check - isort-check - optional_mpl_dependency: - - "" - - -mpl - exclude: - # MPL is not supported on <3.11 - - python: 3.8 - optional_mpl_dependency: -mpl steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 diff --git a/examples/src/keyrings/hierarchical_keyring.py b/examples/src/keyrings/hierarchical_keyring.py index 21108d9a0..56af60115 100644 --- a/examples/src/keyrings/hierarchical_keyring.py +++ b/examples/src/keyrings/hierarchical_keyring.py @@ -77,6 +77,7 @@ def encrypt_and_decrypt_with_keyring( class ExampleBranchKeyIdSupplier(IBranchKeyIdSupplier): """Example implementation of a branch key ID supplier.""" + branch_key_id_for_tenant_A: str branch_key_id_for_tenant_B: str @@ -226,4 +227,4 @@ def get_branch_key_id( source=ciphertext_B, keyring=hierarchical_keyring_B ) - assert plaintext_bytes_B == EXAMPLE_DATA \ No newline at end of file + assert plaintext_bytes_B == EXAMPLE_DATA diff --git a/src/aws_encryption_sdk/internal/mpl/cmm_handler.py b/src/aws_encryption_sdk/internal/mpl/cmm_handler.py index c285afa04..9789651e5 100644 --- a/src/aws_encryption_sdk/internal/mpl/cmm_handler.py +++ b/src/aws_encryption_sdk/internal/mpl/cmm_handler.py @@ -76,9 +76,7 @@ def get_encryption_materials( mpl_input: GetEncryptionMaterialsInput = CMMHandler._native_to_mpl_get_encryption_materials( request ) - print(f"mpl_input: {mpl_input}") mpl_output: GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) - print(f"mpl_output: {mpl_output}") return EncryptionMaterialsHandler(mpl_output.encryption_materials) except AwsCryptographicMaterialProvidersException as mpl_exception: # Wrap MPL error into the ESDK error type From 2b5fc7281f1ec263529f4b9c0c9635d62a7a4524 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 15:27:26 -0800 Subject: [PATCH 063/376] refactor, fix --- src/aws_encryption_sdk/internal/mpl/mpl_import_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aws_encryption_sdk/internal/mpl/mpl_import_handler.py b/src/aws_encryption_sdk/internal/mpl/mpl_import_handler.py index 55319bc43..5dd0a7b3e 100644 --- a/src/aws_encryption_sdk/internal/mpl/mpl_import_handler.py +++ b/src/aws_encryption_sdk/internal/mpl/mpl_import_handler.py @@ -17,5 +17,5 @@ def _import_mpl(): This only exists to help with unit test coverage. This is not directly tested. """ - # noqa pylint:disable=unused-import,import-outside-toplevel,import-error - import aws_cryptographic_materialproviders + # pylint:disable=unused-import,import-outside-toplevel,import-error + import aws_cryptographic_materialproviders # noqa F401 From a940dc57a7c9a11296d5200446dde48f0def7d77 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 15:33:45 -0800 Subject: [PATCH 064/376] refactor, fix --- .github/workflows/ci_static-analysis.yaml | 2 +- src/aws_encryption_sdk/internal/crypto/authentication.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_static-analysis.yaml b/.github/workflows/ci_static-analysis.yaml index 802bad2bc..03fa62165 100644 --- a/.github/workflows/ci_static-analysis.yaml +++ b/.github/workflows/ci_static-analysis.yaml @@ -38,5 +38,5 @@ jobs: pip install --upgrade -r dev_requirements/ci-requirements.txt - name: run test env: - TOXENV: ${{ matrix.category }}${{ matrix.optional_mpl_dependency }} + TOXENV: ${{ matrix.category }} run: tox -- -vv diff --git a/src/aws_encryption_sdk/internal/crypto/authentication.py b/src/aws_encryption_sdk/internal/crypto/authentication.py index a6446981e..d7ff35278 100644 --- a/src/aws_encryption_sdk/internal/crypto/authentication.py +++ b/src/aws_encryption_sdk/internal/crypto/authentication.py @@ -78,7 +78,7 @@ def from_key_bytes(cls, algorithm, key_bytes, encoding=serialization.Encoding.DE """ if encoding == serialization.Encoding.DER: key = serialization.load_der_private_key(data=key_bytes, password=None, backend=default_backend()) - elif serialization.Encoding.PEM: + elif encoding == serialization.Encoding.PEM: key = serialization.load_pem_private_key(data=key_bytes, password=None, backend=default_backend()) else: raise ValueError(f"Unsupported encoding for Signer: {encoding}") From 708ab5e26f1227cc5f72b58abd5d92bcf27cc89a Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 15:55:36 -0800 Subject: [PATCH 065/376] it works locally but fails on gha --- test/unit/test_crypto_authentication_signer.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/unit/test_crypto_authentication_signer.py b/test/unit/test_crypto_authentication_signer.py index 11271abfb..9584cf441 100644 --- a/test/unit/test_crypto_authentication_signer.py +++ b/test/unit/test_crypto_authentication_signer.py @@ -81,7 +81,13 @@ def test_signer_from_key_bytes(patch_default_backend, patch_serialization, patch mock_algorithm_info = MagicMock(return_value=sentinel.algorithm_info, spec=patch_ec.EllipticCurve) _algorithm = MagicMock(signing_algorithm_info=mock_algorithm_info) - signer = Signer.from_key_bytes(algorithm=_algorithm, key_bytes=sentinel.key_bytes) + # signer = Signer.from_key_bytes(algorithm=_algorithm, key_bytes=sentinel.key_bytes) + + signer = Signer.from_key_bytes( + algorithm=_algorithm, + key_bytes=sentinel.key_bytes, + encoding=patch_serialization.encoding.DER + ) patch_serialization.load_der_private_key.assert_called_once_with( data=sentinel.key_bytes, password=None, backend=patch_default_backend.return_value From ffd295c10253a7c0f1437309396eef28f076118f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 15:57:57 -0800 Subject: [PATCH 066/376] it works locally but fails on gha --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 20b4d9426..1c133adaa 100644 --- a/tox.ini +++ b/tox.ini @@ -110,7 +110,7 @@ passenv = setenv = ######################################################### deps = -rdev_requirements/test-requirements.txt -commands = {[testenv:base-command]commands} test/ -m local +commands = {[testenv:base-command]commands} test/ -m local --ignore test/unit/mpl/ # Collect requirements for use in upstream tests [testenv:freeze-upstream-requirements-base] @@ -142,7 +142,7 @@ commands = {[testenv:freeze-upstream-requirements-base]commands} test/upstream-r [testenv:test-upstream-requirements-base] sitepackages = False recreate = True -commands = {[testenv:base-command]commands} test/ -m local +commands = {[testenv:base-command]commands} test/ -m local --ignore test/unit/mpl/ # Test frozen upstream requirements for Python 3.7 [testenv:test-upstream-requirements-py37] From 1ba175c5391b7cf673b5c8e56be661951a072730 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 16:01:21 -0800 Subject: [PATCH 067/376] it works locally but fails on gha --- test/unit/test_crypto_authentication_signer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/test_crypto_authentication_signer.py b/test/unit/test_crypto_authentication_signer.py index 9584cf441..2e5f5a4fd 100644 --- a/test/unit/test_crypto_authentication_signer.py +++ b/test/unit/test_crypto_authentication_signer.py @@ -86,7 +86,7 @@ def test_signer_from_key_bytes(patch_default_backend, patch_serialization, patch signer = Signer.from_key_bytes( algorithm=_algorithm, key_bytes=sentinel.key_bytes, - encoding=patch_serialization.encoding.DER + encoding=patch_serialization.Encoding.DER ) patch_serialization.load_der_private_key.assert_called_once_with( From fa175ba6ed28758222b39e08569b310bfa08e363 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 16:09:12 -0800 Subject: [PATCH 068/376] it works locally but fails on gha --- .github/workflows/ci_tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 603f54371..d46c19c48 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -24,8 +24,8 @@ jobs: fail-fast: true matrix: os: - - ubuntu-latest - - windows-latest + # - ubuntu-latest + # - windows-latest - macos-latest python: - 3.7 From 2f90a970683bb9cf17469c88d51a28b8d5bd8baf Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 16:15:22 -0800 Subject: [PATCH 069/376] it works locally but fails on gha --- src/aws_encryption_sdk/streaming_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index c4c15559b..4e9d5d07c 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -247,7 +247,7 @@ def __new__(cls, **kwargs): ): raise TypeError("Can't instantiate abstract class {}".format(cls.__name__)) - instance = super(_EncryptionStream, cls).__new__(cls) + instance = super().__new__(cls) config = kwargs.pop("config", None) if not isinstance(config, instance._config_class): # pylint: disable=protected-access From df9215f6e72695a69fc042e2d7d326c341c006ff Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 16:19:20 -0800 Subject: [PATCH 070/376] it works locally but fails on gha --- src/aws_encryption_sdk/streaming_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 4e9d5d07c..c4c15559b 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -247,7 +247,7 @@ def __new__(cls, **kwargs): ): raise TypeError("Can't instantiate abstract class {}".format(cls.__name__)) - instance = super().__new__(cls) + instance = super(_EncryptionStream, cls).__new__(cls) config = kwargs.pop("config", None) if not isinstance(config, instance._config_class): # pylint: disable=protected-access From b57e4a397cfa4ca28f0877df1f0dbc608fbf0cfe Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 16:22:47 -0800 Subject: [PATCH 071/376] it works locally but fails on gha --- src/aws_encryption_sdk/streaming_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index c4c15559b..4e9d5d07c 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -247,7 +247,7 @@ def __new__(cls, **kwargs): ): raise TypeError("Can't instantiate abstract class {}".format(cls.__name__)) - instance = super(_EncryptionStream, cls).__new__(cls) + instance = super().__new__(cls) config = kwargs.pop("config", None) if not isinstance(config, instance._config_class): # pylint: disable=protected-access From 9d7ec6d3a2baca380c2e7bdb466b554c36e4a5f2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 16:25:37 -0800 Subject: [PATCH 072/376] it works locally but fails on gha --- src/aws_encryption_sdk/streaming_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 4e9d5d07c..c4c15559b 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -247,7 +247,7 @@ def __new__(cls, **kwargs): ): raise TypeError("Can't instantiate abstract class {}".format(cls.__name__)) - instance = super().__new__(cls) + instance = super(_EncryptionStream, cls).__new__(cls) config = kwargs.pop("config", None) if not isinstance(config, instance._config_class): # pylint: disable=protected-access From 2cbc8451756cedc0efde5a1c73141a8d8741748c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 16:31:22 -0800 Subject: [PATCH 073/376] it works locally but fails on gha --- src/aws_encryption_sdk/streaming_client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index c4c15559b..3bda700e7 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -247,7 +247,10 @@ def __new__(cls, **kwargs): ): raise TypeError("Can't instantiate abstract class {}".format(cls.__name__)) - instance = super(_EncryptionStream, cls).__new__(cls) + if issubclass(StreamEncryptor, _EncryptionStream): + instance = super(_EncryptionStream, cls).__new__(cls) + else: + raise ValueError(f"issubclass {issubclass(StreamEncryptor, _EncryptionStream)}") config = kwargs.pop("config", None) if not isinstance(config, instance._config_class): # pylint: disable=protected-access From def946d841461aa19a5dc73aeaf1471de7f7e351 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 16:37:51 -0800 Subject: [PATCH 074/376] it works locally but fails on gha --- src/aws_encryption_sdk/streaming_client.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 3bda700e7..4e9d5d07c 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -247,10 +247,7 @@ def __new__(cls, **kwargs): ): raise TypeError("Can't instantiate abstract class {}".format(cls.__name__)) - if issubclass(StreamEncryptor, _EncryptionStream): - instance = super(_EncryptionStream, cls).__new__(cls) - else: - raise ValueError(f"issubclass {issubclass(StreamEncryptor, _EncryptionStream)}") + instance = super().__new__(cls) config = kwargs.pop("config", None) if not isinstance(config, instance._config_class): # pylint: disable=protected-access From dff6ac0134e29ed4b98d046701ee75b11cd26450 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 17:12:17 -0800 Subject: [PATCH 075/376] it works locally but fails on gha --- src/aws_encryption_sdk/streaming_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 4e9d5d07c..4b849b818 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -71,14 +71,14 @@ from aws_encryption_sdk.materials_managers.default import DefaultCryptoMaterialsManager from aws_encryption_sdk.structures import MessageHeader -if mpl_import_handler.has_mpl(): +try: # noqa pylint: disable=import-error from aws_cryptographic_materialproviders.mpl.client import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig from aws_cryptographic_materialproviders.mpl.models import CreateDefaultCryptographicMaterialsManagerInput from aws_cryptographic_materialproviders.mpl.references import IKeyring _HAS_MPL = True -else: +except ImportError: _HAS_MPL = False _LOGGER = logging.getLogger(__name__) From 78f0b0faed85977346c2bb6652899a8316c0e10e Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 17:22:16 -0800 Subject: [PATCH 076/376] it works locally but fails on gha --- src/aws_encryption_sdk/streaming_client.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 4b849b818..1186faf2b 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -158,6 +158,9 @@ class _ClientConfig(object): # pylint: disable=too-many-instance-attributes ) # DEPRECATED: Value is no longer configurable here. Parameter left here to avoid breaking consumers. def _has_mpl_attrs_post_init(self): + if not hasattr(self, "keyring"): + self._no_mpl_attrs_post_init() + return if not _exactly_one_arg_is_not_none(self.materials_manager, self.key_provider, self.keyring): raise TypeError("Exactly one of keyring, materials_manager, or key_provider must be provided") if self.materials_manager is None: From 20a469e3b2e7fb8357096af9462856fa6b43604c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 17:24:24 -0800 Subject: [PATCH 077/376] it works locally but fails on gha --- src/aws_encryption_sdk/streaming_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 1186faf2b..f9b550e6d 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -250,7 +250,7 @@ def __new__(cls, **kwargs): ): raise TypeError("Can't instantiate abstract class {}".format(cls.__name__)) - instance = super().__new__(cls) + instance = super(_EncryptionStream, cls).__new__(cls) config = kwargs.pop("config", None) if not isinstance(config, instance._config_class): # pylint: disable=protected-access From 66859a7b5316013200b42a6b00c2ebb1a51fcc18 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 10:10:57 -0800 Subject: [PATCH 078/376] fix tests --- examples/src/keyrings/hierarchical_keyring.py | 33 +++++++++-------- .../internal/mpl/mpl_import_handler.py | 21 ----------- src/aws_encryption_sdk/streaming_client.py | 4 +-- test/unit/mpl/test_cmm_handler.py | 2 +- test/unit/test_mpl_import_handler.py | 36 ------------------- test/unit/test_streaming_client_mpl_import.py | 32 ++++++++--------- 6 files changed, 33 insertions(+), 95 deletions(-) delete mode 100644 src/aws_encryption_sdk/internal/mpl/mpl_import_handler.py delete mode 100644 test/unit/test_mpl_import_handler.py diff --git a/examples/src/keyrings/hierarchical_keyring.py b/examples/src/keyrings/hierarchical_keyring.py index 56af60115..3e56d1e56 100644 --- a/examples/src/keyrings/hierarchical_keyring.py +++ b/examples/src/keyrings/hierarchical_keyring.py @@ -9,23 +9,22 @@ import aws_encryption_sdk from aws_encryption_sdk import CommitmentPolicy from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError -from aws_encryption_sdk.internal.mpl import mpl_import_handler - -if mpl_import_handler.has_mpl(): - # noqa pylint: disable=import-error - from aws_cryptographic_materialproviders.keystore.client import KeyStore - from aws_cryptographic_materialproviders.keystore.config import KeyStoreConfig - from aws_cryptographic_materialproviders.keystore.models import CreateKeyInput, KMSConfigurationKmsKeyArn - from aws_cryptographic_materialproviders.mpl.client import AwsCryptographicMaterialProviders - from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig - from aws_cryptographic_materialproviders.mpl.models import ( - CacheTypeDefault, - CreateAwsKmsHierarchicalKeyringInput, - DefaultCache, - GetBranchKeyIdInput, - GetBranchKeyIdOutput, - ) - from aws_cryptographic_materialproviders.mpl.references import IBranchKeyIdSupplier, IKeyring + +# ignore missing MPL for pylint, but the MPL is required for this example +# noqa pylint: disable=import-error +from aws_cryptographic_materialproviders.keystore.client import KeyStore +from aws_cryptographic_materialproviders.keystore.config import KeyStoreConfig +from aws_cryptographic_materialproviders.keystore.models import CreateKeyInput, KMSConfigurationKmsKeyArn +from aws_cryptographic_materialproviders.mpl.client import AwsCryptographicMaterialProviders +from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig +from aws_cryptographic_materialproviders.mpl.models import ( + CacheTypeDefault, + CreateAwsKmsHierarchicalKeyringInput, + DefaultCache, + GetBranchKeyIdInput, + GetBranchKeyIdOutput, +) +from aws_cryptographic_materialproviders.mpl.references import IBranchKeyIdSupplier, IKeyring module_root_dir = '/'.join(__file__.split("/")[:-1]) diff --git a/src/aws_encryption_sdk/internal/mpl/mpl_import_handler.py b/src/aws_encryption_sdk/internal/mpl/mpl_import_handler.py deleted file mode 100644 index 5dd0a7b3e..000000000 --- a/src/aws_encryption_sdk/internal/mpl/mpl_import_handler.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Detects whether the MPL is installed for use by internal ESDK code. -External customers should not need to interact with this. -""" - - -def has_mpl(): - """Returns True if the aws-cryptographic-material-providers library is installed, False otherwise.""" - try: - _import_mpl() - return True - except ImportError: - return False - - -def _import_mpl(): - """Private wrapper for import. - This only exists to help with unit test coverage. - This is not directly tested. - """ - # pylint:disable=unused-import,import-outside-toplevel,import-error - import aws_cryptographic_materialproviders # noqa F401 diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index f9b550e6d..8e004e84e 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -58,7 +58,6 @@ serialize_non_framed_close, serialize_non_framed_open, ) -from aws_encryption_sdk.internal.mpl import mpl_import_handler from aws_encryption_sdk.internal.mpl.cmm_handler import CMMHandler from aws_encryption_sdk.internal.utils.commitment import ( validate_commitment_policy_on_decrypt, @@ -72,13 +71,14 @@ from aws_encryption_sdk.structures import MessageHeader try: + # pylint should pass even if the MPL isn't installed # noqa pylint: disable=import-error from aws_cryptographic_materialproviders.mpl.client import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig from aws_cryptographic_materialproviders.mpl.models import CreateDefaultCryptographicMaterialsManagerInput from aws_cryptographic_materialproviders.mpl.references import IKeyring _HAS_MPL = True -except ImportError: +except ImportError as e: _HAS_MPL = False _LOGGER = logging.getLogger(__name__) diff --git a/test/unit/mpl/test_cmm_handler.py b/test/unit/mpl/test_cmm_handler.py index 343ac514b..d16374899 100644 --- a/test/unit/mpl/test_cmm_handler.py +++ b/test/unit/mpl/test_cmm_handler.py @@ -10,7 +10,7 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Test suite to verify the mpl_import_handler module handles importing the MPL correctly.""" +"""Test suite to verify the cmm_handler module delegates correctly.""" import pytest from aws_cryptographic_materialproviders.mpl.models import ( EncryptionMaterials as MPL_EncryptionMaterials, diff --git a/test/unit/test_mpl_import_handler.py b/test/unit/test_mpl_import_handler.py deleted file mode 100644 index b82c3092b..000000000 --- a/test/unit/test_mpl_import_handler.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. -"""Test suite to verify the mpl_import_handler module handles importing the MPL correctly.""" -import pytest -from mock import patch - -from aws_encryption_sdk.internal.mpl import mpl_import_handler - -pytestmark = [pytest.mark.unit, pytest.mark.local] - - -@patch("aws_encryption_sdk.internal.mpl.mpl_import_handler._import_mpl") -def test_GIVEN_import_mpl_succeeds_WHEN_call_has_mpl_THEN_return_True(import_mock): - # Mock a successful import of `aws_cryptographic_material_providers` - import_mock.return_value = None # No exception means successful import - - assert mpl_import_handler.has_mpl() is True - - -@patch("aws_encryption_sdk.internal.mpl.mpl_import_handler._import_mpl") -def test_GIVEN_import_mpl_fails_WHEN_call_has_mpl_THEN_return_False(import_mock): - # Mock not having a `aws_cryptographic_material_providers` module, - # even if it is installed in the Python environment - import_mock.side_effect = ImportError() - - assert not mpl_import_handler.has_mpl() diff --git a/test/unit/test_streaming_client_mpl_import.py b/test/unit/test_streaming_client_mpl_import.py index 594ef3478..ebafd199f 100644 --- a/test/unit/test_streaming_client_mpl_import.py +++ b/test/unit/test_streaming_client_mpl_import.py @@ -23,31 +23,27 @@ pytestmark = [pytest.mark.unit, pytest.mark.local] -@patch.object(aws_encryption_sdk.streaming_client.mpl_import_handler, "has_mpl") -def test_GIVEN_has_mpl_returns_True_WHEN_import_streaming_client_THEN_imports_mpl_modules(has_mpl_mock): - has_mpl_mock.return_value = True +# Check if MPL is installed, and skip tests based on whether it is +try: + import aws_cryptographic_materialproviders + HAS_MPL = True +except ImportError as e: + HAS_MPL = False - # Mock any imports used in the try/catch block - # If more imports are added there, then this needs to be expanded - # This unit test should pass even if the MPL is not installed - sys.modules['aws_cryptographic_materialproviders.mpl.client'] = Mock() - sys.modules['aws_cryptographic_materialproviders.mpl.config'] = Mock() - sys.modules['aws_cryptographic_materialproviders.mpl.models'] = Mock() - sys.modules['aws_cryptographic_materialproviders.mpl.references'] = Mock() - # Reload module given the mock - reload(aws_encryption_sdk.streaming_client) +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_test_has_mpl_is_True_THEN_streaming_client_has_mpl_is_TRUE(): + """If the MPL IS installed in the runtime environment, + assert the streaming client has _HAS_MPL set to True""" assert hasattr(aws_encryption_sdk.streaming_client, "_HAS_MPL") assert aws_encryption_sdk.streaming_client._HAS_MPL is True -@patch.object(aws_encryption_sdk.streaming_client.mpl_import_handler, "has_mpl") -def test_GIVEN_has_mpl_returns_False_WHEN_import_streaming_client_THEN_does_not_import_mpl_modules(has_mpl_mock): - has_mpl_mock.return_value = False - - # Reload module given the mock - reload(aws_encryption_sdk.streaming_client) +@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") +def test_GIVEN_test_has_mpl_is_False_THEN_streaming_client_has_mpl_is_TRUE(): + """If the MPL IS NOT installed in the runtime environment, + assert the streaming client has _HAS_MPL set to False""" assert hasattr(aws_encryption_sdk.streaming_client, "_HAS_MPL") assert aws_encryption_sdk.streaming_client._HAS_MPL is False From bf8f67c1fd4a8523c4a0a76eb5f5e0245e5b25b5 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 10:18:23 -0800 Subject: [PATCH 079/376] cleanup --- examples/src/keyrings/hierarchical_keyring.py | 11 +++++------ src/aws_encryption_sdk/streaming_client.py | 2 +- test/unit/test_streaming_client_mpl_import.py | 12 ++++-------- 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/examples/src/keyrings/hierarchical_keyring.py b/examples/src/keyrings/hierarchical_keyring.py index 3e56d1e56..50f620456 100644 --- a/examples/src/keyrings/hierarchical_keyring.py +++ b/examples/src/keyrings/hierarchical_keyring.py @@ -4,12 +4,6 @@ import sys import boto3 -from typing import Dict - -import aws_encryption_sdk -from aws_encryption_sdk import CommitmentPolicy -from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError - # ignore missing MPL for pylint, but the MPL is required for this example # noqa pylint: disable=import-error from aws_cryptographic_materialproviders.keystore.client import KeyStore @@ -25,6 +19,11 @@ GetBranchKeyIdOutput, ) from aws_cryptographic_materialproviders.mpl.references import IBranchKeyIdSupplier, IKeyring +from typing import Dict + +import aws_encryption_sdk +from aws_encryption_sdk import CommitmentPolicy +from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError module_root_dir = '/'.join(__file__.split("/")[:-1]) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 8e004e84e..ad998088c 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -78,7 +78,7 @@ from aws_cryptographic_materialproviders.mpl.models import CreateDefaultCryptographicMaterialsManagerInput from aws_cryptographic_materialproviders.mpl.references import IKeyring _HAS_MPL = True -except ImportError as e: +except ImportError: _HAS_MPL = False _LOGGER = logging.getLogger(__name__) diff --git a/test/unit/test_streaming_client_mpl_import.py b/test/unit/test_streaming_client_mpl_import.py index ebafd199f..3eda0ad63 100644 --- a/test/unit/test_streaming_client_mpl_import.py +++ b/test/unit/test_streaming_client_mpl_import.py @@ -12,11 +12,7 @@ # language governing permissions and limitations under the License. """Unit test suite to validate aws_encryption_sdk.streaming_client MPL import logic.""" -import sys -from importlib import reload - import pytest -from mock import Mock, patch import aws_encryption_sdk.streaming_client @@ -25,14 +21,14 @@ # Check if MPL is installed, and skip tests based on whether it is try: - import aws_cryptographic_materialproviders + import aws_cryptographic_materialproviders # noqa pylint: disable=unused-import HAS_MPL = True -except ImportError as e: +except ImportError: HAS_MPL = False @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_test_has_mpl_is_True_THEN_streaming_client_has_mpl_is_TRUE(): +def test_GIVEN_test_has_mpl_is_True_THEN_streaming_client_has_mpl_is_True(): """If the MPL IS installed in the runtime environment, assert the streaming client has _HAS_MPL set to True""" @@ -41,7 +37,7 @@ def test_GIVEN_test_has_mpl_is_True_THEN_streaming_client_has_mpl_is_TRUE(): @pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") -def test_GIVEN_test_has_mpl_is_False_THEN_streaming_client_has_mpl_is_TRUE(): +def test_GIVEN_test_has_mpl_is_False_THEN_streaming_client_has_mpl_is_False(): """If the MPL IS NOT installed in the runtime environment, assert the streaming client has _HAS_MPL set to False""" From b24be113a9a179d9cc22602b06a55222fc557ef3 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 10:23:07 -0800 Subject: [PATCH 080/376] re-enable test --- .github/workflows/ci_tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index d46c19c48..603f54371 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -24,8 +24,8 @@ jobs: fail-fast: true matrix: os: - # - ubuntu-latest - # - windows-latest + - ubuntu-latest + - windows-latest - macos-latest python: - 3.7 From acba1b0143ee6b40312b5a02ba6d95ea736b9d0a Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 10:25:52 -0800 Subject: [PATCH 081/376] re-enable test --- test/unit/test_streaming_client_mpl_import.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/unit/test_streaming_client_mpl_import.py b/test/unit/test_streaming_client_mpl_import.py index 3eda0ad63..a4ca87e2a 100644 --- a/test/unit/test_streaming_client_mpl_import.py +++ b/test/unit/test_streaming_client_mpl_import.py @@ -19,7 +19,9 @@ pytestmark = [pytest.mark.unit, pytest.mark.local] -# Check if MPL is installed, and skip tests based on whether it is +# Check if MPL is installed, and skip tests based on its installation status +# Ideally, this logic would be based on mocking imports and testing logic, +# but doing that introduces errors that cause other tests to fail. try: import aws_cryptographic_materialproviders # noqa pylint: disable=unused-import HAS_MPL = True From 42b7b745dec470b06bf1fecaae6d0578b17df2e1 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 10:29:57 -0800 Subject: [PATCH 082/376] longpaths --- .github/workflows/ci_tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 603f54371..a8bad7bfb 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -70,6 +70,7 @@ jobs: python-version: ${{ matrix.python }} architecture: ${{ matrix.architecture }} - run: | + git config --system core.longpaths true python -m pip install --upgrade pip pip install --upgrade -r dev_requirements/ci-requirements.txt - name: run test From f226e7e8bd5b3df06fda12f63be601939a0a8711 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 10:34:14 -0800 Subject: [PATCH 083/376] longpaths --- .github/workflows/ci_tests.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index a8bad7bfb..cab32dcc7 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -64,13 +64,16 @@ jobs: - python: 3.10 optional_mpl_dependency: -mpl steps: + - name: Support longpaths + run: | + git config --global core.longpaths true + - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} architecture: ${{ matrix.architecture }} - run: | - git config --system core.longpaths true python -m pip install --upgrade pip pip install --upgrade -r dev_requirements/ci-requirements.txt - name: run test From aa2f80a2aaff191082ed0844ded4ebe812eafb2c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 10:39:24 -0800 Subject: [PATCH 084/376] debug windows fail --- .github/workflows/ci_tests.yaml | 4 ++-- src/aws_encryption_sdk/streaming_client.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index cab32dcc7..132c65bd0 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -24,9 +24,9 @@ jobs: fail-fast: true matrix: os: - - ubuntu-latest + # - ubuntu-latest - windows-latest - - macos-latest + # - macos-latest python: - 3.7 - 3.8 diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index ad998088c..bc37d7de5 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -78,7 +78,8 @@ from aws_cryptographic_materialproviders.mpl.models import CreateDefaultCryptographicMaterialsManagerInput from aws_cryptographic_materialproviders.mpl.references import IKeyring _HAS_MPL = True -except ImportError: +except ImportError as e: + print(e) _HAS_MPL = False _LOGGER = logging.getLogger(__name__) From bc002b682dd7113734cf8494e17e48a1462e18c6 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 10:42:48 -0800 Subject: [PATCH 085/376] debug windows fail --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 1c133adaa..fa3e8530f 100644 --- a/tox.ini +++ b/tox.ini @@ -56,7 +56,7 @@ envlist = # coverage :: Runs code coverage, failing the build if coverage is below the configured threshold [testenv:base-command] -commands = pytest --basetemp={envtmpdir} -l {posargs} +commands = pytest --basetemp={envtmpdir} -l {posargs} -s -v [testenv] ; passenv = AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2,AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_SESSION_TOKEN,AWS_CONTAINER_CREDENTIALS_RELATIVE_URI,AWS_PROFILE,PIP_CONFIG_FILE From 8dd0303615d72712dc7c190f6e0724656c26bfb2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 10:46:34 -0800 Subject: [PATCH 086/376] debug windows fail --- src/aws_encryption_sdk/streaming_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index bc37d7de5..4a00b99f4 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -79,7 +79,7 @@ from aws_cryptographic_materialproviders.mpl.references import IKeyring _HAS_MPL = True except ImportError as e: - print(e) + print("IMPORT ERROR" + str(e)) _HAS_MPL = False _LOGGER = logging.getLogger(__name__) From 1e9db3b3b0d32da9c55e4e918c9320318026a611 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 10:50:22 -0800 Subject: [PATCH 087/376] debug windows fail --- .github/workflows/ci_tests.yaml | 9 ++++++--- src/aws_encryption_sdk/streaming_client.py | 1 - 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 132c65bd0..d64d3ce79 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -24,9 +24,12 @@ jobs: fail-fast: true matrix: os: - # - ubuntu-latest - - windows-latest - # - macos-latest + - ubuntu-latest + # Windows fails due to "No module named 'Wrappers'" + # This SHOULD be fixed once Dafny generates fully-qualified import statements + # Disable for now + # - windows-latest + - macos-latest python: - 3.7 - 3.8 diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 4a00b99f4..8e004e84e 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -79,7 +79,6 @@ from aws_cryptographic_materialproviders.mpl.references import IKeyring _HAS_MPL = True except ImportError as e: - print("IMPORT ERROR" + str(e)) _HAS_MPL = False _LOGGER = logging.getLogger(__name__) From 74d4e667d1828dd6437f1aeb42436c0b44c2d7c0 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 10:53:26 -0800 Subject: [PATCH 088/376] disable windows until pythonpath --- src/aws_encryption_sdk/streaming_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 8e004e84e..ad998088c 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -78,7 +78,7 @@ from aws_cryptographic_materialproviders.mpl.models import CreateDefaultCryptographicMaterialsManagerInput from aws_cryptographic_materialproviders.mpl.references import IKeyring _HAS_MPL = True -except ImportError as e: +except ImportError: _HAS_MPL = False _LOGGER = logging.getLogger(__name__) From 1bb23e862a73d96a376e207aeb82b3b2d3e19ec4 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 11:07:51 -0800 Subject: [PATCH 089/376] expand testing --- .github/workflows/ci_static-analysis.yaml | 4 +--- .github/workflows/ci_tests.yaml | 10 ++++++++++ codebuild/py311/examples_mpl.yml | 7 ++++--- codebuild/py312/awses_local_mpl.yml | 2 ++ codebuild/py312/examples_mpl.yml | 10 +++++++--- codebuild/py312/integ_mpl.yml | 2 ++ tox.ini | 11 ++++++++--- 7 files changed, 34 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci_static-analysis.yaml b/.github/workflows/ci_static-analysis.yaml index 03fa62165..0093ae9a9 100644 --- a/.github/workflows/ci_static-analysis.yaml +++ b/.github/workflows/ci_static-analysis.yaml @@ -13,8 +13,6 @@ jobs: strategy: fail-fast: false matrix: - python: - - 3.8 category: - bandit - doc8 @@ -32,7 +30,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: ${{ matrix.python }} + python-version: 3.8 - run: | python -m pip install --upgrade pip pip install --upgrade -r dev_requirements/ci-requirements.txt diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index d64d3ce79..58f9b1b11 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -44,10 +44,14 @@ jobs: category: - local - accept + - mpllocal + - mplaccept # These require credentials. # Enable them once we sort how to provide them. # - integ # - examples + # Append '-mpl' to some test environments. + # This suffix signals to tox to install the MPL in the test environment. optional_mpl_dependency: - "" - -mpl @@ -66,7 +70,13 @@ jobs: optional_mpl_dependency: -mpl - python: 3.10 optional_mpl_dependency: -mpl + # mpllocal and mplaccept require the MPL to be installed + - python: mpllocal + optional_mpl_dependency: + - python: mplaccept + optional_mpl_dependency: steps: + # Support long Dafny filenames (used in MPL and DBESDK repos) - name: Support longpaths run: | git config --global core.longpaths true diff --git a/codebuild/py311/examples_mpl.yml b/codebuild/py311/examples_mpl.yml index f8f2a6a01..19a5dec05 100644 --- a/codebuild/py311/examples_mpl.yml +++ b/codebuild/py311/examples_mpl.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - # No TOXENV; examples using the MPL switch envs + # No TOXENV. This runs multiple environments. REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f @@ -20,14 +20,15 @@ phases: build: commands: - pip install "tox < 4.0" + # Run non-MPL-specific tests with the MPL installed - tox -e py311-examples-mpl - # Assume special role + # Assume special role to access keystore - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-Py311ExamplesMpl") - export TMP_ROLE - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') - aws sts get-caller-identity - # Run special role-specific examples + # Run MPL-specific tests with special role - tox -e py311-mplexamples-mpl diff --git a/codebuild/py312/awses_local_mpl.yml b/codebuild/py312/awses_local_mpl.yml index db25f4f57..1d0f80319 100644 --- a/codebuild/py312/awses_local_mpl.yml +++ b/codebuild/py312/awses_local_mpl.yml @@ -1,3 +1,5 @@ +# Runs the same tests as awses_local in an environment with the MPL installed. +# This asserts existing tests continue to pass with the MPL installed. version: 0.2 env: diff --git a/codebuild/py312/examples_mpl.yml b/codebuild/py312/examples_mpl.yml index ba0660024..366222441 100644 --- a/codebuild/py312/examples_mpl.yml +++ b/codebuild/py312/examples_mpl.yml @@ -1,8 +1,11 @@ +# Runs the same tests as examples in an environment with the MPL installed +# to assert existing tests continue to pass with the MPL installed. +# Then, run MPL-specific tests. version: 0.2 env: variables: - # No TOXENV; examples using the MPL switch envs + # No TOXENV. This runs multiple environments. REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f @@ -25,13 +28,14 @@ phases: - pip install --upgrade pip - pip install setuptools - pip install "tox < 4.0" + # Run non-MPL-specific tests with the MPL installed - tox -e py312-examples-mpl - # Assume special role + # Assume special role to access keystore - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-Py311ExamplesMpl") - export TMP_ROLE - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') - aws sts get-caller-identity - # Run special role-specific examples + # Run MPL-specific tests with special role - tox -e py312-mplexamples-mpl diff --git a/codebuild/py312/integ_mpl.yml b/codebuild/py312/integ_mpl.yml index 553f41e8a..e292acc57 100644 --- a/codebuild/py312/integ_mpl.yml +++ b/codebuild/py312/integ_mpl.yml @@ -1,3 +1,5 @@ +# Runs the same tests as integ in an environment with the MPL installed. +# This asserts existing tests continue to pass with the MPL installed. version: 0.2 env: diff --git a/tox.ini b/tox.ini index fa3e8530f..e5a585ca5 100644 --- a/tox.ini +++ b/tox.ini @@ -15,7 +15,7 @@ envlist = # These must be separate from the above target. # These require the `-mpl` suffix so tox installs the MPL. # The `mpl` prefix runs only MPL-specific tests - py{311,312}-mpl{local,examples}-mpl + py{311,312}-mpl{local,integ,accept,examples}-mpl nocmk, bandit, doc8, readme, docs, {flake8,pylint}{,-tests,-examples}, @@ -56,7 +56,7 @@ envlist = # coverage :: Runs code coverage, failing the build if coverage is below the configured threshold [testenv:base-command] -commands = pytest --basetemp={envtmpdir} -l {posargs} -s -v +commands = pytest --basetemp={envtmpdir} -l {posargs} [testenv] ; passenv = AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2,AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_SESSION_TOKEN,AWS_CONTAINER_CREDENTIALS_RELATIVE_URI,AWS_PROFILE,PIP_CONFIG_FILE @@ -87,11 +87,16 @@ commands = # MPL unit tests require the MPL to be installed mpllocal: {[testenv:base-command]commands} test/unit/mpl/ -m local integ: {[testenv:base-command]commands} test/ -m integ --ignore test/unit/mpl/ + # MPL integ tests require the MPL to be installed + mplinteg: {[testenv:base-command]commands} test/unit/mpl -m integ accept: {[testenv:base-command]commands} test/ -m accept --ignore test/unit/mpl/ + # MPL accept tests require the MPL to be installed + mplaccept: {[testenv:base-command]commands} test/unit/mpl -m accept examples: {[testenv:base-command]commands} examples/test/ -m examples --ignore examples/test/keyrings/ # MPL keyring examples require a special IAM role; run these separately under a separate set of permissions mplexamples: {[testenv:base-command]commands} examples/test/keyrings -m examples - all: {[testenv:base-command]commands} test/ examples/test/ --ignore test/unit/mpl/ + all: {[testenv:base-command]commands} test/ examples/test/ --ignore test/unit/mpl/ --ignore examples/test/keyrings/ + mplall: {[testenv:base-command]commands} test/unit/mpl/ examples/test/keyrings/ manual: {[testenv:base-command]commands} # Run code coverage on the unit tests From 1ee69cefbb3f0d3486c2d0e873a1ac6c0a4bf8ff Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 11:11:05 -0800 Subject: [PATCH 090/376] expand testing --- .github/workflows/ci_tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 58f9b1b11..daa108060 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -72,9 +72,9 @@ jobs: optional_mpl_dependency: -mpl # mpllocal and mplaccept require the MPL to be installed - python: mpllocal - optional_mpl_dependency: + optional_mpl_dependency: "" - python: mplaccept - optional_mpl_dependency: + optional_mpl_dependency: "" steps: # Support long Dafny filenames (used in MPL and DBESDK repos) - name: Support longpaths From b33f2f706fb1dbb9f98d64b06a0c7ad6e97d08b2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 11:12:01 -0800 Subject: [PATCH 091/376] expand testing --- .github/workflows/ci_tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index daa108060..495192a81 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -71,9 +71,9 @@ jobs: - python: 3.10 optional_mpl_dependency: -mpl # mpllocal and mplaccept require the MPL to be installed - - python: mpllocal + - category: mpllocal optional_mpl_dependency: "" - - python: mplaccept + - category: mplaccept optional_mpl_dependency: "" steps: # Support long Dafny filenames (used in MPL and DBESDK repos) From c582888a3f064ab1ab258f9c8bcb8fac80df6a36 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 11:16:22 -0800 Subject: [PATCH 092/376] expand testing --- tox.ini | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index e5a585ca5..cee1e35a4 100644 --- a/tox.ini +++ b/tox.ini @@ -15,7 +15,7 @@ envlist = # These must be separate from the above target. # These require the `-mpl` suffix so tox installs the MPL. # The `mpl` prefix runs only MPL-specific tests - py{311,312}-mpl{local,integ,accept,examples}-mpl + py{311,312}-mpl{local,examples}-mpl nocmk, bandit, doc8, readme, docs, {flake8,pylint}{,-tests,-examples}, @@ -87,11 +87,9 @@ commands = # MPL unit tests require the MPL to be installed mpllocal: {[testenv:base-command]commands} test/unit/mpl/ -m local integ: {[testenv:base-command]commands} test/ -m integ --ignore test/unit/mpl/ - # MPL integ tests require the MPL to be installed - mplinteg: {[testenv:base-command]commands} test/unit/mpl -m integ + # No MPL-specific integ tests accept: {[testenv:base-command]commands} test/ -m accept --ignore test/unit/mpl/ - # MPL accept tests require the MPL to be installed - mplaccept: {[testenv:base-command]commands} test/unit/mpl -m accept + # No MPL-specific accept tests examples: {[testenv:base-command]commands} examples/test/ -m examples --ignore examples/test/keyrings/ # MPL keyring examples require a special IAM role; run these separately under a separate set of permissions mplexamples: {[testenv:base-command]commands} examples/test/keyrings -m examples From 5ae44f5a077a240c1a9cd49bd49b08d165a4ad6c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 11:19:22 -0800 Subject: [PATCH 093/376] expand testing --- .github/workflows/ci_tests.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 495192a81..803d4741e 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -45,7 +45,6 @@ jobs: - local - accept - mpllocal - - mplaccept # These require credentials. # Enable them once we sort how to provide them. # - integ @@ -70,11 +69,9 @@ jobs: optional_mpl_dependency: -mpl - python: 3.10 optional_mpl_dependency: -mpl - # mpllocal and mplaccept require the MPL to be installed + # mpllocal requires the MPL to be installed - category: mpllocal optional_mpl_dependency: "" - - category: mplaccept - optional_mpl_dependency: "" steps: # Support long Dafny filenames (used in MPL and DBESDK repos) - name: Support longpaths From cb7e3d1c8c8a58548ef03cdbc22f663d676c179f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 11:45:49 -0800 Subject: [PATCH 094/376] cleanup --- codebuild/coverage/coverage_mpl.yml | 2 +- examples/src/keyrings/hierarchical_keyring.py | 60 ++++++------------- examples/src/keyrings/module_.py | 2 +- examples/src/module_.py | 2 +- .../unit/test_crypto_authentication_signer.py | 6 +- test_vector_handlers/tox.ini | 4 +- tox.ini | 3 +- 7 files changed, 27 insertions(+), 52 deletions(-) diff --git a/codebuild/coverage/coverage_mpl.yml b/codebuild/coverage/coverage_mpl.yml index 5dcc65382..922705569 100644 --- a/codebuild/coverage/coverage_mpl.yml +++ b/codebuild/coverage/coverage_mpl.yml @@ -7,7 +7,7 @@ env: phases: install: runtime-versions: - python: latest + python: 3.11 build: commands: - pip install "tox < 4.0" diff --git a/examples/src/keyrings/hierarchical_keyring.py b/examples/src/keyrings/hierarchical_keyring.py index 50f620456..8f8707013 100644 --- a/examples/src/keyrings/hierarchical_keyring.py +++ b/examples/src/keyrings/hierarchical_keyring.py @@ -15,8 +15,6 @@ CacheTypeDefault, CreateAwsKmsHierarchicalKeyringInput, DefaultCache, - GetBranchKeyIdInput, - GetBranchKeyIdOutput, ) from aws_cryptographic_materialproviders.mpl.references import IBranchKeyIdSupplier, IKeyring from typing import Dict @@ -25,6 +23,8 @@ from aws_encryption_sdk import CommitmentPolicy from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError +from .example_branch_key_id_supplier import ExampleBranchKeyIdSupplier + module_root_dir = '/'.join(__file__.split("/")[:-1]) sys.path.append(module_root_dir) @@ -73,39 +73,6 @@ def encrypt_and_decrypt_with_keyring( branch_key_id_A: str = keystore.create_key(input=CreateKeyInput()).branch_key_identifier branch_key_id_B: str = keystore.create_key(input=CreateKeyInput()).branch_key_identifier - class ExampleBranchKeyIdSupplier(IBranchKeyIdSupplier): - """Example implementation of a branch key ID supplier.""" - - branch_key_id_for_tenant_A: str - branch_key_id_for_tenant_B: str - - def __init__(self, tenant_1_id, tenant_2_id): - self.branch_key_id_for_tenant_A = tenant_1_id - self.branch_key_id_for_tenant_B = tenant_2_id - - def get_branch_key_id( - self, - # Change this to `native_input` - input: GetBranchKeyIdInput # noqa pylint: disable=redefined-builtin - ) -> GetBranchKeyIdOutput: - """Returns branch key ID from the tenant ID in input's encryption context.""" - encryption_context: Dict[str, str] = input.encryption_context - - if b"tenant" not in encryption_context: - raise ValueError("EncryptionContext invalid, does not contain expected tenant key value pair.") - - tenant_key_id: str = encryption_context.get(b"tenant") - branch_key_id: str - - if tenant_key_id == b"TenantA": - branch_key_id = self.branch_key_id_for_tenant_A - elif tenant_key_id == b"TenantB": - branch_key_id = self.branch_key_id_for_tenant_B - else: - raise ValueError(f"Item does not contain valid tenant ID: {tenant_key_id=}") - - return GetBranchKeyIdOutput(branch_key_id=branch_key_id) - # 5. Create a branch key supplier that maps the branch key id to a more readable format branch_key_id_supplier: IBranchKeyIdSupplier = ExampleBranchKeyIdSupplier( tenant_1_id=branch_key_id_A, @@ -132,8 +99,10 @@ def get_branch_key_id( input=keyring_input ) - # The Branch Key Id supplier uses the encryption context to determine which branch key id will - # be used to encrypt data. + # 7. Create encryption context for both tenants. + # The Branch Key Id supplier uses the encryption context to determine which branch key id will + # be used to encrypt data. + # Create encryption context for TenantA encryption_context_A: Dict[str, str] = { "tenant": "TenantA", @@ -154,7 +123,7 @@ def get_branch_key_id( "the data you are handling": "is what you think it is", } - # Encrypt the data for encryptionContextA & encryptionContextB + # 8. Encrypt the data for encryptionContextA & encryptionContextB ciphertext_A, _ = client.encrypt( source=EXAMPLE_DATA, keyring=hierarchical_keyring, @@ -166,8 +135,8 @@ def get_branch_key_id( encryption_context=encryption_context_B ) - # To attest that TenantKeyB cannot decrypt a message written by TenantKeyA - # let's construct more restrictive hierarchical keyrings. + # 9. To attest that TenantKeyB cannot decrypt a message written by TenantKeyA, + # let's construct more restrictive hierarchical keyrings. keyring_input_A: CreateAwsKmsHierarchicalKeyringInput = CreateAwsKmsHierarchicalKeyringInput( key_store=keystore, branch_key_id=branch_key_id_A, @@ -198,6 +167,11 @@ def get_branch_key_id( input=keyring_input_B ) + # 10. Demonstrate that data encrypted by one tenant's key + # cannot be decrypted with by a keyring specific to another tenant. + + # Keyring with tenant B's branch key cannot decrypt data encrypted with tenant A's branch key + # This will fail and raise a AWSEncryptionSDKClientError, which we swallow ONLY for demonstration purposes. try: client.decrypt( source=ciphertext_A, @@ -206,7 +180,8 @@ def get_branch_key_id( except AWSEncryptionSDKClientError: pass - # This should fail + # Keyring with tenant A's branch key cannot decrypt data encrypted with tenant B's branch key. + # This will fail and raise a AWSEncryptionSDKClientError, which we swallow ONLY for demonstration purposes. try: client.decrypt( source=ciphertext_B, @@ -215,7 +190,8 @@ def get_branch_key_id( except AWSEncryptionSDKClientError: pass - # These should succeed + # 10. Demonstrate that data encrypted by one tenant's branch key can be decrypted by that tenant, + # and that the decrypted data matches the input data. plaintext_bytes_A, _ = client.decrypt( source=ciphertext_A, keyring=hierarchical_keyring_A diff --git a/examples/src/keyrings/module_.py b/examples/src/keyrings/module_.py index d9a8c058f..3e8d3062a 100644 --- a/examples/src/keyrings/module_.py +++ b/examples/src/keyrings/module_.py @@ -1 +1 @@ -"""Should remove this.""" +"""Should remove this once PYTHONPATH issues are resolved by adding doo files.""" diff --git a/examples/src/module_.py b/examples/src/module_.py index d9a8c058f..3e8d3062a 100644 --- a/examples/src/module_.py +++ b/examples/src/module_.py @@ -1 +1 @@ -"""Should remove this.""" +"""Should remove this once PYTHONPATH issues are resolved by adding doo files.""" diff --git a/test/unit/test_crypto_authentication_signer.py b/test/unit/test_crypto_authentication_signer.py index 2e5f5a4fd..bd7227fd3 100644 --- a/test/unit/test_crypto_authentication_signer.py +++ b/test/unit/test_crypto_authentication_signer.py @@ -81,8 +81,10 @@ def test_signer_from_key_bytes(patch_default_backend, patch_serialization, patch mock_algorithm_info = MagicMock(return_value=sentinel.algorithm_info, spec=patch_ec.EllipticCurve) _algorithm = MagicMock(signing_algorithm_info=mock_algorithm_info) - # signer = Signer.from_key_bytes(algorithm=_algorithm, key_bytes=sentinel.key_bytes) - + # Explicitly pass in patched serialization module. + # Patching the module introduces namespace issues + # which causes the method's `isinstance` checks to fail + # by changing the namespace from `serialization.Encoding.DER` to `Encoding.DER`. signer = Signer.from_key_bytes( algorithm=_algorithm, key_bytes=sentinel.key_bytes, diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index 7004080e3..df2707f6a 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -2,7 +2,7 @@ envlist = # The test vectors depend on new features now, # so until release we can only effectively test the local version of the ESDK. - py{37,38,39,310}-awses_local{,-mpl}, + py{37,38,39,310}-awses_local # 1.2.0 and 1.2.max are being difficult because of attrs bandit, doc8, readme, {flake8,pylint}{,-tests}, @@ -48,8 +48,6 @@ passenv = sitepackages = False deps = -rtest/requirements.txt - # install the MPL if in environment - mpl: -r../requirements_mpl.txt .. commands = {[testenv:base-command]commands} diff --git a/tox.ini b/tox.ini index cee1e35a4..ae30f3122 100644 --- a/tox.ini +++ b/tox.ini @@ -59,7 +59,6 @@ envlist = commands = pytest --basetemp={envtmpdir} -l {posargs} [testenv] -; passenv = AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2,AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_SESSION_TOKEN,AWS_CONTAINER_CREDENTIALS_RELATIVE_URI,AWS_PROFILE,PIP_CONFIG_FILE passenv = # Identifies AWS KMS key id to use in integration tests AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID \ @@ -80,7 +79,7 @@ passenv = sitepackages = False deps = -rdev_requirements/test-requirements.txt - # install the MPL if in environment + # install the MPL requirements if the `-mpl` suffix is present mpl: -rrequirements_mpl.txt commands = local: {[testenv:base-command]commands} test/ -m local --ignore test/unit/mpl/ From b026b532b59b177cfb63be0019c508d829b41aea Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 11:53:41 -0800 Subject: [PATCH 095/376] cleanup --- .../internal/mpl/cmm_handler.py | 1 - .../internal/mpl/materials_handlers.py | 6 --- .../internal/utils/__init__.py | 18 +++++++++ src/aws_encryption_sdk/streaming_client.py | 37 +++++++------------ 4 files changed, 31 insertions(+), 31 deletions(-) diff --git a/src/aws_encryption_sdk/internal/mpl/cmm_handler.py b/src/aws_encryption_sdk/internal/mpl/cmm_handler.py index 9789651e5..1575e0187 100644 --- a/src/aws_encryption_sdk/internal/mpl/cmm_handler.py +++ b/src/aws_encryption_sdk/internal/mpl/cmm_handler.py @@ -29,7 +29,6 @@ from aws_encryption_sdk.structures import EncryptedDataKey as Native_EncryptedDataKey -# TODO-MPL Should this implement interface...? seems like yes since it implements all of interface methods class CMMHandler(CryptoMaterialsManager): """ In instances where encryption materials may be provided by either diff --git a/src/aws_encryption_sdk/internal/mpl/materials_handlers.py b/src/aws_encryption_sdk/internal/mpl/materials_handlers.py index bf32c2718..79312f863 100644 --- a/src/aws_encryption_sdk/internal/mpl/materials_handlers.py +++ b/src/aws_encryption_sdk/internal/mpl/materials_handlers.py @@ -93,9 +93,6 @@ def data_encryption_key(self) -> DataKey: if hasattr(self, "native_materials"): return self.native_materials.data_encryption_key else: - # TODO-MPL This impl is probably wrong, but works for for now - # If this works for all features, great! Remove this comment before launch. - # Otherwise, fix the implementation. mpl_dek = self.mpl_materials.plaintext_data_key return DataKey( # key_provider is unused, but the return type is DataKey @@ -149,9 +146,6 @@ def data_key(self) -> DataKey: if hasattr(self, "native_materials"): return self.native_materials.data_key else: - # TODO-MPL This impl is probably wrong, but works for for now - # If this works for all features, great! Remove this comment before launch. - # Otherwise, fix the implementation. return DataKey( key_provider=MasterKeyInfo( provider_id="", diff --git a/src/aws_encryption_sdk/internal/utils/__init__.py b/src/aws_encryption_sdk/internal/utils/__init__.py index dac38ac73..b65f6df0f 100644 --- a/src/aws_encryption_sdk/internal/utils/__init__.py +++ b/src/aws_encryption_sdk/internal/utils/__init__.py @@ -163,3 +163,21 @@ def source_data_key_length_check(source_data_key, algorithm): actual=len(source_data_key.data_key), required=algorithm.kdf_input_len ) ) + +def exactly_one_arg_is_not_none(*args): + """ + Helper function for internal ESDK logic. + Returns `True` if exactly one item in the list is not `None`. + Returns `False` otherwise. + """ + # Have not found any `not None` + found_one = False + for arg in args: + if arg is not None: + if found_one is False: + # Have not already found a `not None`, found a `not None` => only one `not None` (so far) + found_one = True + else: + # Already found a `not None`, found another `not None` => not exactly one `not None` + return False + return found_one \ No newline at end of file diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index ad998088c..72f18c117 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -59,6 +59,7 @@ serialize_non_framed_open, ) from aws_encryption_sdk.internal.mpl.cmm_handler import CMMHandler +from aws_encryption_sdk.internal.utils import exactly_one_arg_is_not_none from aws_encryption_sdk.internal.utils.commitment import ( validate_commitment_policy_on_decrypt, validate_commitment_policy_on_encrypt, @@ -84,25 +85,6 @@ _LOGGER = logging.getLogger(__name__) -def _exactly_one_arg_is_not_none(*args): - """ - Private helper function. - Returns `True` if exactly one item in the list is not `None`. - Returns `False` otherwise. - """ - # Have not found any `not None` - found_one = False - for arg in args: - if arg is not None: - if found_one is False: - # Have not already found a `not None`, found a `not None` => only one `not None` (so far) - found_one = True - else: - # Already found a `not None`, found another `not None` => not exactly one `not None` - return False - return found_one - - @attr.s(hash=True) # pylint: disable=too-many-instance-attributes @six.add_metaclass(abc.ABCMeta) class _ClientConfig(object): # pylint: disable=too-many-instance-attributes @@ -147,6 +129,7 @@ class _ClientConfig(object): # pylint: disable=too-many-instance-attributes hash=True, default=None, validator=attr.validators.optional(attr.validators.instance_of(MasterKeyProvider)) ) if _HAS_MPL: + # Keyrings are only available if the MPL is installed in the runtime keyring = attr.ib( hash=True, default=None, validator=attr.validators.optional(attr.validators.instance_of(IKeyring)) ) @@ -158,10 +141,13 @@ class _ClientConfig(object): # pylint: disable=too-many-instance-attributes ) # DEPRECATED: Value is no longer configurable here. Parameter left here to avoid breaking consumers. def _has_mpl_attrs_post_init(self): + """If the MPL is present in the runtime, perform MPL-specific post-init logic + to validate the new object has a valid state. + """ if not hasattr(self, "keyring"): self._no_mpl_attrs_post_init() return - if not _exactly_one_arg_is_not_none(self.materials_manager, self.key_provider, self.keyring): + if not exactly_one_arg_is_not_none(self.materials_manager, self.key_provider, self.keyring): raise TypeError("Exactly one of keyring, materials_manager, or key_provider must be provided") if self.materials_manager is None: if self.key_provider is not None: @@ -187,6 +173,9 @@ def _has_mpl_attrs_post_init(self): self.materials_manager = cmm_handler def _no_mpl_attrs_post_init(self): + """If the MPL is NOT present in the runtime, perform post-init logic + to validate the new object has a valid state. + """ both_cmm_and_mkp_defined = self.materials_manager is not None and self.key_provider is not None neither_cmm_nor_mkp_defined = self.materials_manager is None and self.key_provider is None @@ -560,8 +549,8 @@ def _prep_message(self): if self._encryption_materials.signing_key is None: self.signer = None else: - # MPL verification key is NOT key bytes, it is bytes of the compressed point - # TODO-MPL: clean this up, least-privilege violation. + # MPL verification key is PEM bytes, not DER bytes. + # If the underlying CMM is from the MPL, load PEM bytes. if (isinstance(self.config.materials_manager, CMMHandler) and hasattr(self.config.materials_manager, "mpl_cmm")): self.signer = Signer.from_key_bytes( @@ -928,8 +917,8 @@ def _read_header(self): if decryption_materials.verification_key is None: self.verifier = None else: - # MPL verification key is NOT key bytes, it is bytes of the compressed point - # TODO-MPL: clean this up, least-privilege violation. + # MPL verification key is NOT key bytes; it is bytes of the compressed point. + # If the underlying CMM is from the MPL, load PEM bytes. if (isinstance(self.config.materials_manager, CMMHandler) and hasattr(self.config.materials_manager, "mpl_cmm")): self.verifier = Verifier.from_encoded_point( From 50afa3ade9d8c5f2712392b925fe3b621e97ba4d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 11:54:25 -0800 Subject: [PATCH 096/376] cleanup --- src/aws_encryption_sdk/streaming_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 72f18c117..032ed7d15 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -918,7 +918,7 @@ def _read_header(self): self.verifier = None else: # MPL verification key is NOT key bytes; it is bytes of the compressed point. - # If the underlying CMM is from the MPL, load PEM bytes. + # If the underlying CMM is from the MPL, load bytes from encoded point. if (isinstance(self.config.materials_manager, CMMHandler) and hasattr(self.config.materials_manager, "mpl_cmm")): self.verifier = Verifier.from_encoded_point( From 1c612a0b9e9cc85bae1f71a7d1027d2901c5de82 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 12:01:11 -0800 Subject: [PATCH 097/376] cleanup --- examples/src/keyrings/hierarchical_keyring.py | 2 +- src/aws_encryption_sdk/internal/utils/__init__.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/examples/src/keyrings/hierarchical_keyring.py b/examples/src/keyrings/hierarchical_keyring.py index 8f8707013..c71719346 100644 --- a/examples/src/keyrings/hierarchical_keyring.py +++ b/examples/src/keyrings/hierarchical_keyring.py @@ -169,7 +169,7 @@ def encrypt_and_decrypt_with_keyring( # 10. Demonstrate that data encrypted by one tenant's key # cannot be decrypted with by a keyring specific to another tenant. - + # Keyring with tenant B's branch key cannot decrypt data encrypted with tenant A's branch key # This will fail and raise a AWSEncryptionSDKClientError, which we swallow ONLY for demonstration purposes. try: diff --git a/src/aws_encryption_sdk/internal/utils/__init__.py b/src/aws_encryption_sdk/internal/utils/__init__.py index b65f6df0f..b08121281 100644 --- a/src/aws_encryption_sdk/internal/utils/__init__.py +++ b/src/aws_encryption_sdk/internal/utils/__init__.py @@ -164,11 +164,15 @@ def source_data_key_length_check(source_data_key, algorithm): ) ) + def exactly_one_arg_is_not_none(*args): """ Helper function for internal ESDK logic. - Returns `True` if exactly one item in the list is not `None`. + Returns `True` if exactly one item in the provided arguments is not `None`. Returns `False` otherwise. + + :param args: Input arguments to check + :returns: `True` if exactly one item in the provided arguments is not `None`; `False` otherwise """ # Have not found any `not None` found_one = False @@ -180,4 +184,4 @@ def exactly_one_arg_is_not_none(*args): else: # Already found a `not None`, found another `not None` => not exactly one `not None` return False - return found_one \ No newline at end of file + return found_one From bcdb4ba37f189d8fe6407c66f3ee3ccb1dbc7ebe Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 12:19:22 -0800 Subject: [PATCH 098/376] add missing file --- .../example_branch_key_id_supplier.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 examples/src/keyrings/example_branch_key_id_supplier.py diff --git a/examples/src/keyrings/example_branch_key_id_supplier.py b/examples/src/keyrings/example_branch_key_id_supplier.py new file mode 100644 index 000000000..a3ef0df6f --- /dev/null +++ b/examples/src/keyrings/example_branch_key_id_supplier.py @@ -0,0 +1,37 @@ +from aws_cryptographic_materialproviders.mpl.models import GetBranchKeyIdInput, GetBranchKeyIdOutput +from aws_cryptographic_materialproviders.mpl.references import IBranchKeyIdSupplier +from typing import Dict + + +class ExampleBranchKeyIdSupplier(IBranchKeyIdSupplier): + """Example implementation of a branch key ID supplier.""" + + branch_key_id_for_tenant_A: str + branch_key_id_for_tenant_B: str + + def __init__(self, tenant_1_id, tenant_2_id): + self.branch_key_id_for_tenant_A = tenant_1_id + self.branch_key_id_for_tenant_B = tenant_2_id + + def get_branch_key_id( + self, + # Change this to `native_input` + input: GetBranchKeyIdInput # noqa pylint: disable=redefined-builtin + ) -> GetBranchKeyIdOutput: + """Returns branch key ID from the tenant ID in input's encryption context.""" + encryption_context: Dict[str, str] = input.encryption_context + + if b"tenant" not in encryption_context: + raise ValueError("EncryptionContext invalid, does not contain expected tenant key value pair.") + + tenant_key_id: str = encryption_context.get(b"tenant") + branch_key_id: str + + if tenant_key_id == b"TenantA": + branch_key_id = self.branch_key_id_for_tenant_A + elif tenant_key_id == b"TenantB": + branch_key_id = self.branch_key_id_for_tenant_B + else: + raise ValueError(f"Item does not contain valid tenant ID: {tenant_key_id=}") + + return GetBranchKeyIdOutput(branch_key_id=branch_key_id) \ No newline at end of file From 41fe2f9facf04bbbdf0ccf0168c20aa9e27e059c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 12:26:47 -0800 Subject: [PATCH 099/376] add missing file --- test_vector_handlers/tox.ini | 3 +++ tox.ini | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index df2707f6a..e5e467d8a 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -3,6 +3,7 @@ envlist = # The test vectors depend on new features now, # so until release we can only effectively test the local version of the ESDK. py{37,38,39,310}-awses_local + py{311,312}-awses_local{,-mpl} # 1.2.0 and 1.2.max are being difficult because of attrs bandit, doc8, readme, {flake8,pylint}{,-tests}, @@ -48,6 +49,8 @@ passenv = sitepackages = False deps = -rtest/requirements.txt + # Install the MPL requirements if the `-mpl` suffix is present + mpl: -rrequirements_mpl.txt .. commands = {[testenv:base-command]commands} diff --git a/tox.ini b/tox.ini index ae30f3122..72e8ec9fa 100644 --- a/tox.ini +++ b/tox.ini @@ -79,7 +79,7 @@ passenv = sitepackages = False deps = -rdev_requirements/test-requirements.txt - # install the MPL requirements if the `-mpl` suffix is present + # Install the MPL requirements if the `-mpl` suffix is present mpl: -rrequirements_mpl.txt commands = local: {[testenv:base-command]commands} test/ -m local --ignore test/unit/mpl/ From 1ba857e74a1e7e117a8208ac21e235d1c5d2e18a Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 12:31:48 -0800 Subject: [PATCH 100/376] add missing file --- test_vector_handlers/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index e5e467d8a..580b641e0 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -50,7 +50,7 @@ sitepackages = False deps = -rtest/requirements.txt # Install the MPL requirements if the `-mpl` suffix is present - mpl: -rrequirements_mpl.txt + mpl: -r../requirements_mpl.txt .. commands = {[testenv:base-command]commands} From 74bfe127d6ea86de4637b1dfea3621f703bff0cd Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 12:38:32 -0800 Subject: [PATCH 101/376] cleanup --- examples/src/keyrings/example_branch_key_id_supplier.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/src/keyrings/example_branch_key_id_supplier.py b/examples/src/keyrings/example_branch_key_id_supplier.py index a3ef0df6f..a06280fa1 100644 --- a/examples/src/keyrings/example_branch_key_id_supplier.py +++ b/examples/src/keyrings/example_branch_key_id_supplier.py @@ -1,3 +1,5 @@ +"""Example implementation of a branch key ID supplier.""" + from aws_cryptographic_materialproviders.mpl.models import GetBranchKeyIdInput, GetBranchKeyIdOutput from aws_cryptographic_materialproviders.mpl.references import IBranchKeyIdSupplier from typing import Dict @@ -10,12 +12,13 @@ class ExampleBranchKeyIdSupplier(IBranchKeyIdSupplier): branch_key_id_for_tenant_B: str def __init__(self, tenant_1_id, tenant_2_id): + """Example constructor for a branch key ID supplier.""" self.branch_key_id_for_tenant_A = tenant_1_id self.branch_key_id_for_tenant_B = tenant_2_id def get_branch_key_id( self, - # Change this to `native_input` + # TODO-MPL: Change this to `native_input` in Smithy-Dafny input: GetBranchKeyIdInput # noqa pylint: disable=redefined-builtin ) -> GetBranchKeyIdOutput: """Returns branch key ID from the tenant ID in input's encryption context.""" @@ -34,4 +37,4 @@ def get_branch_key_id( else: raise ValueError(f"Item does not contain valid tenant ID: {tenant_key_id=}") - return GetBranchKeyIdOutput(branch_key_id=branch_key_id) \ No newline at end of file + return GetBranchKeyIdOutput(branch_key_id=branch_key_id) From b3b9a0ffd82d962ef4ccea15813b1ba09d6aac3d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 22 Feb 2024 12:07:24 -0800 Subject: [PATCH 102/376] refactor --- .github/workflows/ci_tests.yaml | 1 + .../internal/mpl/cmm_handler.py | 157 ----------------- .../internal/mpl/materials_handlers.py | 164 ------------------ src/aws_encryption_sdk/streaming_client.py | 10 +- test/unit/mpl/README.md | 1 - test/unit/mpl/test_cmm_handler.py | 97 ----------- 6 files changed, 5 insertions(+), 425 deletions(-) delete mode 100644 src/aws_encryption_sdk/internal/mpl/cmm_handler.py delete mode 100644 src/aws_encryption_sdk/internal/mpl/materials_handlers.py delete mode 100644 test/unit/mpl/README.md delete mode 100644 test/unit/mpl/test_cmm_handler.py diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 803d4741e..3a6b16d45 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -27,6 +27,7 @@ jobs: - ubuntu-latest # Windows fails due to "No module named 'Wrappers'" # This SHOULD be fixed once Dafny generates fully-qualified import statements + # (i.e. doo files, per-package module names) # Disable for now # - windows-latest - macos-latest diff --git a/src/aws_encryption_sdk/internal/mpl/cmm_handler.py b/src/aws_encryption_sdk/internal/mpl/cmm_handler.py deleted file mode 100644 index 1575e0187..000000000 --- a/src/aws_encryption_sdk/internal/mpl/cmm_handler.py +++ /dev/null @@ -1,157 +0,0 @@ -"""Retrieves encryption/decryption materials from an underlying materials provider.""" - -# These dependencies are only loaded if you install the MPL. -try: - # pylint seems to struggle with this conditional import - # pylint: disable=unused-import - from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException - from aws_cryptographic_materialproviders.mpl.models import ( - AlgorithmSuiteIdESDK, - CommitmentPolicyESDK, - DecryptMaterialsInput, - DecryptMaterialsOutput, - EncryptedDataKey as MPL_EncryptedDataKey, - GetEncryptionMaterialsInput, - GetEncryptionMaterialsOutput, - ) - from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager - -except ImportError: - pass - -from typing import List - -from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError -from aws_encryption_sdk.identifiers import CommitmentPolicy -from aws_encryption_sdk.internal.mpl.materials_handlers import DecryptionMaterialsHandler, EncryptionMaterialsHandler -from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest -from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager -from aws_encryption_sdk.structures import EncryptedDataKey as Native_EncryptedDataKey - - -class CMMHandler(CryptoMaterialsManager): - """ - In instances where encryption materials may be provided by either - an implementation of the native - `aws_encryption_sdk.materials_managers.base.CryptoMaterialsManager` - or an implementation of the MPL's - `aws_cryptographic_materialproviders.mpl.references.ICryptographicMaterialsManager`, - this provides the correct materials based on the underlying materials manager. - """ - - native_cmm: CryptoMaterialsManager - mpl_cmm: 'ICryptographicMaterialsManager' - - def _is_using_native_cmm(self): - return hasattr(self, "native_cmm") and not hasattr(self, "mpl_cmm") - - def __init__( - self, - cmm: 'CryptoMaterialsManager | ICryptographicMaterialsManager' - ): - """ - Create DecryptionMaterialsHandler. - :param cmm: Underlying cryptographic materials manager - """ - if isinstance(cmm, CryptoMaterialsManager): - self.native_cmm = cmm - elif isinstance(cmm, ICryptographicMaterialsManager): - self.mpl_cmm = cmm - else: - raise ValueError(f"Invalid CMM passed to CMMHandler. cmm: {cmm}") - - def get_encryption_materials( - self, - request: EncryptionMaterialsRequest - ) -> EncryptionMaterialsHandler: - """ - Returns an EncryptionMaterialsHandler for the configured CMM. - :param request: Request for encryption materials - """ - if self._is_using_native_cmm(): - return EncryptionMaterialsHandler(self.native_cmm.get_encryption_materials(request)) - else: - try: - mpl_input: GetEncryptionMaterialsInput = CMMHandler._native_to_mpl_get_encryption_materials( - request - ) - mpl_output: GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) - return EncryptionMaterialsHandler(mpl_output.encryption_materials) - except AwsCryptographicMaterialProvidersException as mpl_exception: - # Wrap MPL error into the ESDK error type - # so customers only have to catch ESDK error types. - raise AWSEncryptionSDKClientError(mpl_exception) - - @staticmethod - def _native_to_mpl_get_encryption_materials( - request: EncryptionMaterialsRequest - ) -> 'GetEncryptionMaterialsInput': - output: GetEncryptionMaterialsInput = GetEncryptionMaterialsInput( - encryption_context=request.encryption_context, - commitment_policy=CMMHandler._native_to_mpl_commmitment_policy( - request.commitment_policy - ), - max_plaintext_length=request.plaintext_length, - ) - return output - - @staticmethod - def _native_to_mpl_commmitment_policy( - native_commitment_policy: CommitmentPolicy - ) -> 'CommitmentPolicyESDK': - if native_commitment_policy == CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT: - return CommitmentPolicyESDK(value="FORBID_ENCRYPT_ALLOW_DECRYPT") - elif native_commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT: - return CommitmentPolicyESDK(value="REQUIRE_ENCRYPT_ALLOW_DECRYPT") - elif native_commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT: - return CommitmentPolicyESDK(value="REQUIRE_ENCRYPT_REQUIRE_DECRYPT") - else: - raise ValueError(f"Invalid native_commitment_policy: {native_commitment_policy}") - - def decrypt_materials( - self, - request: DecryptionMaterialsRequest - ) -> DecryptionMaterialsHandler: - """ - Returns a DecryptionMaterialsHandler for the configured CMM. - :param request: Request for decryption materials - """ - if self._is_using_native_cmm(): - return DecryptionMaterialsHandler(self.native_cmm.decrypt_materials(request)) - else: - try: - mpl_input: 'DecryptMaterialsInput' = \ - CMMHandler._create_mpl_decrypt_materials_input_from_request(request) - mpl_output: 'DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(mpl_input) - return DecryptionMaterialsHandler(mpl_output.decryption_materials) - except AwsCryptographicMaterialProvidersException as mpl_exception: - # Wrap MPL error into the ESDK error type - # so customers only have to catch ESDK error types. - raise AWSEncryptionSDKClientError(mpl_exception) - - @staticmethod - def _native_algorithm_id_to_mpl_algorithm_id(native_algorithm_id: str) -> 'AlgorithmSuiteIdESDK': - # MPL algorithm suite ID = hexstr(native_algorithm_id) padded to 4 digits post-`x`. - return AlgorithmSuiteIdESDK(f"{native_algorithm_id:#0{6}x}") - - @staticmethod - def _create_mpl_decrypt_materials_input_from_request( - request: DecryptionMaterialsRequest - ) -> 'DecryptMaterialsInput': - key_blob_list: List[Native_EncryptedDataKey] = request.encrypted_data_keys - list_edks = [MPL_EncryptedDataKey( - key_provider_id=key_blob.key_provider.provider_id, - key_provider_info=key_blob.key_provider.key_info, - ciphertext=key_blob.encrypted_data_key, - ) for key_blob in key_blob_list] - output: DecryptMaterialsInput = DecryptMaterialsInput( - algorithm_suite_id=CMMHandler._native_algorithm_id_to_mpl_algorithm_id( - request.algorithm.algorithm_id - ), - commitment_policy=CMMHandler._native_to_mpl_commmitment_policy( - request.commitment_policy - ), - encrypted_data_keys=list_edks, - encryption_context=request.encryption_context, - ) - return output diff --git a/src/aws_encryption_sdk/internal/mpl/materials_handlers.py b/src/aws_encryption_sdk/internal/mpl/materials_handlers.py deleted file mode 100644 index 79312f863..000000000 --- a/src/aws_encryption_sdk/internal/mpl/materials_handlers.py +++ /dev/null @@ -1,164 +0,0 @@ -"""Provides encryption/decryption materials from an underlying materials provider.""" -# These dependencies are only loaded if you install the MPL. -try: - from aws_cryptographic_materialproviders.mpl.models import ( - DecryptionMaterials as MPL_DecryptionMaterials, - EncryptedDataKey as MPL_EncryptedDataKey, - EncryptionMaterials as MPL_EncryptionMaterials, - ) -except ImportError: - pass - -from typing import Dict, List, Set - -from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite -from aws_encryption_sdk.materials_managers import ( - DecryptionMaterials as Native_DecryptionMaterials, - EncryptionMaterials as Native_EncryptionMaterials, -) -from aws_encryption_sdk.structures import DataKey, EncryptedDataKey as Native_EncryptedDataKey, MasterKeyInfo - - -def _mpl_algorithm_id_to_native_algorithm_id(mpl_algorithm_id: str): - # MPL algorithm suite ID == hex(native algorithm suite ID) - return int(mpl_algorithm_id, 16) - - -class EncryptionMaterialsHandler: - """ - In instances where encryption materials may be provided by either - the native `aws_encryption_sdk.materials_managers.EncryptionMaterials` - or the MPL's `aws_cryptographic_materialproviders.mpl.models.EncryptionMaterials`, - this provides the correct materials based on the configured materials provider. - """ - - native_materials: Native_EncryptionMaterials - mpl_materials: 'MPL_EncryptionMaterials' - - def __init__( - self, - materials: 'Native_EncryptionMaterials | MPL_EncryptionMaterials' - ): - """ - Create EncryptionMaterialsHandler. - :param materials: Underlying encryption materials - """ - if isinstance(materials, Native_EncryptionMaterials): - self.native_materials = materials - elif isinstance(materials, MPL_EncryptionMaterials): - self.mpl_materials = materials - else: - raise ValueError("Invalid EncryptionMaterials passed to EncryptionMaterialsHandler. " - f"materials: {materials}") - - @property - def algorithm(self) -> Algorithm: - """Materials' native Algorithm.""" - if hasattr(self, "native_materials"): - return self.native_materials.algorithm - else: - return AlgorithmSuite.get_by_id( - _mpl_algorithm_id_to_native_algorithm_id( - self.mpl_materials.algorithm_suite.id.value - ) - ) - - @property - def encryption_context(self) -> Dict[str, str]: - """Materials' encryption context.""" - if hasattr(self, "native_materials"): - return self.native_materials.encryption_context - else: - return self.mpl_materials.encryption_context - - @property - def encrypted_data_keys(self) -> List[Native_EncryptedDataKey]: - """Materials' encrypted data keys.""" - if hasattr(self, "native_materials"): - return self.native_materials.encrypted_data_keys - else: - mpl_edk_list: List[MPL_EncryptedDataKey] = self.mpl_materials.encrypted_data_keys - key_blob_list: Set[Native_EncryptedDataKey] = {Native_EncryptedDataKey( - key_provider=MasterKeyInfo( - provider_id=mpl_edk.key_provider_id, - key_info=mpl_edk.key_provider_info, - ), - encrypted_data_key=mpl_edk.ciphertext, - ) for mpl_edk in mpl_edk_list} - return key_blob_list - - @property - def data_encryption_key(self) -> DataKey: - """Materials' data encryption key.""" - if hasattr(self, "native_materials"): - return self.native_materials.data_encryption_key - else: - mpl_dek = self.mpl_materials.plaintext_data_key - return DataKey( - # key_provider is unused, but the return type is DataKey - key_provider=MasterKeyInfo( - provider_id="", - key_info=b'' - ), - data_key=mpl_dek, - encrypted_data_key=b'', # No encrypted DEK - ) - - @property - def signing_key(self) -> bytes: - """Materials' signing key.""" - if hasattr(self, "native_materials"): - return self.native_materials.signing_key - else: - return self.mpl_materials.signing_key - - -class DecryptionMaterialsHandler: - """ - In instances where decryption materials may be provided by either - the native `aws_encryption_sdk.materials_managers.DecryptionMaterials` - or the MPL's `aws_cryptographic_materialproviders.mpl.models.DecryptionMaterials`, - this provides the correct materials based on the configured materials provider. - """ - - native_materials: Native_DecryptionMaterials - mpl_materials: 'MPL_DecryptionMaterials' - - def __init__( - self, - materials: 'Native_DecryptionMaterials | MPL_DecryptionMaterials' - ): - """ - Create DecryptionMaterialsHandler. - :param materials: Underlying decryption materials - """ - if isinstance(materials, Native_DecryptionMaterials): - self.native_materials = materials - elif isinstance(materials, MPL_DecryptionMaterials): - self.mpl_materials = materials - else: - raise ValueError(f"Invalid DecryptionMaterials passed to DecryptionMaterialsHandler.\ - materials: {materials}") - - @property - def data_key(self) -> DataKey: - """Materials' data key.""" - if hasattr(self, "native_materials"): - return self.native_materials.data_key - else: - return DataKey( - key_provider=MasterKeyInfo( - provider_id="", - key_info=b'' - ), - data_key=self.mpl_materials.plaintext_data_key, - encrypted_data_key=b'', - ) - - @property - def verification_key(self) -> bytes: - """Materials' verification key.""" - if hasattr(self, "native_materials"): - return self.native_materials.verification_key - else: - return self.mpl_materials.verification_key diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 032ed7d15..61f2f88c6 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -58,7 +58,7 @@ serialize_non_framed_close, serialize_non_framed_open, ) -from aws_encryption_sdk.internal.mpl.cmm_handler import CMMHandler +from aws_encryption_sdk.materials_managers.mpl.cmm import MPLCMMHandler from aws_encryption_sdk.internal.utils import exactly_one_arg_is_not_none from aws_encryption_sdk.internal.utils.commitment import ( validate_commitment_policy_on_decrypt, @@ -169,7 +169,7 @@ def _has_mpl_attrs_post_init(self): keyring=self.keyring ) ) - cmm_handler: CryptoMaterialsManager = CMMHandler(cmm) + cmm_handler: CryptoMaterialsManager = MPLCMMHandler(cmm) self.materials_manager = cmm_handler def _no_mpl_attrs_post_init(self): @@ -551,8 +551,7 @@ def _prep_message(self): else: # MPL verification key is PEM bytes, not DER bytes. # If the underlying CMM is from the MPL, load PEM bytes. - if (isinstance(self.config.materials_manager, CMMHandler) - and hasattr(self.config.materials_manager, "mpl_cmm")): + if (isinstance(self.config.materials_manager, MPLCMMHandler)): self.signer = Signer.from_key_bytes( algorithm=self._encryption_materials.algorithm, key_bytes=self._encryption_materials.signing_key, encoding=serialization.Encoding.PEM, @@ -919,8 +918,7 @@ def _read_header(self): else: # MPL verification key is NOT key bytes; it is bytes of the compressed point. # If the underlying CMM is from the MPL, load bytes from encoded point. - if (isinstance(self.config.materials_manager, CMMHandler) - and hasattr(self.config.materials_manager, "mpl_cmm")): + if (isinstance(self.config.materials_manager, MPLCMMHandler)): self.verifier = Verifier.from_encoded_point( algorithm=header.algorithm, encoded_point=base64.b64encode(decryption_materials.verification_key) diff --git a/test/unit/mpl/README.md b/test/unit/mpl/README.md deleted file mode 100644 index 839feb7a2..000000000 --- a/test/unit/mpl/README.md +++ /dev/null @@ -1 +0,0 @@ -Tests in this file REQUIRE the aws-cryptographic-material-providers module to be installed in order to run. \ No newline at end of file diff --git a/test/unit/mpl/test_cmm_handler.py b/test/unit/mpl/test_cmm_handler.py deleted file mode 100644 index d16374899..000000000 --- a/test/unit/mpl/test_cmm_handler.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. -"""Test suite to verify the cmm_handler module delegates correctly.""" -import pytest -from aws_cryptographic_materialproviders.mpl.models import ( - EncryptionMaterials as MPL_EncryptionMaterials, - GetEncryptionMaterialsInput, - GetEncryptionMaterialsOutput, -) -from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager -from mock import MagicMock, patch - -from aws_encryption_sdk.internal.mpl.cmm_handler import CMMHandler -from aws_encryption_sdk.internal.mpl.materials_handlers import EncryptionMaterialsHandler -from aws_encryption_sdk.materials_managers import ( - EncryptionMaterials as Native_EncryptionMaterials, - EncryptionMaterialsRequest, -) -from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager - -mock_native_cmm = MagicMock(__class__=CryptoMaterialsManager) -mock_mpl_cmm = MagicMock(__class__=ICryptographicMaterialsManager) -mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) -mock_encryption_materials_handler = MagicMock(__class__=EncryptionMaterialsHandler) -mock_native_encryption_materials = MagicMock(__class__=Native_EncryptionMaterials) -mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) - -pytestmark = [pytest.mark.unit, pytest.mark.local] - - -def test_GIVEN_native_CMM_WHEN_create_CMMHandler_THEN_is_using_native_cmm_returns_True(): - cmm_handler = CMMHandler(cmm=mock_native_cmm) - assert cmm_handler._is_using_native_cmm() - - -def test_GIVEN_mpl_CMM_WHEN_create_CMMHandler_THEN_is_using_native_cmm_returns_False(): - cmm_handler = CMMHandler(cmm=mock_mpl_cmm) - assert not cmm_handler._is_using_native_cmm() - - -def test_GIVEN_unknown_CMM_WHEN_create_CMMHandler_THEN_raise_ValueError(): - with pytest.raises(ValueError): - CMMHandler(cmm="not a CMM") - - -@patch.object(mock_native_cmm, "get_encryption_materials") -def test_GIVEN_native_CMM_WHEN_get_encryption_materials_THEN_return_native_encryption_materials( - mock_get_encryption_materials -): - # Mock: native_cmm.get_encryption_materials returns mock native encryption materials - mock_get_encryption_materials.return_value = mock_native_encryption_materials - - cmm_handler = CMMHandler(cmm=mock_native_cmm) - test = cmm_handler.get_encryption_materials(mock_encryption_materials_request) - - # Verify cmm_handler returns EncryptionMaterialsHandler - assert isinstance(test, EncryptionMaterialsHandler) - # Verify returned EncryptionMaterialsHandler uses the output of `get_encryption_materials` - assert test.native_materials == mock_native_encryption_materials - # Verify we actually called `get_encryption_materials` - mock_native_cmm.get_encryption_materials.assert_called_once_with(mock_encryption_materials_request) - - -@patch.object(mock_mpl_cmm, "get_encryption_materials") -@patch("aws_encryption_sdk.internal.mpl.cmm_handler.CMMHandler._native_to_mpl_get_encryption_materials") -def test_GIVEN_mpl_CMM_WHEN_get_encryption_materials_THEN_return_mpl_encryption_materials( - mock_native_to_mpl_get_encryption_materials, - mock_get_encryption_materials, -): - # Mock: mpl_cmm.get_encryption_materials returns mock MPL encryption materials - mock_get_encryption_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) - mock_get_encryption_materials_output.encryption_materials = mock_mpl_encryption_materials - mock_get_encryption_materials.return_value = mock_get_encryption_materials_output - - # Mock: CMMHandler._native_to_mpl_get_encryption_materials creates a GetEncryptionMaterialsInput - mock_get_encryption_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) - mock_native_to_mpl_get_encryption_materials.return_value = mock_get_encryption_materials_input - - cmm_handler = CMMHandler(cmm=mock_mpl_cmm) - test = cmm_handler.get_encryption_materials(mock_encryption_materials_request) - - # Verify cmm_handler returns EncryptionMaterialsHandler - assert isinstance(test, EncryptionMaterialsHandler) - # Verify returned EncryptionMaterialsHandler uses the output of `get_encryption_materials` - assert test.mpl_materials == mock_mpl_encryption_materials - # Verify we actually called `get_encryption_materials` - mock_mpl_cmm.get_encryption_materials.assert_called_once_with(mock_get_encryption_materials_input) From a594125a635c0741e121d87bf367940b22f7610e Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 22 Feb 2024 12:07:33 -0800 Subject: [PATCH 103/376] refactor --- .../materials_managers/mpl/__init__.py | 13 ++ .../materials_managers/mpl/cmm.py | 142 ++++++++++++++++++ .../materials_managers/mpl/materials.py | 135 +++++++++++++++++ 3 files changed, 290 insertions(+) create mode 100644 src/aws_encryption_sdk/materials_managers/mpl/__init__.py create mode 100644 src/aws_encryption_sdk/materials_managers/mpl/cmm.py create mode 100644 src/aws_encryption_sdk/materials_managers/mpl/materials.py diff --git a/src/aws_encryption_sdk/materials_managers/mpl/__init__.py b/src/aws_encryption_sdk/materials_managers/mpl/__init__.py new file mode 100644 index 000000000..295400d76 --- /dev/null +++ b/src/aws_encryption_sdk/materials_managers/mpl/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Modules related to the MPL's materials managers interfaces.""" diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py new file mode 100644 index 000000000..e16b49d51 --- /dev/null +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -0,0 +1,142 @@ +"""Retrieves encryption/decryption materials from the MPL.""" + +# These dependencies are only loaded if you install the MPL. +try: + # pylint seems to struggle with this conditional import + # pylint: disable=unused-import + from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException + from aws_cryptographic_materialproviders.mpl.models import ( + AlgorithmSuiteIdESDK, + CommitmentPolicyESDK, + DecryptMaterialsInput, + DecryptMaterialsOutput, + EncryptedDataKey as MPL_EncryptedDataKey, + GetEncryptionMaterialsInput, + GetEncryptionMaterialsOutput, + ) + from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager + +except ImportError: + pass + +from typing import List + +from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError +from aws_encryption_sdk.identifiers import CommitmentPolicy +from aws_encryption_sdk.materials_managers.mpl.materials import MPLEncryptionMaterials, MPLDecryptionMaterials +from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest +from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager +from aws_encryption_sdk.structures import EncryptedDataKey as Native_EncryptedDataKey + + +class MPLCMMHandler(CryptoMaterialsManager): + """ + In instances where encryption materials are provided by an implementation of the MPL's + `aws_cryptographic_materialproviders.mpl.references.ICryptographicMaterialsManager`, + this maps the ESDK CMM interfaces to the MPL CMM. + """ + + mpl_cmm: 'ICryptographicMaterialsManager' + + def __init__( + self, + mpl_cmm: 'ICryptographicMaterialsManager' + ): + """ + Create DecryptionMaterialsHandler. + :param cmm: Underlying cryptographic materials manager + """ + if isinstance(mpl_cmm, ICryptographicMaterialsManager): + self.mpl_cmm = mpl_cmm + else: + raise ValueError(f"Invalid CMM passed to MPLCMMHandler. cmm: {mpl_cmm}") + + def get_encryption_materials( + self, + request: EncryptionMaterialsRequest + ) -> MPLEncryptionMaterials: + """ + Returns an EncryptionMaterialsHandler for the configured CMM. + :param request: Request for encryption materials + """ + try: + mpl_input: GetEncryptionMaterialsInput = MPLCMMHandler._native_to_mpl_get_encryption_materials( + request + ) + mpl_output: GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) + return MPLEncryptionMaterials(mpl_output.encryption_materials) + except AwsCryptographicMaterialProvidersException as mpl_exception: + # Wrap MPL error into the ESDK error type + # so customers only have to catch ESDK error types. + raise AWSEncryptionSDKClientError(mpl_exception) + + @staticmethod + def _native_to_mpl_get_encryption_materials( + request: EncryptionMaterialsRequest + ) -> 'GetEncryptionMaterialsInput': + output: GetEncryptionMaterialsInput = GetEncryptionMaterialsInput( + encryption_context=request.encryption_context, + commitment_policy=MPLCMMHandler._native_to_mpl_commmitment_policy( + request.commitment_policy + ), + max_plaintext_length=request.plaintext_length, + ) + return output + + @staticmethod + def _native_to_mpl_commmitment_policy( + native_commitment_policy: CommitmentPolicy + ) -> 'CommitmentPolicyESDK': + if native_commitment_policy == CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT: + return CommitmentPolicyESDK(value="FORBID_ENCRYPT_ALLOW_DECRYPT") + elif native_commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT: + return CommitmentPolicyESDK(value="REQUIRE_ENCRYPT_ALLOW_DECRYPT") + elif native_commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT: + return CommitmentPolicyESDK(value="REQUIRE_ENCRYPT_REQUIRE_DECRYPT") + else: + raise ValueError(f"Invalid native_commitment_policy: {native_commitment_policy}") + + def decrypt_materials( + self, + request: DecryptionMaterialsRequest + ) -> MPLDecryptionMaterials: + """ + Returns a DecryptionMaterialsHandler for the configured CMM. + :param request: Request for decryption materials + """ + try: + mpl_input: 'DecryptMaterialsInput' = \ + MPLCMMHandler._create_mpl_decrypt_materials_input_from_request(request) + mpl_output: 'DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(mpl_input) + return MPLDecryptionMaterials(mpl_output.decryption_materials) + except AwsCryptographicMaterialProvidersException as mpl_exception: + # Wrap MPL error into the ESDK error type + # so customers only have to catch ESDK error types. + raise AWSEncryptionSDKClientError(mpl_exception) + + @staticmethod + def _native_algorithm_id_to_mpl_algorithm_id(native_algorithm_id: str) -> 'AlgorithmSuiteIdESDK': + # MPL algorithm suite ID = hexstr(native_algorithm_id) padded to 4 digits post-`x`. + return AlgorithmSuiteIdESDK(f"{native_algorithm_id:#0{6}x}") + + @staticmethod + def _create_mpl_decrypt_materials_input_from_request( + request: DecryptionMaterialsRequest + ) -> 'DecryptMaterialsInput': + key_blob_list: List[Native_EncryptedDataKey] = request.encrypted_data_keys + list_edks = [MPL_EncryptedDataKey( + key_provider_id=key_blob.key_provider.provider_id, + key_provider_info=key_blob.key_provider.key_info, + ciphertext=key_blob.encrypted_data_key, + ) for key_blob in key_blob_list] + output: DecryptMaterialsInput = DecryptMaterialsInput( + algorithm_suite_id=MPLCMMHandler._native_algorithm_id_to_mpl_algorithm_id( + request.algorithm.algorithm_id + ), + commitment_policy=MPLCMMHandler._native_to_mpl_commmitment_policy( + request.commitment_policy + ), + encrypted_data_keys=list_edks, + encryption_context=request.encryption_context, + ) + return output diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py new file mode 100644 index 000000000..fdcf2ec06 --- /dev/null +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -0,0 +1,135 @@ +"""Provides encryption/decryption materials from an underlying materials provider.""" +# These dependencies are only loaded if you install the MPL. +try: + from aws_cryptographic_materialproviders.mpl.models import ( + DecryptionMaterials as MPL_DecryptionMaterials, + EncryptedDataKey as MPL_EncryptedDataKey, + EncryptionMaterials as MPL_EncryptionMaterials, + ) +except ImportError: + pass + +from typing import Dict, List, Set + +from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite +from aws_encryption_sdk.materials_managers import ( + DecryptionMaterials as Native_DecryptionMaterials, + EncryptionMaterials as Native_EncryptionMaterials, +) +from aws_encryption_sdk.structures import DataKey, EncryptedDataKey as Native_EncryptedDataKey, MasterKeyInfo + + +def _mpl_algorithm_id_to_native_algorithm_id(mpl_algorithm_id: str) -> int: + # MPL algorithm suite ID == hex(native algorithm suite ID) + return int(mpl_algorithm_id, 16) + + +class MPLEncryptionMaterials(Native_EncryptionMaterials): + """ + In instances where encryption materials are be provided by + the MPL's `aws_cryptographic_materialproviders.mpl.models.EncryptionMaterials`, + this maps the ESDK interfaces to the underlying MPL materials. + """ + + mpl_materials: 'MPL_EncryptionMaterials' + + def __init__( + self, + materials: 'MPL_EncryptionMaterials' + ): + """ + Create MPLEncryptionMaterialsHandler. + :param materials: Underlying encryption materials + """ + if isinstance(materials, MPL_EncryptionMaterials): + self.mpl_materials = materials + else: + raise ValueError("Invalid EncryptionMaterials passed to EncryptionMaterialsHandler. " + f"materials: {materials}") + + @property + def algorithm(self) -> Algorithm: + """Materials' native Algorithm.""" + return AlgorithmSuite.get_by_id( + _mpl_algorithm_id_to_native_algorithm_id( + self.mpl_materials.algorithm_suite.id.value + ) + ) + + @property + def encryption_context(self) -> Dict[str, str]: + """Materials' encryption context.""" + return self.mpl_materials.encryption_context + + @property + def encrypted_data_keys(self) -> List[Native_EncryptedDataKey]: + """Materials' encrypted data keys.""" + mpl_edk_list: List[MPL_EncryptedDataKey] = self.mpl_materials.encrypted_data_keys + key_blob_list: Set[Native_EncryptedDataKey] = {Native_EncryptedDataKey( + key_provider=MasterKeyInfo( + provider_id=mpl_edk.key_provider_id, + key_info=mpl_edk.key_provider_info, + ), + encrypted_data_key=mpl_edk.ciphertext, + ) for mpl_edk in mpl_edk_list} + return key_blob_list + + @property + def data_encryption_key(self) -> DataKey: + """Materials' data encryption key.""" + mpl_dek = self.mpl_materials.plaintext_data_key + return DataKey( + # key_provider is unused, but the return type is DataKey + key_provider=MasterKeyInfo( + provider_id="", + key_info=b'' + ), + data_key=mpl_dek, + encrypted_data_key=b'', # No encrypted DEK + ) + + @property + def signing_key(self) -> bytes: + """Materials' signing key.""" + return self.mpl_materials.signing_key + + +class MPLDecryptionMaterials(Native_DecryptionMaterials): + """ + In instances where decryption materials are be provided by + the MPL's `aws_cryptographic_materialproviders.mpl.models.DecryptionMaterials`, + this maps the ESDK interfaces to the underlying MPL materials. + """ + + mpl_materials: 'MPL_DecryptionMaterials' + + def __init__( + self, + materials: 'MPL_DecryptionMaterials' + ): + """ + Create DecryptionMaterialsHandler. + :param materials: Underlying decryption materials + """ + if isinstance(materials, MPL_DecryptionMaterials): + self.mpl_materials = materials + else: + raise ValueError(f"Invalid DecryptionMaterials passed to DecryptionMaterialsHandler.\ + materials: {materials}") + + @property + def data_key(self) -> DataKey: + """Materials' data key.""" + return DataKey( + key_provider=MasterKeyInfo( + provider_id="", + key_info=b'' + ), + data_key=self.mpl_materials.plaintext_data_key, + encrypted_data_key=b'', + ) + + @property + def verification_key(self) -> bytes: + """Materials' verification key.""" + return self.mpl_materials.verification_key From fdd2eda60f42aeea832b7851db1487dfb2784882 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 10:03:34 -0800 Subject: [PATCH 104/376] unit tests --- .../internal/mpl/__init__.py | 13 -------- .../materials_managers/mpl/cmm.py | 20 ++++++----- .../materials_managers/mpl/materials.py | 33 +++++++++++-------- 3 files changed, 32 insertions(+), 34 deletions(-) delete mode 100644 src/aws_encryption_sdk/internal/mpl/__init__.py diff --git a/src/aws_encryption_sdk/internal/mpl/__init__.py b/src/aws_encryption_sdk/internal/mpl/__init__.py deleted file mode 100644 index 41497cc20..000000000 --- a/src/aws_encryption_sdk/internal/mpl/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. -"""Modules related to the MPL.""" diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index e16b49d51..cd789b994 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -15,9 +15,9 @@ GetEncryptionMaterialsOutput, ) from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager - + _HAS_MPL = True except ImportError: - pass + _HAS_MPL = False from typing import List @@ -43,9 +43,12 @@ def __init__( mpl_cmm: 'ICryptographicMaterialsManager' ): """ - Create DecryptionMaterialsHandler. - :param cmm: Underlying cryptographic materials manager + Create MPLCMMHandler. + :param mpl_cmm: Underlying MPL cryptographic materials manager """ + if not _HAS_MPL: + raise ImportError("You MUST install the aws-cryptographic-material-providers " + f"library to create an instance of {MPLCMMHandler}") if isinstance(mpl_cmm, ICryptographicMaterialsManager): self.mpl_cmm = mpl_cmm else: @@ -74,11 +77,12 @@ def get_encryption_materials( def _native_to_mpl_get_encryption_materials( request: EncryptionMaterialsRequest ) -> 'GetEncryptionMaterialsInput': + commitment_policy = MPLCMMHandler._native_to_mpl_commmitment_policy( + request.commitment_policy + ) output: GetEncryptionMaterialsInput = GetEncryptionMaterialsInput( encryption_context=request.encryption_context, - commitment_policy=MPLCMMHandler._native_to_mpl_commmitment_policy( - request.commitment_policy - ), + commitment_policy=commitment_policy, max_plaintext_length=request.plaintext_length, ) return output @@ -101,7 +105,7 @@ def decrypt_materials( request: DecryptionMaterialsRequest ) -> MPLDecryptionMaterials: """ - Returns a DecryptionMaterialsHandler for the configured CMM. + Returns a MPLDecryptionMaterials for the configured CMM. :param request: Request for decryption materials """ try: diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index fdcf2ec06..bd4b5f729 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -6,8 +6,9 @@ EncryptedDataKey as MPL_EncryptedDataKey, EncryptionMaterials as MPL_EncryptionMaterials, ) + _HAS_MPL = True except ImportError: - pass + _HAS_MPL = False from typing import Dict, List, Set @@ -35,17 +36,20 @@ class MPLEncryptionMaterials(Native_EncryptionMaterials): def __init__( self, - materials: 'MPL_EncryptionMaterials' + mpl_materials: 'MPL_EncryptionMaterials' ): """ - Create MPLEncryptionMaterialsHandler. + Create MPLEncryptionMaterials. :param materials: Underlying encryption materials """ - if isinstance(materials, MPL_EncryptionMaterials): - self.mpl_materials = materials + if not _HAS_MPL: + raise ImportError("You MUST install the aws-cryptographic-material-providers " + f"library to create an instance of {MPLEncryptionMaterials}") + if isinstance(mpl_materials, MPL_EncryptionMaterials): + self.mpl_materials = mpl_materials else: - raise ValueError("Invalid EncryptionMaterials passed to EncryptionMaterialsHandler. " - f"materials: {materials}") + raise ValueError("Invalid EncryptionMaterials passed to MPLEncryptionMaterials. " + f"materials: {mpl_materials}") @property def algorithm(self) -> Algorithm: @@ -105,17 +109,20 @@ class MPLDecryptionMaterials(Native_DecryptionMaterials): def __init__( self, - materials: 'MPL_DecryptionMaterials' + mpl_materials: 'MPL_DecryptionMaterials' ): """ - Create DecryptionMaterialsHandler. + Create MPLDecryptionMaterials. :param materials: Underlying decryption materials """ - if isinstance(materials, MPL_DecryptionMaterials): - self.mpl_materials = materials + if not _HAS_MPL: + raise ImportError("You MUST install the aws-cryptographic-material-providers " + f"library to create an instance of {MPLDecryptionMaterials}") + if isinstance(mpl_materials, MPL_DecryptionMaterials): + self.mpl_materials = mpl_materials else: - raise ValueError(f"Invalid DecryptionMaterials passed to DecryptionMaterialsHandler.\ - materials: {materials}") + raise ValueError(f"Invalid DecryptionMaterials passed to MPLDecryptionMaterials.\ + materials: {mpl_materials}") @property def data_key(self) -> DataKey: From 0138f226a73bff13fe1e24d865b09ec7e2ff42b2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 10:03:47 -0800 Subject: [PATCH 105/376] unit tests --- test/unit/test_material_managers_mpl_cmm.py | 278 ++++++++++++++++++ .../test_material_managers_mpl_materials.py | 221 ++++++++++++++ 2 files changed, 499 insertions(+) create mode 100644 test/unit/test_material_managers_mpl_cmm.py create mode 100644 test/unit/test_material_managers_mpl_materials.py diff --git a/test/unit/test_material_managers_mpl_cmm.py b/test/unit/test_material_managers_mpl_cmm.py new file mode 100644 index 000000000..77bf5502d --- /dev/null +++ b/test/unit/test_material_managers_mpl_cmm.py @@ -0,0 +1,278 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" + +import pytest +from mock import MagicMock, patch + + +from aws_encryption_sdk.identifiers import CommitmentPolicy +import aws_encryption_sdk.materials_managers.mpl.cmm +from aws_encryption_sdk.materials_managers.mpl.cmm import MPLCMMHandler +from aws_encryption_sdk.materials_managers.mpl.materials import ( + MPLEncryptionMaterials, + MPLDecryptionMaterials, +) + +pytestmark = [pytest.mark.unit, pytest.mark.local] + + +# Check if MPL is installed, and skip tests based on its installation status +# Ideally, this logic would be based on mocking imports and testing logic, +# but doing that introduces errors that cause other tests to fail. +try: + from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException + from aws_cryptographic_materialproviders.mpl.models import ( + AlgorithmSuiteIdESDK, + CommitmentPolicyESDK, + DecryptMaterialsInput, + DecryptionMaterials as MPL_DecryptionMaterials, + EncryptionMaterials as MPL_EncryptionMaterials, + GetEncryptionMaterialsInput, + GetEncryptionMaterialsOutput, + ) + from aws_cryptographic_materialproviders.mpl.references import ( + ICryptographicMaterialsManager + ) + HAS_MPL = True + + mock_mpl_cmm = MagicMock(__class__=ICryptographicMaterialsManager) + mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) + mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) + +except ImportError: + HAS_MPL = False + + # Ensure references to these mocks exist, even if they aren't used in a non-MPL context + mock_mpl_cmm = None + mock_mpl_encryption_materials = None + mock_mpl_decrypt_materials = None + +from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError +from aws_encryption_sdk.materials_managers import ( + EncryptionMaterialsRequest, + DecryptionMaterialsRequest, +) + + +mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) +mock_encryption_materials_handler = MagicMock(__class__=MPLEncryptionMaterials) +mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) + +@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") +def test_GIVEN_test_has_mpl_is_False_THEN_cmm_has_mpl_is_False(): + """If the MPL IS NOT installed in the runtime environment, + assert the cmm has _HAS_MPL set to False""" + + assert hasattr(aws_encryption_sdk.materials_managers.mpl.cmm, "_HAS_MPL") + assert aws_encryption_sdk.materials_managers.mpl.cmm._HAS_MPL is False + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_test_has_mpl_is_True_THEN_cmm_has_mpl_is_True(): + """If the MPL IS installed in the runtime environment, + assert the cmm has _HAS_MPL set to True""" + + assert hasattr(aws_encryption_sdk.materials_managers.mpl.cmm, "_HAS_MPL") + assert aws_encryption_sdk.materials_managers.mpl.cmm._HAS_MPL is True + + +@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_THEN_raise_ImportError(): + with pytest.raises(ImportError): + MPLCMMHandler(mpl_cmm="doesn't matter") + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_valid_mpl_cmm_THEN_return_new_MPLCMMHandler(): + mpl_cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) + + assert mpl_cmm_handler.mpl_cmm == mock_mpl_cmm + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_invalid_mpl_cmm_THEN_raise_ValueError(): + with pytest.raises(ValueError): + MPLCMMHandler(mpl_cmm="not a valid mpl_cmm") + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +@patch.object(mock_mpl_cmm, "get_encryption_materials") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_get_encryption_materials") +def test_GIVEN_valid_request_WHEN_call_get_encryption_materials_THEN_return_MPLEncryptionMaterials( + mock_native_to_mpl_get_encryption_materials, + mock_get_encryption_materials, +): + + # Mock: mpl_cmm.get_encryption_materials returns mock MPL encryption materials + mock_get_encryption_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) + mock_get_encryption_materials_output.encryption_materials = mock_mpl_encryption_materials + mock_get_encryption_materials.return_value = mock_get_encryption_materials_output + + # Mock: CMMHandler._native_to_mpl_get_encryption_materials creates a GetEncryptionMaterialsInput + mock_get_encryption_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) + mock_native_to_mpl_get_encryption_materials.return_value = mock_get_encryption_materials_input + + cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) + test = cmm_handler.get_encryption_materials(mock_encryption_materials_request) + + # Verify cmm_handler returns MPLEncryptionMaterials + assert isinstance(test, MPLEncryptionMaterials) + # Verify returned EncryptionMaterialsHandler uses the output of `get_encryption_materials` + assert test.mpl_materials == mock_mpl_encryption_materials + # Verify we actually called `get_encryption_materials` + mock_mpl_cmm.get_encryption_materials.assert_called_once_with(mock_get_encryption_materials_input) + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") +def test_GIVEN_get_encryption_materials_raises_MPL_Exception_WHEN_call_get_encryption_materials_THEN_raise_ESDK_Exception( + _ +): + with pytest.raises(AWSEncryptionSDKClientError): + with patch.object(mock_mpl_cmm, "get_encryption_materials", + side_effect=AwsCryptographicMaterialProvidersException("any")): + + cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) + cmm_handler.get_encryption_materials(mock_encryption_materials_request) + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") +def test_GIVEN_native_to_mpl_commmitment_policy_returns_valid_policy_WHEN_call_native_to_mpl_get_encryption_materials_THEN_returns_GetEncryptionMaterialsInput( + mock_mpl_commitment_policy +): + mock_commitment_policy = MagicMock(__class__=CommitmentPolicyESDK) + mock_mpl_commitment_policy.return_value = mock_commitment_policy + + output = MPLCMMHandler._native_to_mpl_get_encryption_materials(mock_encryption_materials_request) + + # verify correctness of returned value + assert isinstance(output, GetEncryptionMaterialsInput) + assert output.encryption_context == mock_encryption_materials_request.encryption_context + assert output.commitment_policy == mock_commitment_policy + assert output.max_plaintext_length == mock_encryption_materials_request.plaintext_length + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_FORBID_ENCRYPT_ALLOW_DECRYPT(): + native_commitment_policy = CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT + + output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) + + assert isinstance(output, CommitmentPolicyESDK) + assert output.value == "FORBID_ENCRYPT_ALLOW_DECRYPT" + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_ALLOW_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_REQUIRE_ENCRYPT_ALLOW_DECRYPT(): + native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT + + output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) + + assert isinstance(output, CommitmentPolicyESDK) + assert output.value == "REQUIRE_ENCRYPT_ALLOW_DECRYPT" + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_REQUIRE_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_REQUIRE_ENCRYPT_REQUIRE_DECRYPT(): + native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT + + output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) + + assert isinstance(output, CommitmentPolicyESDK) + assert output.value == "REQUIRE_ENCRYPT_REQUIRE_DECRYPT" + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_CommitmentPolicy_unrecognized_WHEN_call_native_to_mpl_commmitment_policyTHEN_raise_ValueError(): + native_commitment_policy = "not a commitment policy" + + with pytest.raises(ValueError): + MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +@patch.object(mock_mpl_cmm, "decrypt_materials") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._create_mpl_decrypt_materials_input_from_request") +def test_GIVEN_valid_request_WHEN_call_decrypt_materials_THEN_return_MPLDecryptionMaterials( + mock_native_to_mpl_decrypt_materials, + mock_get_encryption_materials, +): + + # Mock: mpl_cmm.get_decryption_materials returns mock MPL decryption materials + mock_decrypt_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) + mock_decrypt_materials_output.decryption_materials = mock_mpl_decrypt_materials + mock_get_encryption_materials.return_value = mock_decrypt_materials_output + + # Mock: CMMHandler._create_mpl_decrypt_materials_input_from_request creates a DecryptMaterialsInput + mock_decrypt_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) + mock_native_to_mpl_decrypt_materials.return_value = mock_decrypt_materials_input + + cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) + output = cmm_handler.decrypt_materials(mock_decryption_materials_request) + + # Verify cmm_handler returns MPLDecryptionMaterials + assert isinstance(output, MPLDecryptionMaterials) + # Verify returned MPLDecryptionMaterials uses the output of `decrypt_materials` + assert output.mpl_materials == mock_mpl_decrypt_materials + # Verify we actually called `decrypt_materials` + mock_mpl_cmm.decrypt_materials.assert_called_once_with(mock_decrypt_materials_input) + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._create_mpl_decrypt_materials_input_from_request") +def test_GIVEN_decrypt_materials_raises_MPL_Exception_WHEN_call_decrypt_materials_THEN_raise_ESDK_Exception( + _ +): + with pytest.raises(AWSEncryptionSDKClientError): + with patch.object(mock_mpl_cmm, "decrypt_materials", + side_effect=AwsCryptographicMaterialProvidersException("any")): + + cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) + cmm_handler.decrypt_materials(mock_decryption_materials_request) + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_WHEN_call_native_algorithm_id_to_mpl_algorithm_id_THEN_returns_valid_AlgorithmSuiteIdESDK(): + some_native_algorithm_id = 0x0000 # Not a real algorithm ID, but fits the format + + mpl_output = MPLCMMHandler._native_algorithm_id_to_mpl_algorithm_id( + some_native_algorithm_id + ) + + assert isinstance(mpl_output, AlgorithmSuiteIdESDK) + assert mpl_output.value == "0x0000" + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_algorithm_id_to_mpl_algorithm_id") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") +def test__create_mpl_decrypt_materials_input_from_request( + mock_mpl_commitment_policy, + mock_mpl_algorithm_id, +): + mock_algorithm_id = "0x1234" # Some fake algorithm ID that fits the format + mock_mpl_algorithm_id.return_value = mock_algorithm_id + mock_commitment_policy = MagicMock(__class__=CommitmentPolicyESDK) + mock_mpl_commitment_policy.return_value = mock_commitment_policy + + # mock_decryption_materials_request.algorithm = + + output = MPLCMMHandler._create_mpl_decrypt_materials_input_from_request(mock_decryption_materials_request) + + assert isinstance(output, DecryptMaterialsInput) + assert output.algorithm_suite_id == mock_algorithm_id + assert output.commitment_policy == mock_commitment_policy + assert output.encryption_context == mock_decryption_materials_request.encryption_context + + assert len(output.encrypted_data_keys) == len(mock_decryption_materials_request.encrypted_data_keys) + for i in range(len(output.encrypted_data_keys)): + # Assume input[i] == output[i], seems to work + output_edk = output.encrypted_data_keys[i] + input_edk = mock_decryption_materials_request[i] + assert output_edk.key_provider_id == input_edk.key_provider.provider_id + assert output_edk.key_provider_info == input_edk.key_provider.key_info + assert output_edk.ciphertext == input_edk.encrypted_data_key diff --git a/test/unit/test_material_managers_mpl_materials.py b/test/unit/test_material_managers_mpl_materials.py new file mode 100644 index 000000000..250efeb7e --- /dev/null +++ b/test/unit/test_material_managers_mpl_materials.py @@ -0,0 +1,221 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" + +import pytest +from mock import MagicMock, patch, PropertyMock +from typing import Dict, List + +from aws_encryption_sdk.identifiers import CommitmentPolicy +import aws_encryption_sdk.materials_managers.mpl.materials +from aws_encryption_sdk.materials_managers.mpl.materials import ( + MPLEncryptionMaterials, + MPLDecryptionMaterials, +) +from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite + +pytestmark = [pytest.mark.unit, pytest.mark.local] + + +# Check if MPL is installed, and skip tests based on its installation status +# Ideally, this logic would be based on mocking imports and testing logic, +# but doing that introduces errors that cause other tests to fail. +try: + from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException + from aws_cryptographic_materialproviders.mpl.models import ( + AlgorithmSuiteIdESDK, + CommitmentPolicyESDK, + DecryptMaterialsInput, + DecryptionMaterials as MPL_DecryptionMaterials, + EncryptedDataKey as MPL_EncryptedDataKey, + EncryptionMaterials as MPL_EncryptionMaterials, + GetEncryptionMaterialsInput, + GetEncryptionMaterialsOutput, + ) + from aws_cryptographic_materialproviders.mpl.references import ( + ICryptographicMaterialsManager + ) + HAS_MPL = True + + mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) + mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) + +except ImportError: + HAS_MPL = False + + # Ensure references to these mocks exist, even if they aren't used in a non-MPL context + mock_mpl_cmm = None + mock_mpl_encryption_materials = None + mock_mpl_decrypt_materials = None + +from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError +from aws_encryption_sdk.materials_managers import ( + EncryptionMaterialsRequest, + DecryptionMaterialsRequest, +) + + +mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) +mock_encryption_materials_handler = MagicMock(__class__=MPLEncryptionMaterials) +mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) + +@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") +def test_GIVEN_test_has_mpl_is_False_THEN_cmm_has_mpl_is_False(): + """If the MPL IS NOT installed in the runtime environment, + assert the cmm has _HAS_MPL set to False""" + + assert hasattr(aws_encryption_sdk.materials_managers.mpl.materials, "_HAS_MPL") + assert aws_encryption_sdk.materials_managers.mpl.materials._HAS_MPL is False + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_test_has_mpl_is_True_THEN_cmm_has_mpl_is_True(): + """If the MPL IS installed in the runtime environment, + assert the cmm has _HAS_MPL set to True""" + + assert hasattr(aws_encryption_sdk.materials_managers.mpl.materials, "_HAS_MPL") + assert aws_encryption_sdk.materials_managers.mpl.materials._HAS_MPL is True + + +@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_THEN_raise_ImportError(): + with pytest.raises(ImportError): + MPLEncryptionMaterials(mpl_materials="doesn't matter") + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_valid_mpl_cmm_THEN_return_new_MPLCMMHandler(): + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + + assert mpl_encryption_materials.mpl_materials == mock_mpl_encryption_materials + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_invalid_mpl_cmm_THEN_raise_ValueError(): + with pytest.raises(ValueError): + MPLEncryptionMaterials(mpl_materials="not a valid mpl_materials") + +def test_mpl_to_native(): + some_mpl_algorithm_id = "0x1234" # Not a real algorithm ID, but fits the format + + native_output = aws_encryption_sdk.materials_managers.mpl.materials._mpl_algorithm_id_to_native_algorithm_id( + some_mpl_algorithm_id + ) + + assert native_output == 0x1234 + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +@patch("aws_encryption_sdk.materials_managers.mpl.materials._mpl_algorithm_id_to_native_algorithm_id") +@patch("aws_encryption_sdk.materials_managers.mpl.materials.AlgorithmSuite.get_by_id") +def test_GIVEN_valid_mpl_algorithm_id_WHEN_get_algorithm_THEN_valid_native_algorithm_id( + mock_algorithm, + mock_native_algorithm_id, +): + # Mock valid conversion from MPL to native algorithm ID + mock_native_algorithm_id.return_value = 0x1234 + + # Mock valid lookup in native AlgorithmSuite lookup + mock_algorithm.return_value = MagicMock(__class__=AlgorithmSuite) + + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + output = mpl_encryption_materials.algorithm + assert output == mock_algorithm() # property calls automatically, we need to call the mock + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GecTHEN_valid_native_algorithm_id(): + mock_encryption_context = MagicMock(__class__=Dict[str, str]) + mock_mpl_encryption_materials.encryption_context = mock_encryption_context + + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + output = mpl_encryption_materials.encryption_context + + assert output == mock_encryption_context + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GecTHEN_valid_nativefadsf_algorithm_id(): + mock_edk = MagicMock(__class__=MPL_EncryptedDataKey) + mock_mpl_key_provider_id = MagicMock(__class__=str) + mock_edk.key_provider_id = mock_mpl_key_provider_id + mock_mpl_key_provider_info = MagicMock(__class__=bytes) + mock_edk.key_provider_info = mock_mpl_key_provider_info + mock_mpl_ciphertext = MagicMock(__class__=bytes) + mock_edk.ciphertext = mock_mpl_ciphertext + + mock_edks = [ mock_edk ] + mock_mpl_encryption_materials.encrypted_data_keys = mock_edks + + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + output = mpl_encryption_materials.encrypted_data_keys + output_as_list = list(output) + + assert len(output_as_list) == len(mock_edks) + for i in range(len(output_as_list)): + # assume output[i] corresponds to input[i] + native_edk = output_as_list[i] + mpl_edk = mock_edks[i] + + assert native_edk.encrypted_data_key == mpl_edk.ciphertext + assert native_edk.key_provider.provider_id == mpl_edk.key_provider_id + assert native_edk.key_provider.key_info == mpl_edk.key_provider_info + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GecTHEN_valid_nativefadsffadsfa_algorithm_id(): + mock_data_key = MagicMock(__class__=bytes) + mock_mpl_encryption_materials.plaintext_data_key = mock_data_key + + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + output = mpl_encryption_materials.data_encryption_key + + assert output.key_provider.provider_id == "" + assert output.key_provider.key_info == b"" + assert output.data_key == mock_data_key + assert output.encrypted_data_key == b"" + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GecTHEN_valid_nativefasdfasdffadsf_algorithm_id(): + mock_signing_key = MagicMock(__class__=bytes) + mock_mpl_encryption_materials.signing_key = mock_signing_key + + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + output = mpl_encryption_materials.signing_key + + assert output == mock_signing_key + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GecTHEN_valid_nativeffasdfasdadsffadsfa_algorithm_id(): + mock_data_key = MagicMock(__class__=bytes) + mock_mpl_decrypt_materials.plaintext_data_key = mock_data_key + + mpl_decryption_materials = MPLDecryptionMaterials(mpl_materials=mock_mpl_decrypt_materials) + output = mpl_decryption_materials.data_key + + assert output.key_provider.provider_id == "" + assert output.key_provider.key_info == b"" + assert output.data_key == mock_data_key + assert output.encrypted_data_key == b"" + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GecTHEN_validadsfasdf_nativefasdfasdffadsf_algorithm_id(): + mock_verification_key = MagicMock(__class__=bytes) + mock_mpl_decrypt_materials.verification_key = mock_verification_key + + mpl_decryption_materials = MPLDecryptionMaterials(mpl_materials=mock_mpl_decrypt_materials) + output = mpl_decryption_materials.verification_key + + assert output == mock_verification_key From f213e1912c4c87ead95eb92734c959d5ea91a388 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 10:14:13 -0800 Subject: [PATCH 106/376] upgrade image --- buildspec.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/buildspec.yml b/buildspec.yml index 3d70c144d..5dbd3f2b8 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -108,6 +108,8 @@ batch: buildspec: codebuild/coverage/coverage.yml - identifier: code_coverage_mpl buildspec: codebuild/coverage/coverage_mpl.yml + env: + image: aws/codebuild/standard:7.0 - identifier: compliance buildspec: codebuild/compliance/compliance.yml From d55f2963b82270f1a3377ff524a55ae663b5675a Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 10:23:56 -0800 Subject: [PATCH 107/376] refactor tests --- test/unit/mpl/__init__.py | 12 + .../mpl/test_material_managers_mpl_cmm.py | 278 ++++++++++++++++++ .../test_material_managers_mpl_materials.py | 221 ++++++++++++++ tox.ini | 12 +- 4 files changed, 516 insertions(+), 7 deletions(-) create mode 100644 test/unit/mpl/__init__.py create mode 100644 test/unit/mpl/test_material_managers_mpl_cmm.py create mode 100644 test/unit/mpl/test_material_managers_mpl_materials.py diff --git a/test/unit/mpl/__init__.py b/test/unit/mpl/__init__.py new file mode 100644 index 000000000..53a960891 --- /dev/null +++ b/test/unit/mpl/__init__.py @@ -0,0 +1,12 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. diff --git a/test/unit/mpl/test_material_managers_mpl_cmm.py b/test/unit/mpl/test_material_managers_mpl_cmm.py new file mode 100644 index 000000000..77bf5502d --- /dev/null +++ b/test/unit/mpl/test_material_managers_mpl_cmm.py @@ -0,0 +1,278 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" + +import pytest +from mock import MagicMock, patch + + +from aws_encryption_sdk.identifiers import CommitmentPolicy +import aws_encryption_sdk.materials_managers.mpl.cmm +from aws_encryption_sdk.materials_managers.mpl.cmm import MPLCMMHandler +from aws_encryption_sdk.materials_managers.mpl.materials import ( + MPLEncryptionMaterials, + MPLDecryptionMaterials, +) + +pytestmark = [pytest.mark.unit, pytest.mark.local] + + +# Check if MPL is installed, and skip tests based on its installation status +# Ideally, this logic would be based on mocking imports and testing logic, +# but doing that introduces errors that cause other tests to fail. +try: + from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException + from aws_cryptographic_materialproviders.mpl.models import ( + AlgorithmSuiteIdESDK, + CommitmentPolicyESDK, + DecryptMaterialsInput, + DecryptionMaterials as MPL_DecryptionMaterials, + EncryptionMaterials as MPL_EncryptionMaterials, + GetEncryptionMaterialsInput, + GetEncryptionMaterialsOutput, + ) + from aws_cryptographic_materialproviders.mpl.references import ( + ICryptographicMaterialsManager + ) + HAS_MPL = True + + mock_mpl_cmm = MagicMock(__class__=ICryptographicMaterialsManager) + mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) + mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) + +except ImportError: + HAS_MPL = False + + # Ensure references to these mocks exist, even if they aren't used in a non-MPL context + mock_mpl_cmm = None + mock_mpl_encryption_materials = None + mock_mpl_decrypt_materials = None + +from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError +from aws_encryption_sdk.materials_managers import ( + EncryptionMaterialsRequest, + DecryptionMaterialsRequest, +) + + +mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) +mock_encryption_materials_handler = MagicMock(__class__=MPLEncryptionMaterials) +mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) + +@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") +def test_GIVEN_test_has_mpl_is_False_THEN_cmm_has_mpl_is_False(): + """If the MPL IS NOT installed in the runtime environment, + assert the cmm has _HAS_MPL set to False""" + + assert hasattr(aws_encryption_sdk.materials_managers.mpl.cmm, "_HAS_MPL") + assert aws_encryption_sdk.materials_managers.mpl.cmm._HAS_MPL is False + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_test_has_mpl_is_True_THEN_cmm_has_mpl_is_True(): + """If the MPL IS installed in the runtime environment, + assert the cmm has _HAS_MPL set to True""" + + assert hasattr(aws_encryption_sdk.materials_managers.mpl.cmm, "_HAS_MPL") + assert aws_encryption_sdk.materials_managers.mpl.cmm._HAS_MPL is True + + +@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_THEN_raise_ImportError(): + with pytest.raises(ImportError): + MPLCMMHandler(mpl_cmm="doesn't matter") + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_valid_mpl_cmm_THEN_return_new_MPLCMMHandler(): + mpl_cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) + + assert mpl_cmm_handler.mpl_cmm == mock_mpl_cmm + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_invalid_mpl_cmm_THEN_raise_ValueError(): + with pytest.raises(ValueError): + MPLCMMHandler(mpl_cmm="not a valid mpl_cmm") + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +@patch.object(mock_mpl_cmm, "get_encryption_materials") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_get_encryption_materials") +def test_GIVEN_valid_request_WHEN_call_get_encryption_materials_THEN_return_MPLEncryptionMaterials( + mock_native_to_mpl_get_encryption_materials, + mock_get_encryption_materials, +): + + # Mock: mpl_cmm.get_encryption_materials returns mock MPL encryption materials + mock_get_encryption_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) + mock_get_encryption_materials_output.encryption_materials = mock_mpl_encryption_materials + mock_get_encryption_materials.return_value = mock_get_encryption_materials_output + + # Mock: CMMHandler._native_to_mpl_get_encryption_materials creates a GetEncryptionMaterialsInput + mock_get_encryption_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) + mock_native_to_mpl_get_encryption_materials.return_value = mock_get_encryption_materials_input + + cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) + test = cmm_handler.get_encryption_materials(mock_encryption_materials_request) + + # Verify cmm_handler returns MPLEncryptionMaterials + assert isinstance(test, MPLEncryptionMaterials) + # Verify returned EncryptionMaterialsHandler uses the output of `get_encryption_materials` + assert test.mpl_materials == mock_mpl_encryption_materials + # Verify we actually called `get_encryption_materials` + mock_mpl_cmm.get_encryption_materials.assert_called_once_with(mock_get_encryption_materials_input) + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") +def test_GIVEN_get_encryption_materials_raises_MPL_Exception_WHEN_call_get_encryption_materials_THEN_raise_ESDK_Exception( + _ +): + with pytest.raises(AWSEncryptionSDKClientError): + with patch.object(mock_mpl_cmm, "get_encryption_materials", + side_effect=AwsCryptographicMaterialProvidersException("any")): + + cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) + cmm_handler.get_encryption_materials(mock_encryption_materials_request) + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") +def test_GIVEN_native_to_mpl_commmitment_policy_returns_valid_policy_WHEN_call_native_to_mpl_get_encryption_materials_THEN_returns_GetEncryptionMaterialsInput( + mock_mpl_commitment_policy +): + mock_commitment_policy = MagicMock(__class__=CommitmentPolicyESDK) + mock_mpl_commitment_policy.return_value = mock_commitment_policy + + output = MPLCMMHandler._native_to_mpl_get_encryption_materials(mock_encryption_materials_request) + + # verify correctness of returned value + assert isinstance(output, GetEncryptionMaterialsInput) + assert output.encryption_context == mock_encryption_materials_request.encryption_context + assert output.commitment_policy == mock_commitment_policy + assert output.max_plaintext_length == mock_encryption_materials_request.plaintext_length + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_FORBID_ENCRYPT_ALLOW_DECRYPT(): + native_commitment_policy = CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT + + output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) + + assert isinstance(output, CommitmentPolicyESDK) + assert output.value == "FORBID_ENCRYPT_ALLOW_DECRYPT" + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_ALLOW_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_REQUIRE_ENCRYPT_ALLOW_DECRYPT(): + native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT + + output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) + + assert isinstance(output, CommitmentPolicyESDK) + assert output.value == "REQUIRE_ENCRYPT_ALLOW_DECRYPT" + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_REQUIRE_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_REQUIRE_ENCRYPT_REQUIRE_DECRYPT(): + native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT + + output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) + + assert isinstance(output, CommitmentPolicyESDK) + assert output.value == "REQUIRE_ENCRYPT_REQUIRE_DECRYPT" + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_CommitmentPolicy_unrecognized_WHEN_call_native_to_mpl_commmitment_policyTHEN_raise_ValueError(): + native_commitment_policy = "not a commitment policy" + + with pytest.raises(ValueError): + MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +@patch.object(mock_mpl_cmm, "decrypt_materials") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._create_mpl_decrypt_materials_input_from_request") +def test_GIVEN_valid_request_WHEN_call_decrypt_materials_THEN_return_MPLDecryptionMaterials( + mock_native_to_mpl_decrypt_materials, + mock_get_encryption_materials, +): + + # Mock: mpl_cmm.get_decryption_materials returns mock MPL decryption materials + mock_decrypt_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) + mock_decrypt_materials_output.decryption_materials = mock_mpl_decrypt_materials + mock_get_encryption_materials.return_value = mock_decrypt_materials_output + + # Mock: CMMHandler._create_mpl_decrypt_materials_input_from_request creates a DecryptMaterialsInput + mock_decrypt_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) + mock_native_to_mpl_decrypt_materials.return_value = mock_decrypt_materials_input + + cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) + output = cmm_handler.decrypt_materials(mock_decryption_materials_request) + + # Verify cmm_handler returns MPLDecryptionMaterials + assert isinstance(output, MPLDecryptionMaterials) + # Verify returned MPLDecryptionMaterials uses the output of `decrypt_materials` + assert output.mpl_materials == mock_mpl_decrypt_materials + # Verify we actually called `decrypt_materials` + mock_mpl_cmm.decrypt_materials.assert_called_once_with(mock_decrypt_materials_input) + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._create_mpl_decrypt_materials_input_from_request") +def test_GIVEN_decrypt_materials_raises_MPL_Exception_WHEN_call_decrypt_materials_THEN_raise_ESDK_Exception( + _ +): + with pytest.raises(AWSEncryptionSDKClientError): + with patch.object(mock_mpl_cmm, "decrypt_materials", + side_effect=AwsCryptographicMaterialProvidersException("any")): + + cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) + cmm_handler.decrypt_materials(mock_decryption_materials_request) + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_WHEN_call_native_algorithm_id_to_mpl_algorithm_id_THEN_returns_valid_AlgorithmSuiteIdESDK(): + some_native_algorithm_id = 0x0000 # Not a real algorithm ID, but fits the format + + mpl_output = MPLCMMHandler._native_algorithm_id_to_mpl_algorithm_id( + some_native_algorithm_id + ) + + assert isinstance(mpl_output, AlgorithmSuiteIdESDK) + assert mpl_output.value == "0x0000" + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_algorithm_id_to_mpl_algorithm_id") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") +def test__create_mpl_decrypt_materials_input_from_request( + mock_mpl_commitment_policy, + mock_mpl_algorithm_id, +): + mock_algorithm_id = "0x1234" # Some fake algorithm ID that fits the format + mock_mpl_algorithm_id.return_value = mock_algorithm_id + mock_commitment_policy = MagicMock(__class__=CommitmentPolicyESDK) + mock_mpl_commitment_policy.return_value = mock_commitment_policy + + # mock_decryption_materials_request.algorithm = + + output = MPLCMMHandler._create_mpl_decrypt_materials_input_from_request(mock_decryption_materials_request) + + assert isinstance(output, DecryptMaterialsInput) + assert output.algorithm_suite_id == mock_algorithm_id + assert output.commitment_policy == mock_commitment_policy + assert output.encryption_context == mock_decryption_materials_request.encryption_context + + assert len(output.encrypted_data_keys) == len(mock_decryption_materials_request.encrypted_data_keys) + for i in range(len(output.encrypted_data_keys)): + # Assume input[i] == output[i], seems to work + output_edk = output.encrypted_data_keys[i] + input_edk = mock_decryption_materials_request[i] + assert output_edk.key_provider_id == input_edk.key_provider.provider_id + assert output_edk.key_provider_info == input_edk.key_provider.key_info + assert output_edk.ciphertext == input_edk.encrypted_data_key diff --git a/test/unit/mpl/test_material_managers_mpl_materials.py b/test/unit/mpl/test_material_managers_mpl_materials.py new file mode 100644 index 000000000..250efeb7e --- /dev/null +++ b/test/unit/mpl/test_material_managers_mpl_materials.py @@ -0,0 +1,221 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" + +import pytest +from mock import MagicMock, patch, PropertyMock +from typing import Dict, List + +from aws_encryption_sdk.identifiers import CommitmentPolicy +import aws_encryption_sdk.materials_managers.mpl.materials +from aws_encryption_sdk.materials_managers.mpl.materials import ( + MPLEncryptionMaterials, + MPLDecryptionMaterials, +) +from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite + +pytestmark = [pytest.mark.unit, pytest.mark.local] + + +# Check if MPL is installed, and skip tests based on its installation status +# Ideally, this logic would be based on mocking imports and testing logic, +# but doing that introduces errors that cause other tests to fail. +try: + from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException + from aws_cryptographic_materialproviders.mpl.models import ( + AlgorithmSuiteIdESDK, + CommitmentPolicyESDK, + DecryptMaterialsInput, + DecryptionMaterials as MPL_DecryptionMaterials, + EncryptedDataKey as MPL_EncryptedDataKey, + EncryptionMaterials as MPL_EncryptionMaterials, + GetEncryptionMaterialsInput, + GetEncryptionMaterialsOutput, + ) + from aws_cryptographic_materialproviders.mpl.references import ( + ICryptographicMaterialsManager + ) + HAS_MPL = True + + mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) + mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) + +except ImportError: + HAS_MPL = False + + # Ensure references to these mocks exist, even if they aren't used in a non-MPL context + mock_mpl_cmm = None + mock_mpl_encryption_materials = None + mock_mpl_decrypt_materials = None + +from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError +from aws_encryption_sdk.materials_managers import ( + EncryptionMaterialsRequest, + DecryptionMaterialsRequest, +) + + +mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) +mock_encryption_materials_handler = MagicMock(__class__=MPLEncryptionMaterials) +mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) + +@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") +def test_GIVEN_test_has_mpl_is_False_THEN_cmm_has_mpl_is_False(): + """If the MPL IS NOT installed in the runtime environment, + assert the cmm has _HAS_MPL set to False""" + + assert hasattr(aws_encryption_sdk.materials_managers.mpl.materials, "_HAS_MPL") + assert aws_encryption_sdk.materials_managers.mpl.materials._HAS_MPL is False + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_test_has_mpl_is_True_THEN_cmm_has_mpl_is_True(): + """If the MPL IS installed in the runtime environment, + assert the cmm has _HAS_MPL set to True""" + + assert hasattr(aws_encryption_sdk.materials_managers.mpl.materials, "_HAS_MPL") + assert aws_encryption_sdk.materials_managers.mpl.materials._HAS_MPL is True + + +@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_THEN_raise_ImportError(): + with pytest.raises(ImportError): + MPLEncryptionMaterials(mpl_materials="doesn't matter") + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_valid_mpl_cmm_THEN_return_new_MPLCMMHandler(): + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + + assert mpl_encryption_materials.mpl_materials == mock_mpl_encryption_materials + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_invalid_mpl_cmm_THEN_raise_ValueError(): + with pytest.raises(ValueError): + MPLEncryptionMaterials(mpl_materials="not a valid mpl_materials") + +def test_mpl_to_native(): + some_mpl_algorithm_id = "0x1234" # Not a real algorithm ID, but fits the format + + native_output = aws_encryption_sdk.materials_managers.mpl.materials._mpl_algorithm_id_to_native_algorithm_id( + some_mpl_algorithm_id + ) + + assert native_output == 0x1234 + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +@patch("aws_encryption_sdk.materials_managers.mpl.materials._mpl_algorithm_id_to_native_algorithm_id") +@patch("aws_encryption_sdk.materials_managers.mpl.materials.AlgorithmSuite.get_by_id") +def test_GIVEN_valid_mpl_algorithm_id_WHEN_get_algorithm_THEN_valid_native_algorithm_id( + mock_algorithm, + mock_native_algorithm_id, +): + # Mock valid conversion from MPL to native algorithm ID + mock_native_algorithm_id.return_value = 0x1234 + + # Mock valid lookup in native AlgorithmSuite lookup + mock_algorithm.return_value = MagicMock(__class__=AlgorithmSuite) + + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + output = mpl_encryption_materials.algorithm + assert output == mock_algorithm() # property calls automatically, we need to call the mock + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GecTHEN_valid_native_algorithm_id(): + mock_encryption_context = MagicMock(__class__=Dict[str, str]) + mock_mpl_encryption_materials.encryption_context = mock_encryption_context + + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + output = mpl_encryption_materials.encryption_context + + assert output == mock_encryption_context + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GecTHEN_valid_nativefadsf_algorithm_id(): + mock_edk = MagicMock(__class__=MPL_EncryptedDataKey) + mock_mpl_key_provider_id = MagicMock(__class__=str) + mock_edk.key_provider_id = mock_mpl_key_provider_id + mock_mpl_key_provider_info = MagicMock(__class__=bytes) + mock_edk.key_provider_info = mock_mpl_key_provider_info + mock_mpl_ciphertext = MagicMock(__class__=bytes) + mock_edk.ciphertext = mock_mpl_ciphertext + + mock_edks = [ mock_edk ] + mock_mpl_encryption_materials.encrypted_data_keys = mock_edks + + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + output = mpl_encryption_materials.encrypted_data_keys + output_as_list = list(output) + + assert len(output_as_list) == len(mock_edks) + for i in range(len(output_as_list)): + # assume output[i] corresponds to input[i] + native_edk = output_as_list[i] + mpl_edk = mock_edks[i] + + assert native_edk.encrypted_data_key == mpl_edk.ciphertext + assert native_edk.key_provider.provider_id == mpl_edk.key_provider_id + assert native_edk.key_provider.key_info == mpl_edk.key_provider_info + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GecTHEN_valid_nativefadsffadsfa_algorithm_id(): + mock_data_key = MagicMock(__class__=bytes) + mock_mpl_encryption_materials.plaintext_data_key = mock_data_key + + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + output = mpl_encryption_materials.data_encryption_key + + assert output.key_provider.provider_id == "" + assert output.key_provider.key_info == b"" + assert output.data_key == mock_data_key + assert output.encrypted_data_key == b"" + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GecTHEN_valid_nativefasdfasdffadsf_algorithm_id(): + mock_signing_key = MagicMock(__class__=bytes) + mock_mpl_encryption_materials.signing_key = mock_signing_key + + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + output = mpl_encryption_materials.signing_key + + assert output == mock_signing_key + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GecTHEN_valid_nativeffasdfasdadsffadsfa_algorithm_id(): + mock_data_key = MagicMock(__class__=bytes) + mock_mpl_decrypt_materials.plaintext_data_key = mock_data_key + + mpl_decryption_materials = MPLDecryptionMaterials(mpl_materials=mock_mpl_decrypt_materials) + output = mpl_decryption_materials.data_key + + assert output.key_provider.provider_id == "" + assert output.key_provider.key_info == b"" + assert output.data_key == mock_data_key + assert output.encrypted_data_key == b"" + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GecTHEN_validadsfasdf_nativefasdfasdffadsf_algorithm_id(): + mock_verification_key = MagicMock(__class__=bytes) + mock_mpl_decrypt_materials.verification_key = mock_verification_key + + mpl_decryption_materials = MPLDecryptionMaterials(mpl_materials=mock_mpl_decrypt_materials) + output = mpl_decryption_materials.verification_key + + assert output == mock_verification_key diff --git a/tox.ini b/tox.ini index 72e8ec9fa..346e4fae0 100644 --- a/tox.ini +++ b/tox.ini @@ -84,23 +84,21 @@ deps = commands = local: {[testenv:base-command]commands} test/ -m local --ignore test/unit/mpl/ # MPL unit tests require the MPL to be installed - mpllocal: {[testenv:base-command]commands} test/unit/mpl/ -m local - integ: {[testenv:base-command]commands} test/ -m integ --ignore test/unit/mpl/ - # No MPL-specific integ tests - accept: {[testenv:base-command]commands} test/ -m accept --ignore test/unit/mpl/ - # No MPL-specific accept tests + mpllocal: {[testenv:base-command]commands} test/ -m local + integ: {[testenv:base-command]commands} test/ -m integ + accept: {[testenv:base-command]commands} test/ -m accept examples: {[testenv:base-command]commands} examples/test/ -m examples --ignore examples/test/keyrings/ # MPL keyring examples require a special IAM role; run these separately under a separate set of permissions mplexamples: {[testenv:base-command]commands} examples/test/keyrings -m examples all: {[testenv:base-command]commands} test/ examples/test/ --ignore test/unit/mpl/ --ignore examples/test/keyrings/ - mplall: {[testenv:base-command]commands} test/unit/mpl/ examples/test/keyrings/ + mplall: {[testenv:base-command]commands} test/ examples/test/ manual: {[testenv:base-command]commands} # Run code coverage on the unit tests [testenv:coverage] commands = {[testenv:base-command]commands} --cov aws_encryption_sdk test/ -m local --ignore test/unit/mpl/ [testenv:mplcoverage-mpl] -commands = {[testenv:base-command]commands} --cov aws_encryption_sdk test/unit/mpl/ -m local +commands = {[testenv:base-command]commands} --cov aws_encryption_sdk test/ -m local # Verify that local tests work without environment variables present [testenv:nocmk] From 5ec46687b47ab907cc4a53ca9cc18a4f677c65e6 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 10:33:22 -0800 Subject: [PATCH 108/376] refactor tests --- .../materials_managers/mpl/cmm.py | 32 +- .../materials_managers/mpl/materials.py | 22 +- test/unit/mpl/__init__.py | 12 - .../mpl/test_material_managers_mpl_cmm.py | 278 ------------------ .../test_material_managers_mpl_materials.py | 221 -------------- test/unit/test_material_managers_mpl_cmm.py | 278 ------------------ .../test_material_managers_mpl_materials.py | 221 -------------- tox.ini | 6 +- 8 files changed, 20 insertions(+), 1050 deletions(-) delete mode 100644 test/unit/mpl/__init__.py delete mode 100644 test/unit/mpl/test_material_managers_mpl_cmm.py delete mode 100644 test/unit/mpl/test_material_managers_mpl_materials.py delete mode 100644 test/unit/test_material_managers_mpl_cmm.py delete mode 100644 test/unit/test_material_managers_mpl_materials.py diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index cd789b994..1e3e3fb34 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -1,23 +1,16 @@ """Retrieves encryption/decryption materials from the MPL.""" -# These dependencies are only loaded if you install the MPL. -try: - # pylint seems to struggle with this conditional import - # pylint: disable=unused-import - from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException - from aws_cryptographic_materialproviders.mpl.models import ( - AlgorithmSuiteIdESDK, - CommitmentPolicyESDK, - DecryptMaterialsInput, - DecryptMaterialsOutput, - EncryptedDataKey as MPL_EncryptedDataKey, - GetEncryptionMaterialsInput, - GetEncryptionMaterialsOutput, - ) - from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager - _HAS_MPL = True -except ImportError: - _HAS_MPL = False +from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException +from aws_cryptographic_materialproviders.mpl.models import ( + AlgorithmSuiteIdESDK, + CommitmentPolicyESDK, + DecryptMaterialsInput, + DecryptMaterialsOutput, + EncryptedDataKey as MPL_EncryptedDataKey, + GetEncryptionMaterialsInput, + GetEncryptionMaterialsOutput, +) +from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager from typing import List @@ -46,9 +39,6 @@ def __init__( Create MPLCMMHandler. :param mpl_cmm: Underlying MPL cryptographic materials manager """ - if not _HAS_MPL: - raise ImportError("You MUST install the aws-cryptographic-material-providers " - f"library to create an instance of {MPLCMMHandler}") if isinstance(mpl_cmm, ICryptographicMaterialsManager): self.mpl_cmm = mpl_cmm else: diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index bd4b5f729..1ea2a199d 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -1,14 +1,10 @@ """Provides encryption/decryption materials from an underlying materials provider.""" -# These dependencies are only loaded if you install the MPL. -try: - from aws_cryptographic_materialproviders.mpl.models import ( - DecryptionMaterials as MPL_DecryptionMaterials, - EncryptedDataKey as MPL_EncryptedDataKey, - EncryptionMaterials as MPL_EncryptionMaterials, - ) - _HAS_MPL = True -except ImportError: - _HAS_MPL = False + +from aws_cryptographic_materialproviders.mpl.models import ( + DecryptionMaterials as MPL_DecryptionMaterials, + EncryptedDataKey as MPL_EncryptedDataKey, + EncryptionMaterials as MPL_EncryptionMaterials, +) from typing import Dict, List, Set @@ -42,9 +38,6 @@ def __init__( Create MPLEncryptionMaterials. :param materials: Underlying encryption materials """ - if not _HAS_MPL: - raise ImportError("You MUST install the aws-cryptographic-material-providers " - f"library to create an instance of {MPLEncryptionMaterials}") if isinstance(mpl_materials, MPL_EncryptionMaterials): self.mpl_materials = mpl_materials else: @@ -115,9 +108,6 @@ def __init__( Create MPLDecryptionMaterials. :param materials: Underlying decryption materials """ - if not _HAS_MPL: - raise ImportError("You MUST install the aws-cryptographic-material-providers " - f"library to create an instance of {MPLDecryptionMaterials}") if isinstance(mpl_materials, MPL_DecryptionMaterials): self.mpl_materials = mpl_materials else: diff --git a/test/unit/mpl/__init__.py b/test/unit/mpl/__init__.py deleted file mode 100644 index 53a960891..000000000 --- a/test/unit/mpl/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. diff --git a/test/unit/mpl/test_material_managers_mpl_cmm.py b/test/unit/mpl/test_material_managers_mpl_cmm.py deleted file mode 100644 index 77bf5502d..000000000 --- a/test/unit/mpl/test_material_managers_mpl_cmm.py +++ /dev/null @@ -1,278 +0,0 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. -"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" - -import pytest -from mock import MagicMock, patch - - -from aws_encryption_sdk.identifiers import CommitmentPolicy -import aws_encryption_sdk.materials_managers.mpl.cmm -from aws_encryption_sdk.materials_managers.mpl.cmm import MPLCMMHandler -from aws_encryption_sdk.materials_managers.mpl.materials import ( - MPLEncryptionMaterials, - MPLDecryptionMaterials, -) - -pytestmark = [pytest.mark.unit, pytest.mark.local] - - -# Check if MPL is installed, and skip tests based on its installation status -# Ideally, this logic would be based on mocking imports and testing logic, -# but doing that introduces errors that cause other tests to fail. -try: - from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException - from aws_cryptographic_materialproviders.mpl.models import ( - AlgorithmSuiteIdESDK, - CommitmentPolicyESDK, - DecryptMaterialsInput, - DecryptionMaterials as MPL_DecryptionMaterials, - EncryptionMaterials as MPL_EncryptionMaterials, - GetEncryptionMaterialsInput, - GetEncryptionMaterialsOutput, - ) - from aws_cryptographic_materialproviders.mpl.references import ( - ICryptographicMaterialsManager - ) - HAS_MPL = True - - mock_mpl_cmm = MagicMock(__class__=ICryptographicMaterialsManager) - mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) - mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) - -except ImportError: - HAS_MPL = False - - # Ensure references to these mocks exist, even if they aren't used in a non-MPL context - mock_mpl_cmm = None - mock_mpl_encryption_materials = None - mock_mpl_decrypt_materials = None - -from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError -from aws_encryption_sdk.materials_managers import ( - EncryptionMaterialsRequest, - DecryptionMaterialsRequest, -) - - -mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) -mock_encryption_materials_handler = MagicMock(__class__=MPLEncryptionMaterials) -mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) - -@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") -def test_GIVEN_test_has_mpl_is_False_THEN_cmm_has_mpl_is_False(): - """If the MPL IS NOT installed in the runtime environment, - assert the cmm has _HAS_MPL set to False""" - - assert hasattr(aws_encryption_sdk.materials_managers.mpl.cmm, "_HAS_MPL") - assert aws_encryption_sdk.materials_managers.mpl.cmm._HAS_MPL is False - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_test_has_mpl_is_True_THEN_cmm_has_mpl_is_True(): - """If the MPL IS installed in the runtime environment, - assert the cmm has _HAS_MPL set to True""" - - assert hasattr(aws_encryption_sdk.materials_managers.mpl.cmm, "_HAS_MPL") - assert aws_encryption_sdk.materials_managers.mpl.cmm._HAS_MPL is True - - -@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_THEN_raise_ImportError(): - with pytest.raises(ImportError): - MPLCMMHandler(mpl_cmm="doesn't matter") - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_valid_mpl_cmm_THEN_return_new_MPLCMMHandler(): - mpl_cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) - - assert mpl_cmm_handler.mpl_cmm == mock_mpl_cmm - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_invalid_mpl_cmm_THEN_raise_ValueError(): - with pytest.raises(ValueError): - MPLCMMHandler(mpl_cmm="not a valid mpl_cmm") - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -@patch.object(mock_mpl_cmm, "get_encryption_materials") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_get_encryption_materials") -def test_GIVEN_valid_request_WHEN_call_get_encryption_materials_THEN_return_MPLEncryptionMaterials( - mock_native_to_mpl_get_encryption_materials, - mock_get_encryption_materials, -): - - # Mock: mpl_cmm.get_encryption_materials returns mock MPL encryption materials - mock_get_encryption_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) - mock_get_encryption_materials_output.encryption_materials = mock_mpl_encryption_materials - mock_get_encryption_materials.return_value = mock_get_encryption_materials_output - - # Mock: CMMHandler._native_to_mpl_get_encryption_materials creates a GetEncryptionMaterialsInput - mock_get_encryption_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) - mock_native_to_mpl_get_encryption_materials.return_value = mock_get_encryption_materials_input - - cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) - test = cmm_handler.get_encryption_materials(mock_encryption_materials_request) - - # Verify cmm_handler returns MPLEncryptionMaterials - assert isinstance(test, MPLEncryptionMaterials) - # Verify returned EncryptionMaterialsHandler uses the output of `get_encryption_materials` - assert test.mpl_materials == mock_mpl_encryption_materials - # Verify we actually called `get_encryption_materials` - mock_mpl_cmm.get_encryption_materials.assert_called_once_with(mock_get_encryption_materials_input) - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") -def test_GIVEN_get_encryption_materials_raises_MPL_Exception_WHEN_call_get_encryption_materials_THEN_raise_ESDK_Exception( - _ -): - with pytest.raises(AWSEncryptionSDKClientError): - with patch.object(mock_mpl_cmm, "get_encryption_materials", - side_effect=AwsCryptographicMaterialProvidersException("any")): - - cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) - cmm_handler.get_encryption_materials(mock_encryption_materials_request) - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") -def test_GIVEN_native_to_mpl_commmitment_policy_returns_valid_policy_WHEN_call_native_to_mpl_get_encryption_materials_THEN_returns_GetEncryptionMaterialsInput( - mock_mpl_commitment_policy -): - mock_commitment_policy = MagicMock(__class__=CommitmentPolicyESDK) - mock_mpl_commitment_policy.return_value = mock_commitment_policy - - output = MPLCMMHandler._native_to_mpl_get_encryption_materials(mock_encryption_materials_request) - - # verify correctness of returned value - assert isinstance(output, GetEncryptionMaterialsInput) - assert output.encryption_context == mock_encryption_materials_request.encryption_context - assert output.commitment_policy == mock_commitment_policy - assert output.max_plaintext_length == mock_encryption_materials_request.plaintext_length - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_FORBID_ENCRYPT_ALLOW_DECRYPT(): - native_commitment_policy = CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT - - output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) - - assert isinstance(output, CommitmentPolicyESDK) - assert output.value == "FORBID_ENCRYPT_ALLOW_DECRYPT" - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_ALLOW_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_REQUIRE_ENCRYPT_ALLOW_DECRYPT(): - native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT - - output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) - - assert isinstance(output, CommitmentPolicyESDK) - assert output.value == "REQUIRE_ENCRYPT_ALLOW_DECRYPT" - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_REQUIRE_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_REQUIRE_ENCRYPT_REQUIRE_DECRYPT(): - native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT - - output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) - - assert isinstance(output, CommitmentPolicyESDK) - assert output.value == "REQUIRE_ENCRYPT_REQUIRE_DECRYPT" - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_CommitmentPolicy_unrecognized_WHEN_call_native_to_mpl_commmitment_policyTHEN_raise_ValueError(): - native_commitment_policy = "not a commitment policy" - - with pytest.raises(ValueError): - MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -@patch.object(mock_mpl_cmm, "decrypt_materials") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._create_mpl_decrypt_materials_input_from_request") -def test_GIVEN_valid_request_WHEN_call_decrypt_materials_THEN_return_MPLDecryptionMaterials( - mock_native_to_mpl_decrypt_materials, - mock_get_encryption_materials, -): - - # Mock: mpl_cmm.get_decryption_materials returns mock MPL decryption materials - mock_decrypt_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) - mock_decrypt_materials_output.decryption_materials = mock_mpl_decrypt_materials - mock_get_encryption_materials.return_value = mock_decrypt_materials_output - - # Mock: CMMHandler._create_mpl_decrypt_materials_input_from_request creates a DecryptMaterialsInput - mock_decrypt_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) - mock_native_to_mpl_decrypt_materials.return_value = mock_decrypt_materials_input - - cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) - output = cmm_handler.decrypt_materials(mock_decryption_materials_request) - - # Verify cmm_handler returns MPLDecryptionMaterials - assert isinstance(output, MPLDecryptionMaterials) - # Verify returned MPLDecryptionMaterials uses the output of `decrypt_materials` - assert output.mpl_materials == mock_mpl_decrypt_materials - # Verify we actually called `decrypt_materials` - mock_mpl_cmm.decrypt_materials.assert_called_once_with(mock_decrypt_materials_input) - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._create_mpl_decrypt_materials_input_from_request") -def test_GIVEN_decrypt_materials_raises_MPL_Exception_WHEN_call_decrypt_materials_THEN_raise_ESDK_Exception( - _ -): - with pytest.raises(AWSEncryptionSDKClientError): - with patch.object(mock_mpl_cmm, "decrypt_materials", - side_effect=AwsCryptographicMaterialProvidersException("any")): - - cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) - cmm_handler.decrypt_materials(mock_decryption_materials_request) - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_WHEN_call_native_algorithm_id_to_mpl_algorithm_id_THEN_returns_valid_AlgorithmSuiteIdESDK(): - some_native_algorithm_id = 0x0000 # Not a real algorithm ID, but fits the format - - mpl_output = MPLCMMHandler._native_algorithm_id_to_mpl_algorithm_id( - some_native_algorithm_id - ) - - assert isinstance(mpl_output, AlgorithmSuiteIdESDK) - assert mpl_output.value == "0x0000" - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_algorithm_id_to_mpl_algorithm_id") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") -def test__create_mpl_decrypt_materials_input_from_request( - mock_mpl_commitment_policy, - mock_mpl_algorithm_id, -): - mock_algorithm_id = "0x1234" # Some fake algorithm ID that fits the format - mock_mpl_algorithm_id.return_value = mock_algorithm_id - mock_commitment_policy = MagicMock(__class__=CommitmentPolicyESDK) - mock_mpl_commitment_policy.return_value = mock_commitment_policy - - # mock_decryption_materials_request.algorithm = - - output = MPLCMMHandler._create_mpl_decrypt_materials_input_from_request(mock_decryption_materials_request) - - assert isinstance(output, DecryptMaterialsInput) - assert output.algorithm_suite_id == mock_algorithm_id - assert output.commitment_policy == mock_commitment_policy - assert output.encryption_context == mock_decryption_materials_request.encryption_context - - assert len(output.encrypted_data_keys) == len(mock_decryption_materials_request.encrypted_data_keys) - for i in range(len(output.encrypted_data_keys)): - # Assume input[i] == output[i], seems to work - output_edk = output.encrypted_data_keys[i] - input_edk = mock_decryption_materials_request[i] - assert output_edk.key_provider_id == input_edk.key_provider.provider_id - assert output_edk.key_provider_info == input_edk.key_provider.key_info - assert output_edk.ciphertext == input_edk.encrypted_data_key diff --git a/test/unit/mpl/test_material_managers_mpl_materials.py b/test/unit/mpl/test_material_managers_mpl_materials.py deleted file mode 100644 index 250efeb7e..000000000 --- a/test/unit/mpl/test_material_managers_mpl_materials.py +++ /dev/null @@ -1,221 +0,0 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. -"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" - -import pytest -from mock import MagicMock, patch, PropertyMock -from typing import Dict, List - -from aws_encryption_sdk.identifiers import CommitmentPolicy -import aws_encryption_sdk.materials_managers.mpl.materials -from aws_encryption_sdk.materials_managers.mpl.materials import ( - MPLEncryptionMaterials, - MPLDecryptionMaterials, -) -from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite - -pytestmark = [pytest.mark.unit, pytest.mark.local] - - -# Check if MPL is installed, and skip tests based on its installation status -# Ideally, this logic would be based on mocking imports and testing logic, -# but doing that introduces errors that cause other tests to fail. -try: - from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException - from aws_cryptographic_materialproviders.mpl.models import ( - AlgorithmSuiteIdESDK, - CommitmentPolicyESDK, - DecryptMaterialsInput, - DecryptionMaterials as MPL_DecryptionMaterials, - EncryptedDataKey as MPL_EncryptedDataKey, - EncryptionMaterials as MPL_EncryptionMaterials, - GetEncryptionMaterialsInput, - GetEncryptionMaterialsOutput, - ) - from aws_cryptographic_materialproviders.mpl.references import ( - ICryptographicMaterialsManager - ) - HAS_MPL = True - - mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) - mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) - -except ImportError: - HAS_MPL = False - - # Ensure references to these mocks exist, even if they aren't used in a non-MPL context - mock_mpl_cmm = None - mock_mpl_encryption_materials = None - mock_mpl_decrypt_materials = None - -from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError -from aws_encryption_sdk.materials_managers import ( - EncryptionMaterialsRequest, - DecryptionMaterialsRequest, -) - - -mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) -mock_encryption_materials_handler = MagicMock(__class__=MPLEncryptionMaterials) -mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) - -@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") -def test_GIVEN_test_has_mpl_is_False_THEN_cmm_has_mpl_is_False(): - """If the MPL IS NOT installed in the runtime environment, - assert the cmm has _HAS_MPL set to False""" - - assert hasattr(aws_encryption_sdk.materials_managers.mpl.materials, "_HAS_MPL") - assert aws_encryption_sdk.materials_managers.mpl.materials._HAS_MPL is False - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_test_has_mpl_is_True_THEN_cmm_has_mpl_is_True(): - """If the MPL IS installed in the runtime environment, - assert the cmm has _HAS_MPL set to True""" - - assert hasattr(aws_encryption_sdk.materials_managers.mpl.materials, "_HAS_MPL") - assert aws_encryption_sdk.materials_managers.mpl.materials._HAS_MPL is True - - -@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_THEN_raise_ImportError(): - with pytest.raises(ImportError): - MPLEncryptionMaterials(mpl_materials="doesn't matter") - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_valid_mpl_cmm_THEN_return_new_MPLCMMHandler(): - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) - - assert mpl_encryption_materials.mpl_materials == mock_mpl_encryption_materials - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_invalid_mpl_cmm_THEN_raise_ValueError(): - with pytest.raises(ValueError): - MPLEncryptionMaterials(mpl_materials="not a valid mpl_materials") - -def test_mpl_to_native(): - some_mpl_algorithm_id = "0x1234" # Not a real algorithm ID, but fits the format - - native_output = aws_encryption_sdk.materials_managers.mpl.materials._mpl_algorithm_id_to_native_algorithm_id( - some_mpl_algorithm_id - ) - - assert native_output == 0x1234 - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -@patch("aws_encryption_sdk.materials_managers.mpl.materials._mpl_algorithm_id_to_native_algorithm_id") -@patch("aws_encryption_sdk.materials_managers.mpl.materials.AlgorithmSuite.get_by_id") -def test_GIVEN_valid_mpl_algorithm_id_WHEN_get_algorithm_THEN_valid_native_algorithm_id( - mock_algorithm, - mock_native_algorithm_id, -): - # Mock valid conversion from MPL to native algorithm ID - mock_native_algorithm_id.return_value = 0x1234 - - # Mock valid lookup in native AlgorithmSuite lookup - mock_algorithm.return_value = MagicMock(__class__=AlgorithmSuite) - - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) - output = mpl_encryption_materials.algorithm - assert output == mock_algorithm() # property calls automatically, we need to call the mock - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GecTHEN_valid_native_algorithm_id(): - mock_encryption_context = MagicMock(__class__=Dict[str, str]) - mock_mpl_encryption_materials.encryption_context = mock_encryption_context - - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) - output = mpl_encryption_materials.encryption_context - - assert output == mock_encryption_context - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GecTHEN_valid_nativefadsf_algorithm_id(): - mock_edk = MagicMock(__class__=MPL_EncryptedDataKey) - mock_mpl_key_provider_id = MagicMock(__class__=str) - mock_edk.key_provider_id = mock_mpl_key_provider_id - mock_mpl_key_provider_info = MagicMock(__class__=bytes) - mock_edk.key_provider_info = mock_mpl_key_provider_info - mock_mpl_ciphertext = MagicMock(__class__=bytes) - mock_edk.ciphertext = mock_mpl_ciphertext - - mock_edks = [ mock_edk ] - mock_mpl_encryption_materials.encrypted_data_keys = mock_edks - - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) - output = mpl_encryption_materials.encrypted_data_keys - output_as_list = list(output) - - assert len(output_as_list) == len(mock_edks) - for i in range(len(output_as_list)): - # assume output[i] corresponds to input[i] - native_edk = output_as_list[i] - mpl_edk = mock_edks[i] - - assert native_edk.encrypted_data_key == mpl_edk.ciphertext - assert native_edk.key_provider.provider_id == mpl_edk.key_provider_id - assert native_edk.key_provider.key_info == mpl_edk.key_provider_info - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GecTHEN_valid_nativefadsffadsfa_algorithm_id(): - mock_data_key = MagicMock(__class__=bytes) - mock_mpl_encryption_materials.plaintext_data_key = mock_data_key - - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) - output = mpl_encryption_materials.data_encryption_key - - assert output.key_provider.provider_id == "" - assert output.key_provider.key_info == b"" - assert output.data_key == mock_data_key - assert output.encrypted_data_key == b"" - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GecTHEN_valid_nativefasdfasdffadsf_algorithm_id(): - mock_signing_key = MagicMock(__class__=bytes) - mock_mpl_encryption_materials.signing_key = mock_signing_key - - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) - output = mpl_encryption_materials.signing_key - - assert output == mock_signing_key - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GecTHEN_valid_nativeffasdfasdadsffadsfa_algorithm_id(): - mock_data_key = MagicMock(__class__=bytes) - mock_mpl_decrypt_materials.plaintext_data_key = mock_data_key - - mpl_decryption_materials = MPLDecryptionMaterials(mpl_materials=mock_mpl_decrypt_materials) - output = mpl_decryption_materials.data_key - - assert output.key_provider.provider_id == "" - assert output.key_provider.key_info == b"" - assert output.data_key == mock_data_key - assert output.encrypted_data_key == b"" - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GecTHEN_validadsfasdf_nativefasdfasdffadsf_algorithm_id(): - mock_verification_key = MagicMock(__class__=bytes) - mock_mpl_decrypt_materials.verification_key = mock_verification_key - - mpl_decryption_materials = MPLDecryptionMaterials(mpl_materials=mock_mpl_decrypt_materials) - output = mpl_decryption_materials.verification_key - - assert output == mock_verification_key diff --git a/test/unit/test_material_managers_mpl_cmm.py b/test/unit/test_material_managers_mpl_cmm.py deleted file mode 100644 index 77bf5502d..000000000 --- a/test/unit/test_material_managers_mpl_cmm.py +++ /dev/null @@ -1,278 +0,0 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. -"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" - -import pytest -from mock import MagicMock, patch - - -from aws_encryption_sdk.identifiers import CommitmentPolicy -import aws_encryption_sdk.materials_managers.mpl.cmm -from aws_encryption_sdk.materials_managers.mpl.cmm import MPLCMMHandler -from aws_encryption_sdk.materials_managers.mpl.materials import ( - MPLEncryptionMaterials, - MPLDecryptionMaterials, -) - -pytestmark = [pytest.mark.unit, pytest.mark.local] - - -# Check if MPL is installed, and skip tests based on its installation status -# Ideally, this logic would be based on mocking imports and testing logic, -# but doing that introduces errors that cause other tests to fail. -try: - from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException - from aws_cryptographic_materialproviders.mpl.models import ( - AlgorithmSuiteIdESDK, - CommitmentPolicyESDK, - DecryptMaterialsInput, - DecryptionMaterials as MPL_DecryptionMaterials, - EncryptionMaterials as MPL_EncryptionMaterials, - GetEncryptionMaterialsInput, - GetEncryptionMaterialsOutput, - ) - from aws_cryptographic_materialproviders.mpl.references import ( - ICryptographicMaterialsManager - ) - HAS_MPL = True - - mock_mpl_cmm = MagicMock(__class__=ICryptographicMaterialsManager) - mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) - mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) - -except ImportError: - HAS_MPL = False - - # Ensure references to these mocks exist, even if they aren't used in a non-MPL context - mock_mpl_cmm = None - mock_mpl_encryption_materials = None - mock_mpl_decrypt_materials = None - -from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError -from aws_encryption_sdk.materials_managers import ( - EncryptionMaterialsRequest, - DecryptionMaterialsRequest, -) - - -mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) -mock_encryption_materials_handler = MagicMock(__class__=MPLEncryptionMaterials) -mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) - -@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") -def test_GIVEN_test_has_mpl_is_False_THEN_cmm_has_mpl_is_False(): - """If the MPL IS NOT installed in the runtime environment, - assert the cmm has _HAS_MPL set to False""" - - assert hasattr(aws_encryption_sdk.materials_managers.mpl.cmm, "_HAS_MPL") - assert aws_encryption_sdk.materials_managers.mpl.cmm._HAS_MPL is False - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_test_has_mpl_is_True_THEN_cmm_has_mpl_is_True(): - """If the MPL IS installed in the runtime environment, - assert the cmm has _HAS_MPL set to True""" - - assert hasattr(aws_encryption_sdk.materials_managers.mpl.cmm, "_HAS_MPL") - assert aws_encryption_sdk.materials_managers.mpl.cmm._HAS_MPL is True - - -@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_THEN_raise_ImportError(): - with pytest.raises(ImportError): - MPLCMMHandler(mpl_cmm="doesn't matter") - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_valid_mpl_cmm_THEN_return_new_MPLCMMHandler(): - mpl_cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) - - assert mpl_cmm_handler.mpl_cmm == mock_mpl_cmm - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_invalid_mpl_cmm_THEN_raise_ValueError(): - with pytest.raises(ValueError): - MPLCMMHandler(mpl_cmm="not a valid mpl_cmm") - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -@patch.object(mock_mpl_cmm, "get_encryption_materials") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_get_encryption_materials") -def test_GIVEN_valid_request_WHEN_call_get_encryption_materials_THEN_return_MPLEncryptionMaterials( - mock_native_to_mpl_get_encryption_materials, - mock_get_encryption_materials, -): - - # Mock: mpl_cmm.get_encryption_materials returns mock MPL encryption materials - mock_get_encryption_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) - mock_get_encryption_materials_output.encryption_materials = mock_mpl_encryption_materials - mock_get_encryption_materials.return_value = mock_get_encryption_materials_output - - # Mock: CMMHandler._native_to_mpl_get_encryption_materials creates a GetEncryptionMaterialsInput - mock_get_encryption_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) - mock_native_to_mpl_get_encryption_materials.return_value = mock_get_encryption_materials_input - - cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) - test = cmm_handler.get_encryption_materials(mock_encryption_materials_request) - - # Verify cmm_handler returns MPLEncryptionMaterials - assert isinstance(test, MPLEncryptionMaterials) - # Verify returned EncryptionMaterialsHandler uses the output of `get_encryption_materials` - assert test.mpl_materials == mock_mpl_encryption_materials - # Verify we actually called `get_encryption_materials` - mock_mpl_cmm.get_encryption_materials.assert_called_once_with(mock_get_encryption_materials_input) - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") -def test_GIVEN_get_encryption_materials_raises_MPL_Exception_WHEN_call_get_encryption_materials_THEN_raise_ESDK_Exception( - _ -): - with pytest.raises(AWSEncryptionSDKClientError): - with patch.object(mock_mpl_cmm, "get_encryption_materials", - side_effect=AwsCryptographicMaterialProvidersException("any")): - - cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) - cmm_handler.get_encryption_materials(mock_encryption_materials_request) - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") -def test_GIVEN_native_to_mpl_commmitment_policy_returns_valid_policy_WHEN_call_native_to_mpl_get_encryption_materials_THEN_returns_GetEncryptionMaterialsInput( - mock_mpl_commitment_policy -): - mock_commitment_policy = MagicMock(__class__=CommitmentPolicyESDK) - mock_mpl_commitment_policy.return_value = mock_commitment_policy - - output = MPLCMMHandler._native_to_mpl_get_encryption_materials(mock_encryption_materials_request) - - # verify correctness of returned value - assert isinstance(output, GetEncryptionMaterialsInput) - assert output.encryption_context == mock_encryption_materials_request.encryption_context - assert output.commitment_policy == mock_commitment_policy - assert output.max_plaintext_length == mock_encryption_materials_request.plaintext_length - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_FORBID_ENCRYPT_ALLOW_DECRYPT(): - native_commitment_policy = CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT - - output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) - - assert isinstance(output, CommitmentPolicyESDK) - assert output.value == "FORBID_ENCRYPT_ALLOW_DECRYPT" - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_ALLOW_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_REQUIRE_ENCRYPT_ALLOW_DECRYPT(): - native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT - - output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) - - assert isinstance(output, CommitmentPolicyESDK) - assert output.value == "REQUIRE_ENCRYPT_ALLOW_DECRYPT" - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_REQUIRE_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_REQUIRE_ENCRYPT_REQUIRE_DECRYPT(): - native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT - - output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) - - assert isinstance(output, CommitmentPolicyESDK) - assert output.value == "REQUIRE_ENCRYPT_REQUIRE_DECRYPT" - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_CommitmentPolicy_unrecognized_WHEN_call_native_to_mpl_commmitment_policyTHEN_raise_ValueError(): - native_commitment_policy = "not a commitment policy" - - with pytest.raises(ValueError): - MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -@patch.object(mock_mpl_cmm, "decrypt_materials") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._create_mpl_decrypt_materials_input_from_request") -def test_GIVEN_valid_request_WHEN_call_decrypt_materials_THEN_return_MPLDecryptionMaterials( - mock_native_to_mpl_decrypt_materials, - mock_get_encryption_materials, -): - - # Mock: mpl_cmm.get_decryption_materials returns mock MPL decryption materials - mock_decrypt_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) - mock_decrypt_materials_output.decryption_materials = mock_mpl_decrypt_materials - mock_get_encryption_materials.return_value = mock_decrypt_materials_output - - # Mock: CMMHandler._create_mpl_decrypt_materials_input_from_request creates a DecryptMaterialsInput - mock_decrypt_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) - mock_native_to_mpl_decrypt_materials.return_value = mock_decrypt_materials_input - - cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) - output = cmm_handler.decrypt_materials(mock_decryption_materials_request) - - # Verify cmm_handler returns MPLDecryptionMaterials - assert isinstance(output, MPLDecryptionMaterials) - # Verify returned MPLDecryptionMaterials uses the output of `decrypt_materials` - assert output.mpl_materials == mock_mpl_decrypt_materials - # Verify we actually called `decrypt_materials` - mock_mpl_cmm.decrypt_materials.assert_called_once_with(mock_decrypt_materials_input) - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._create_mpl_decrypt_materials_input_from_request") -def test_GIVEN_decrypt_materials_raises_MPL_Exception_WHEN_call_decrypt_materials_THEN_raise_ESDK_Exception( - _ -): - with pytest.raises(AWSEncryptionSDKClientError): - with patch.object(mock_mpl_cmm, "decrypt_materials", - side_effect=AwsCryptographicMaterialProvidersException("any")): - - cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) - cmm_handler.decrypt_materials(mock_decryption_materials_request) - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_WHEN_call_native_algorithm_id_to_mpl_algorithm_id_THEN_returns_valid_AlgorithmSuiteIdESDK(): - some_native_algorithm_id = 0x0000 # Not a real algorithm ID, but fits the format - - mpl_output = MPLCMMHandler._native_algorithm_id_to_mpl_algorithm_id( - some_native_algorithm_id - ) - - assert isinstance(mpl_output, AlgorithmSuiteIdESDK) - assert mpl_output.value == "0x0000" - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_algorithm_id_to_mpl_algorithm_id") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") -def test__create_mpl_decrypt_materials_input_from_request( - mock_mpl_commitment_policy, - mock_mpl_algorithm_id, -): - mock_algorithm_id = "0x1234" # Some fake algorithm ID that fits the format - mock_mpl_algorithm_id.return_value = mock_algorithm_id - mock_commitment_policy = MagicMock(__class__=CommitmentPolicyESDK) - mock_mpl_commitment_policy.return_value = mock_commitment_policy - - # mock_decryption_materials_request.algorithm = - - output = MPLCMMHandler._create_mpl_decrypt_materials_input_from_request(mock_decryption_materials_request) - - assert isinstance(output, DecryptMaterialsInput) - assert output.algorithm_suite_id == mock_algorithm_id - assert output.commitment_policy == mock_commitment_policy - assert output.encryption_context == mock_decryption_materials_request.encryption_context - - assert len(output.encrypted_data_keys) == len(mock_decryption_materials_request.encrypted_data_keys) - for i in range(len(output.encrypted_data_keys)): - # Assume input[i] == output[i], seems to work - output_edk = output.encrypted_data_keys[i] - input_edk = mock_decryption_materials_request[i] - assert output_edk.key_provider_id == input_edk.key_provider.provider_id - assert output_edk.key_provider_info == input_edk.key_provider.key_info - assert output_edk.ciphertext == input_edk.encrypted_data_key diff --git a/test/unit/test_material_managers_mpl_materials.py b/test/unit/test_material_managers_mpl_materials.py deleted file mode 100644 index 250efeb7e..000000000 --- a/test/unit/test_material_managers_mpl_materials.py +++ /dev/null @@ -1,221 +0,0 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. -"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" - -import pytest -from mock import MagicMock, patch, PropertyMock -from typing import Dict, List - -from aws_encryption_sdk.identifiers import CommitmentPolicy -import aws_encryption_sdk.materials_managers.mpl.materials -from aws_encryption_sdk.materials_managers.mpl.materials import ( - MPLEncryptionMaterials, - MPLDecryptionMaterials, -) -from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite - -pytestmark = [pytest.mark.unit, pytest.mark.local] - - -# Check if MPL is installed, and skip tests based on its installation status -# Ideally, this logic would be based on mocking imports and testing logic, -# but doing that introduces errors that cause other tests to fail. -try: - from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException - from aws_cryptographic_materialproviders.mpl.models import ( - AlgorithmSuiteIdESDK, - CommitmentPolicyESDK, - DecryptMaterialsInput, - DecryptionMaterials as MPL_DecryptionMaterials, - EncryptedDataKey as MPL_EncryptedDataKey, - EncryptionMaterials as MPL_EncryptionMaterials, - GetEncryptionMaterialsInput, - GetEncryptionMaterialsOutput, - ) - from aws_cryptographic_materialproviders.mpl.references import ( - ICryptographicMaterialsManager - ) - HAS_MPL = True - - mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) - mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) - -except ImportError: - HAS_MPL = False - - # Ensure references to these mocks exist, even if they aren't used in a non-MPL context - mock_mpl_cmm = None - mock_mpl_encryption_materials = None - mock_mpl_decrypt_materials = None - -from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError -from aws_encryption_sdk.materials_managers import ( - EncryptionMaterialsRequest, - DecryptionMaterialsRequest, -) - - -mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) -mock_encryption_materials_handler = MagicMock(__class__=MPLEncryptionMaterials) -mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) - -@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") -def test_GIVEN_test_has_mpl_is_False_THEN_cmm_has_mpl_is_False(): - """If the MPL IS NOT installed in the runtime environment, - assert the cmm has _HAS_MPL set to False""" - - assert hasattr(aws_encryption_sdk.materials_managers.mpl.materials, "_HAS_MPL") - assert aws_encryption_sdk.materials_managers.mpl.materials._HAS_MPL is False - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_test_has_mpl_is_True_THEN_cmm_has_mpl_is_True(): - """If the MPL IS installed in the runtime environment, - assert the cmm has _HAS_MPL set to True""" - - assert hasattr(aws_encryption_sdk.materials_managers.mpl.materials, "_HAS_MPL") - assert aws_encryption_sdk.materials_managers.mpl.materials._HAS_MPL is True - - -@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_THEN_raise_ImportError(): - with pytest.raises(ImportError): - MPLEncryptionMaterials(mpl_materials="doesn't matter") - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_valid_mpl_cmm_THEN_return_new_MPLCMMHandler(): - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) - - assert mpl_encryption_materials.mpl_materials == mock_mpl_encryption_materials - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_invalid_mpl_cmm_THEN_raise_ValueError(): - with pytest.raises(ValueError): - MPLEncryptionMaterials(mpl_materials="not a valid mpl_materials") - -def test_mpl_to_native(): - some_mpl_algorithm_id = "0x1234" # Not a real algorithm ID, but fits the format - - native_output = aws_encryption_sdk.materials_managers.mpl.materials._mpl_algorithm_id_to_native_algorithm_id( - some_mpl_algorithm_id - ) - - assert native_output == 0x1234 - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -@patch("aws_encryption_sdk.materials_managers.mpl.materials._mpl_algorithm_id_to_native_algorithm_id") -@patch("aws_encryption_sdk.materials_managers.mpl.materials.AlgorithmSuite.get_by_id") -def test_GIVEN_valid_mpl_algorithm_id_WHEN_get_algorithm_THEN_valid_native_algorithm_id( - mock_algorithm, - mock_native_algorithm_id, -): - # Mock valid conversion from MPL to native algorithm ID - mock_native_algorithm_id.return_value = 0x1234 - - # Mock valid lookup in native AlgorithmSuite lookup - mock_algorithm.return_value = MagicMock(__class__=AlgorithmSuite) - - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) - output = mpl_encryption_materials.algorithm - assert output == mock_algorithm() # property calls automatically, we need to call the mock - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GecTHEN_valid_native_algorithm_id(): - mock_encryption_context = MagicMock(__class__=Dict[str, str]) - mock_mpl_encryption_materials.encryption_context = mock_encryption_context - - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) - output = mpl_encryption_materials.encryption_context - - assert output == mock_encryption_context - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GecTHEN_valid_nativefadsf_algorithm_id(): - mock_edk = MagicMock(__class__=MPL_EncryptedDataKey) - mock_mpl_key_provider_id = MagicMock(__class__=str) - mock_edk.key_provider_id = mock_mpl_key_provider_id - mock_mpl_key_provider_info = MagicMock(__class__=bytes) - mock_edk.key_provider_info = mock_mpl_key_provider_info - mock_mpl_ciphertext = MagicMock(__class__=bytes) - mock_edk.ciphertext = mock_mpl_ciphertext - - mock_edks = [ mock_edk ] - mock_mpl_encryption_materials.encrypted_data_keys = mock_edks - - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) - output = mpl_encryption_materials.encrypted_data_keys - output_as_list = list(output) - - assert len(output_as_list) == len(mock_edks) - for i in range(len(output_as_list)): - # assume output[i] corresponds to input[i] - native_edk = output_as_list[i] - mpl_edk = mock_edks[i] - - assert native_edk.encrypted_data_key == mpl_edk.ciphertext - assert native_edk.key_provider.provider_id == mpl_edk.key_provider_id - assert native_edk.key_provider.key_info == mpl_edk.key_provider_info - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GecTHEN_valid_nativefadsffadsfa_algorithm_id(): - mock_data_key = MagicMock(__class__=bytes) - mock_mpl_encryption_materials.plaintext_data_key = mock_data_key - - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) - output = mpl_encryption_materials.data_encryption_key - - assert output.key_provider.provider_id == "" - assert output.key_provider.key_info == b"" - assert output.data_key == mock_data_key - assert output.encrypted_data_key == b"" - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GecTHEN_valid_nativefasdfasdffadsf_algorithm_id(): - mock_signing_key = MagicMock(__class__=bytes) - mock_mpl_encryption_materials.signing_key = mock_signing_key - - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) - output = mpl_encryption_materials.signing_key - - assert output == mock_signing_key - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GecTHEN_valid_nativeffasdfasdadsffadsfa_algorithm_id(): - mock_data_key = MagicMock(__class__=bytes) - mock_mpl_decrypt_materials.plaintext_data_key = mock_data_key - - mpl_decryption_materials = MPLDecryptionMaterials(mpl_materials=mock_mpl_decrypt_materials) - output = mpl_decryption_materials.data_key - - assert output.key_provider.provider_id == "" - assert output.key_provider.key_info == b"" - assert output.data_key == mock_data_key - assert output.encrypted_data_key == b"" - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GecTHEN_validadsfasdf_nativefasdfasdffadsf_algorithm_id(): - mock_verification_key = MagicMock(__class__=bytes) - mock_mpl_decrypt_materials.verification_key = mock_verification_key - - mpl_decryption_materials = MPLDecryptionMaterials(mpl_materials=mock_mpl_decrypt_materials) - output = mpl_decryption_materials.verification_key - - assert output == mock_verification_key diff --git a/tox.ini b/tox.ini index 346e4fae0..e48f9f3b5 100644 --- a/tox.ini +++ b/tox.ini @@ -82,7 +82,7 @@ deps = # Install the MPL requirements if the `-mpl` suffix is present mpl: -rrequirements_mpl.txt commands = - local: {[testenv:base-command]commands} test/ -m local --ignore test/unit/mpl/ + local: {[testenv:base-command]commands} test/ -m local --ignore test/mpl/ # MPL unit tests require the MPL to be installed mpllocal: {[testenv:base-command]commands} test/ -m local integ: {[testenv:base-command]commands} test/ -m integ @@ -90,13 +90,13 @@ commands = examples: {[testenv:base-command]commands} examples/test/ -m examples --ignore examples/test/keyrings/ # MPL keyring examples require a special IAM role; run these separately under a separate set of permissions mplexamples: {[testenv:base-command]commands} examples/test/keyrings -m examples - all: {[testenv:base-command]commands} test/ examples/test/ --ignore test/unit/mpl/ --ignore examples/test/keyrings/ + all: {[testenv:base-command]commands} test/ examples/test/ --ignore test/mpl/ --ignore examples/test/keyrings/ mplall: {[testenv:base-command]commands} test/ examples/test/ manual: {[testenv:base-command]commands} # Run code coverage on the unit tests [testenv:coverage] -commands = {[testenv:base-command]commands} --cov aws_encryption_sdk test/ -m local --ignore test/unit/mpl/ +commands = {[testenv:base-command]commands} --cov aws_encryption_sdk test/ -m local --ignore test/mpl/ [testenv:mplcoverage-mpl] commands = {[testenv:base-command]commands} --cov aws_encryption_sdk test/ -m local From 61ba4dec6bdc404a13bb245fa5bbb2078d014edc Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 10:36:39 -0800 Subject: [PATCH 109/376] refactor tests --- src/aws_encryption_sdk/streaming_client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 61f2f88c6..01d4ca5ac 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -58,7 +58,6 @@ serialize_non_framed_close, serialize_non_framed_open, ) -from aws_encryption_sdk.materials_managers.mpl.cmm import MPLCMMHandler from aws_encryption_sdk.internal.utils import exactly_one_arg_is_not_none from aws_encryption_sdk.internal.utils.commitment import ( validate_commitment_policy_on_decrypt, @@ -79,6 +78,10 @@ from aws_cryptographic_materialproviders.mpl.models import CreateDefaultCryptographicMaterialsManagerInput from aws_cryptographic_materialproviders.mpl.references import IKeyring _HAS_MPL = True + + # Import internal ESDK modules that depend on the MPL + from aws_encryption_sdk.materials_managers.mpl.cmm import MPLCMMHandler + except ImportError: _HAS_MPL = False From 95c5be6bfb7fea0ef31ebcc8e5ddec0ac07df9fd Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 10:37:17 -0800 Subject: [PATCH 110/376] refactor tests --- src/aws_encryption_sdk/streaming_client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 01d4ca5ac..a3c05bbb7 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -554,7 +554,8 @@ def _prep_message(self): else: # MPL verification key is PEM bytes, not DER bytes. # If the underlying CMM is from the MPL, load PEM bytes. - if (isinstance(self.config.materials_manager, MPLCMMHandler)): + if (_HAS_MPL + and isinstance(self.config.materials_manager, MPLCMMHandler)): self.signer = Signer.from_key_bytes( algorithm=self._encryption_materials.algorithm, key_bytes=self._encryption_materials.signing_key, encoding=serialization.Encoding.PEM, @@ -921,7 +922,8 @@ def _read_header(self): else: # MPL verification key is NOT key bytes; it is bytes of the compressed point. # If the underlying CMM is from the MPL, load bytes from encoded point. - if (isinstance(self.config.materials_manager, MPLCMMHandler)): + if (_HAS_MPL + and isinstance(self.config.materials_manager, MPLCMMHandler)): self.verifier = Verifier.from_encoded_point( algorithm=header.algorithm, encoded_point=base64.b64encode(decryption_materials.verification_key) From 9566873a946acb70622962bdbcc8b2086e88e16d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 10:51:45 -0800 Subject: [PATCH 111/376] refactor tests --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index e48f9f3b5..9e3fe2b98 100644 --- a/tox.ini +++ b/tox.ini @@ -96,7 +96,7 @@ commands = # Run code coverage on the unit tests [testenv:coverage] -commands = {[testenv:base-command]commands} --cov aws_encryption_sdk test/ -m local --ignore test/mpl/ +commands = {[testenv:base-command]commands} --cov aws_encryption_sdk --cov-config=.coveragerc test/ -m local --ignore test/mpl/ [testenv:mplcoverage-mpl] commands = {[testenv:base-command]commands} --cov aws_encryption_sdk test/ -m local From 66420832da4a4a8806254f0b8487ee03452a17d1 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 11:16:25 -0800 Subject: [PATCH 112/376] fix cov --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 9e3fe2b98..70ed4281f 100644 --- a/tox.ini +++ b/tox.ini @@ -96,9 +96,9 @@ commands = # Run code coverage on the unit tests [testenv:coverage] -commands = {[testenv:base-command]commands} --cov aws_encryption_sdk --cov-config=.coveragerc test/ -m local --ignore test/mpl/ +commands = {[testenv:base-command]commands} --cov aws_encryption_sdk test/ -m local --ignore test/mpl/ [testenv:mplcoverage-mpl] -commands = {[testenv:base-command]commands} --cov aws_encryption_sdk test/ -m local +commands = {[testenv:base-command]commands} --cov --cov-config=.coveragercmpl aws_encryption_sdk test/ -m local # Verify that local tests work without environment variables present [testenv:nocmk] From 51d2804343797d98848f7eea2b8f333f9f7f8a46 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 11:22:26 -0800 Subject: [PATCH 113/376] fix cov --- .coveragerc | 7 + .coveragercmpl | 1 + .gitignore | 5 +- test/mpl/README.md | 1 + test/mpl/__init__.py | 13 + .../unit/test_material_managers_mpl_cmm.py | 250 ++++++++++++++++++ .../test_material_managers_mpl_materials.py | 197 ++++++++++++++ 7 files changed, 472 insertions(+), 2 deletions(-) create mode 100644 .coveragerc create mode 100644 .coveragercmpl create mode 100644 test/mpl/README.md create mode 100644 test/mpl/__init__.py create mode 100644 test/mpl/unit/test_material_managers_mpl_cmm.py create mode 100644 test/mpl/unit/test_material_managers_mpl_materials.py diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..8957349aa --- /dev/null +++ b/.coveragerc @@ -0,0 +1,7 @@ +# .coveragerc file when running coverage WITHOUT coverage for the MPL +# This prevents the ESDK without the MPL from considering the MPL-specific modules as "missed" coverage +[run] +omit = */aws_encryption_sdk/materials_managers/mpl/* + +[report] +omit = */aws_encryption_sdk/materials_managers/mpl/* \ No newline at end of file diff --git a/.coveragercmpl b/.coveragercmpl new file mode 100644 index 000000000..31a7b4407 --- /dev/null +++ b/.coveragercmpl @@ -0,0 +1 @@ +# .coveragerc file when running coverage WITH coverage for the MPL diff --git a/.gitignore b/.gitignore index 63097dcba..fc224adc4 100644 --- a/.gitignore +++ b/.gitignore @@ -19,8 +19,9 @@ docs/build __pycache__ *.egg-info -# Coverage.py -.coverage* +# Coverage.py, NOT .coveragerc nor .coveragercmpl +.coverage +.coverage.py # MyPy .mypy_cache diff --git a/test/mpl/README.md b/test/mpl/README.md new file mode 100644 index 000000000..7ae7134d0 --- /dev/null +++ b/test/mpl/README.md @@ -0,0 +1 @@ +Tests in this directory REQUIRE the [aws-cryptographic-material-providers](https://github.com/aws/aws-cryptographic-material-providers-library) library to execute. \ No newline at end of file diff --git a/test/mpl/__init__.py b/test/mpl/__init__.py new file mode 100644 index 000000000..b976c1308 --- /dev/null +++ b/test/mpl/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Module containing tests that REQUIRE the aws-cryptographic-material-providers library to run.""" \ No newline at end of file diff --git a/test/mpl/unit/test_material_managers_mpl_cmm.py b/test/mpl/unit/test_material_managers_mpl_cmm.py new file mode 100644 index 000000000..b1589b1cf --- /dev/null +++ b/test/mpl/unit/test_material_managers_mpl_cmm.py @@ -0,0 +1,250 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" + +import pytest +from mock import MagicMock, patch + + +from aws_encryption_sdk.identifiers import CommitmentPolicy +import aws_encryption_sdk.materials_managers.mpl.cmm +from aws_encryption_sdk.materials_managers.mpl.cmm import MPLCMMHandler +from aws_encryption_sdk.materials_managers.mpl.materials import ( + MPLEncryptionMaterials, + MPLDecryptionMaterials, +) + +pytestmark = [pytest.mark.unit, pytest.mark.local] + + +from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException +from aws_cryptographic_materialproviders.mpl.models import ( + AlgorithmSuiteIdESDK, + CommitmentPolicyESDK, + DecryptMaterialsInput, + DecryptionMaterials as MPL_DecryptionMaterials, + EncryptionMaterials as MPL_EncryptionMaterials, + GetEncryptionMaterialsInput, + GetEncryptionMaterialsOutput, +) +from aws_cryptographic_materialproviders.mpl.references import ( + ICryptographicMaterialsManager +) + +mock_mpl_cmm = MagicMock(__class__=ICryptographicMaterialsManager) +mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) +mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) + + +from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError +from aws_encryption_sdk.materials_managers import ( + EncryptionMaterialsRequest, + DecryptionMaterialsRequest, +) + + +mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) +mock_encryption_materials_handler = MagicMock(__class__=MPLEncryptionMaterials) +mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) + +def test_GIVEN_test_has_mpl_is_False_THEN_cmm_has_mpl_is_False(): + """If the MPL IS NOT installed in the runtime environment, + assert the cmm has _HAS_MPL set to False""" + + assert hasattr(aws_encryption_sdk.materials_managers.mpl.cmm, "_HAS_MPL") + assert aws_encryption_sdk.materials_managers.mpl.cmm._HAS_MPL is False + + +def test_GIVEN_test_has_mpl_is_True_THEN_cmm_has_mpl_is_True(): + """If the MPL IS installed in the runtime environment, + assert the cmm has _HAS_MPL set to True""" + + assert hasattr(aws_encryption_sdk.materials_managers.mpl.cmm, "_HAS_MPL") + assert aws_encryption_sdk.materials_managers.mpl.cmm._HAS_MPL is True + + +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_THEN_raise_ImportError(): + with pytest.raises(ImportError): + MPLCMMHandler(mpl_cmm="doesn't matter") + + +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_valid_mpl_cmm_THEN_return_new_MPLCMMHandler(): + mpl_cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) + + assert mpl_cmm_handler.mpl_cmm == mock_mpl_cmm + + +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_invalid_mpl_cmm_THEN_raise_ValueError(): + with pytest.raises(ValueError): + MPLCMMHandler(mpl_cmm="not a valid mpl_cmm") + + +@patch.object(mock_mpl_cmm, "get_encryption_materials") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_get_encryption_materials") +def test_GIVEN_valid_request_WHEN_call_get_encryption_materials_THEN_return_MPLEncryptionMaterials( + mock_native_to_mpl_get_encryption_materials, + mock_get_encryption_materials, +): + + # Mock: mpl_cmm.get_encryption_materials returns mock MPL encryption materials + mock_get_encryption_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) + mock_get_encryption_materials_output.encryption_materials = mock_mpl_encryption_materials + mock_get_encryption_materials.return_value = mock_get_encryption_materials_output + + # Mock: CMMHandler._native_to_mpl_get_encryption_materials creates a GetEncryptionMaterialsInput + mock_get_encryption_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) + mock_native_to_mpl_get_encryption_materials.return_value = mock_get_encryption_materials_input + + cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) + test = cmm_handler.get_encryption_materials(mock_encryption_materials_request) + + # Verify cmm_handler returns MPLEncryptionMaterials + assert isinstance(test, MPLEncryptionMaterials) + # Verify returned EncryptionMaterialsHandler uses the output of `get_encryption_materials` + assert test.mpl_materials == mock_mpl_encryption_materials + # Verify we actually called `get_encryption_materials` + mock_mpl_cmm.get_encryption_materials.assert_called_once_with(mock_get_encryption_materials_input) + + +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") +def test_GIVEN_get_encryption_materials_raises_MPL_Exception_WHEN_call_get_encryption_materials_THEN_raise_ESDK_Exception( + _ +): + with pytest.raises(AWSEncryptionSDKClientError): + with patch.object(mock_mpl_cmm, "get_encryption_materials", + side_effect=AwsCryptographicMaterialProvidersException("any")): + + cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) + cmm_handler.get_encryption_materials(mock_encryption_materials_request) + +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") +def test_GIVEN_native_to_mpl_commmitment_policy_returns_valid_policy_WHEN_call_native_to_mpl_get_encryption_materials_THEN_returns_GetEncryptionMaterialsInput( + mock_mpl_commitment_policy +): + mock_commitment_policy = MagicMock(__class__=CommitmentPolicyESDK) + mock_mpl_commitment_policy.return_value = mock_commitment_policy + + output = MPLCMMHandler._native_to_mpl_get_encryption_materials(mock_encryption_materials_request) + + # verify correctness of returned value + assert isinstance(output, GetEncryptionMaterialsInput) + assert output.encryption_context == mock_encryption_materials_request.encryption_context + assert output.commitment_policy == mock_commitment_policy + assert output.max_plaintext_length == mock_encryption_materials_request.plaintext_length + + +def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_FORBID_ENCRYPT_ALLOW_DECRYPT(): + native_commitment_policy = CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT + + output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) + + assert isinstance(output, CommitmentPolicyESDK) + assert output.value == "FORBID_ENCRYPT_ALLOW_DECRYPT" + +def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_ALLOW_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_REQUIRE_ENCRYPT_ALLOW_DECRYPT(): + native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT + + output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) + + assert isinstance(output, CommitmentPolicyESDK) + assert output.value == "REQUIRE_ENCRYPT_ALLOW_DECRYPT" + +def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_REQUIRE_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_REQUIRE_ENCRYPT_REQUIRE_DECRYPT(): + native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT + + output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) + + assert isinstance(output, CommitmentPolicyESDK) + assert output.value == "REQUIRE_ENCRYPT_REQUIRE_DECRYPT" + +def test_GIVEN_CommitmentPolicy_unrecognized_WHEN_call_native_to_mpl_commmitment_policyTHEN_raise_ValueError(): + native_commitment_policy = "not a commitment policy" + + with pytest.raises(ValueError): + MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) + +@patch.object(mock_mpl_cmm, "decrypt_materials") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._create_mpl_decrypt_materials_input_from_request") +def test_GIVEN_valid_request_WHEN_call_decrypt_materials_THEN_return_MPLDecryptionMaterials( + mock_native_to_mpl_decrypt_materials, + mock_get_encryption_materials, +): + + # Mock: mpl_cmm.get_decryption_materials returns mock MPL decryption materials + mock_decrypt_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) + mock_decrypt_materials_output.decryption_materials = mock_mpl_decrypt_materials + mock_get_encryption_materials.return_value = mock_decrypt_materials_output + + # Mock: CMMHandler._create_mpl_decrypt_materials_input_from_request creates a DecryptMaterialsInput + mock_decrypt_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) + mock_native_to_mpl_decrypt_materials.return_value = mock_decrypt_materials_input + + cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) + output = cmm_handler.decrypt_materials(mock_decryption_materials_request) + + # Verify cmm_handler returns MPLDecryptionMaterials + assert isinstance(output, MPLDecryptionMaterials) + # Verify returned MPLDecryptionMaterials uses the output of `decrypt_materials` + assert output.mpl_materials == mock_mpl_decrypt_materials + # Verify we actually called `decrypt_materials` + mock_mpl_cmm.decrypt_materials.assert_called_once_with(mock_decrypt_materials_input) + +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._create_mpl_decrypt_materials_input_from_request") +def test_GIVEN_decrypt_materials_raises_MPL_Exception_WHEN_call_decrypt_materials_THEN_raise_ESDK_Exception( + _ +): + with pytest.raises(AWSEncryptionSDKClientError): + with patch.object(mock_mpl_cmm, "decrypt_materials", + side_effect=AwsCryptographicMaterialProvidersException("any")): + + cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) + cmm_handler.decrypt_materials(mock_decryption_materials_request) + +def test_WHEN_call_native_algorithm_id_to_mpl_algorithm_id_THEN_returns_valid_AlgorithmSuiteIdESDK(): + some_native_algorithm_id = 0x0000 # Not a real algorithm ID, but fits the format + + mpl_output = MPLCMMHandler._native_algorithm_id_to_mpl_algorithm_id( + some_native_algorithm_id + ) + + assert isinstance(mpl_output, AlgorithmSuiteIdESDK) + assert mpl_output.value == "0x0000" + +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_algorithm_id_to_mpl_algorithm_id") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") +def test__create_mpl_decrypt_materials_input_from_request( + mock_mpl_commitment_policy, + mock_mpl_algorithm_id, +): + mock_algorithm_id = "0x1234" # Some fake algorithm ID that fits the format + mock_mpl_algorithm_id.return_value = mock_algorithm_id + mock_commitment_policy = MagicMock(__class__=CommitmentPolicyESDK) + mock_mpl_commitment_policy.return_value = mock_commitment_policy + + # mock_decryption_materials_request.algorithm = + + output = MPLCMMHandler._create_mpl_decrypt_materials_input_from_request(mock_decryption_materials_request) + + assert isinstance(output, DecryptMaterialsInput) + assert output.algorithm_suite_id == mock_algorithm_id + assert output.commitment_policy == mock_commitment_policy + assert output.encryption_context == mock_decryption_materials_request.encryption_context + + assert len(output.encrypted_data_keys) == len(mock_decryption_materials_request.encrypted_data_keys) + for i in range(len(output.encrypted_data_keys)): + # Assume input[i] == output[i], seems to work + output_edk = output.encrypted_data_keys[i] + input_edk = mock_decryption_materials_request[i] + assert output_edk.key_provider_id == input_edk.key_provider.provider_id + assert output_edk.key_provider_info == input_edk.key_provider.key_info + assert output_edk.ciphertext == input_edk.encrypted_data_key diff --git a/test/mpl/unit/test_material_managers_mpl_materials.py b/test/mpl/unit/test_material_managers_mpl_materials.py new file mode 100644 index 000000000..dfd6b2769 --- /dev/null +++ b/test/mpl/unit/test_material_managers_mpl_materials.py @@ -0,0 +1,197 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" + +import pytest +from mock import MagicMock, patch, PropertyMock +from typing import Dict, List + +from aws_encryption_sdk.identifiers import CommitmentPolicy +import aws_encryption_sdk.materials_managers.mpl.materials +from aws_encryption_sdk.materials_managers.mpl.materials import ( + MPLEncryptionMaterials, + MPLDecryptionMaterials, +) +from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite + +pytestmark = [pytest.mark.unit, pytest.mark.local] + + +from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException +from aws_cryptographic_materialproviders.mpl.models import ( + AlgorithmSuiteIdESDK, + CommitmentPolicyESDK, + DecryptMaterialsInput, + DecryptionMaterials as MPL_DecryptionMaterials, + EncryptedDataKey as MPL_EncryptedDataKey, + EncryptionMaterials as MPL_EncryptionMaterials, + GetEncryptionMaterialsInput, + GetEncryptionMaterialsOutput, +) +from aws_cryptographic_materialproviders.mpl.references import ( + ICryptographicMaterialsManager +) + +mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) +mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) + + +from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError +from aws_encryption_sdk.materials_managers import ( + EncryptionMaterialsRequest, + DecryptionMaterialsRequest, +) + + +mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) +mock_encryption_materials_handler = MagicMock(__class__=MPLEncryptionMaterials) +mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) + +def test_GIVEN_test_has_mpl_is_False_THEN_cmm_has_mpl_is_False(): + """If the MPL IS NOT installed in the runtime environment, + assert the cmm has _HAS_MPL set to False""" + + assert hasattr(aws_encryption_sdk.materials_managers.mpl.materials, "_HAS_MPL") + assert aws_encryption_sdk.materials_managers.mpl.materials._HAS_MPL is False + + +def test_GIVEN_test_has_mpl_is_True_THEN_cmm_has_mpl_is_True(): + """If the MPL IS installed in the runtime environment, + assert the cmm has _HAS_MPL set to True""" + + assert hasattr(aws_encryption_sdk.materials_managers.mpl.materials, "_HAS_MPL") + assert aws_encryption_sdk.materials_managers.mpl.materials._HAS_MPL is True + + +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_THEN_raise_ImportError(): + with pytest.raises(ImportError): + MPLEncryptionMaterials(mpl_materials="doesn't matter") + + +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_valid_mpl_cmm_THEN_return_new_MPLCMMHandler(): + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + + assert mpl_encryption_materials.mpl_materials == mock_mpl_encryption_materials + + +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_invalid_mpl_cmm_THEN_raise_ValueError(): + with pytest.raises(ValueError): + MPLEncryptionMaterials(mpl_materials="not a valid mpl_materials") + +def test_mpl_to_native(): + some_mpl_algorithm_id = "0x1234" # Not a real algorithm ID, but fits the format + + native_output = aws_encryption_sdk.materials_managers.mpl.materials._mpl_algorithm_id_to_native_algorithm_id( + some_mpl_algorithm_id + ) + + assert native_output == 0x1234 + + +@patch("aws_encryption_sdk.materials_managers.mpl.materials._mpl_algorithm_id_to_native_algorithm_id") +@patch("aws_encryption_sdk.materials_managers.mpl.materials.AlgorithmSuite.get_by_id") +def test_GIVEN_valid_mpl_algorithm_id_WHEN_get_algorithm_THEN_valid_native_algorithm_id( + mock_algorithm, + mock_native_algorithm_id, +): + # Mock valid conversion from MPL to native algorithm ID + mock_native_algorithm_id.return_value = 0x1234 + + # Mock valid lookup in native AlgorithmSuite lookup + mock_algorithm.return_value = MagicMock(__class__=AlgorithmSuite) + + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + output = mpl_encryption_materials.algorithm + assert output == mock_algorithm() # property calls automatically, we need to call the mock + + +def test_GecTHEN_valid_native_algorithm_id(): + mock_encryption_context = MagicMock(__class__=Dict[str, str]) + mock_mpl_encryption_materials.encryption_context = mock_encryption_context + + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + output = mpl_encryption_materials.encryption_context + + assert output == mock_encryption_context + + +def test_GecTHEN_valid_nativefadsf_algorithm_id(): + mock_edk = MagicMock(__class__=MPL_EncryptedDataKey) + mock_mpl_key_provider_id = MagicMock(__class__=str) + mock_edk.key_provider_id = mock_mpl_key_provider_id + mock_mpl_key_provider_info = MagicMock(__class__=bytes) + mock_edk.key_provider_info = mock_mpl_key_provider_info + mock_mpl_ciphertext = MagicMock(__class__=bytes) + mock_edk.ciphertext = mock_mpl_ciphertext + + mock_edks = [ mock_edk ] + mock_mpl_encryption_materials.encrypted_data_keys = mock_edks + + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + output = mpl_encryption_materials.encrypted_data_keys + output_as_list = list(output) + + assert len(output_as_list) == len(mock_edks) + for i in range(len(output_as_list)): + # assume output[i] corresponds to input[i] + native_edk = output_as_list[i] + mpl_edk = mock_edks[i] + + assert native_edk.encrypted_data_key == mpl_edk.ciphertext + assert native_edk.key_provider.provider_id == mpl_edk.key_provider_id + assert native_edk.key_provider.key_info == mpl_edk.key_provider_info + +def test_GecTHEN_valid_nativefadsffadsfa_algorithm_id(): + mock_data_key = MagicMock(__class__=bytes) + mock_mpl_encryption_materials.plaintext_data_key = mock_data_key + + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + output = mpl_encryption_materials.data_encryption_key + + assert output.key_provider.provider_id == "" + assert output.key_provider.key_info == b"" + assert output.data_key == mock_data_key + assert output.encrypted_data_key == b"" + + +def test_GecTHEN_valid_nativefasdfasdffadsf_algorithm_id(): + mock_signing_key = MagicMock(__class__=bytes) + mock_mpl_encryption_materials.signing_key = mock_signing_key + + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + output = mpl_encryption_materials.signing_key + + assert output == mock_signing_key + + +def test_GecTHEN_valid_nativeffasdfasdadsffadsfa_algorithm_id(): + mock_data_key = MagicMock(__class__=bytes) + mock_mpl_decrypt_materials.plaintext_data_key = mock_data_key + + mpl_decryption_materials = MPLDecryptionMaterials(mpl_materials=mock_mpl_decrypt_materials) + output = mpl_decryption_materials.data_key + + assert output.key_provider.provider_id == "" + assert output.key_provider.key_info == b"" + assert output.data_key == mock_data_key + assert output.encrypted_data_key == b"" + + +def test_GecTHEN_validadsfasdf_nativefasdfasdffadsf_algorithm_id(): + mock_verification_key = MagicMock(__class__=bytes) + mock_mpl_decrypt_materials.verification_key = mock_verification_key + + mpl_decryption_materials = MPLDecryptionMaterials(mpl_materials=mock_mpl_decrypt_materials) + output = mpl_decryption_materials.verification_key + + assert output == mock_verification_key From 51e5db501c41d5c8e2351daa7b4331a21132f2b7 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 11:26:15 -0800 Subject: [PATCH 114/376] fix cov --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 70ed4281f..1cc78c45e 100644 --- a/tox.ini +++ b/tox.ini @@ -98,7 +98,7 @@ commands = [testenv:coverage] commands = {[testenv:base-command]commands} --cov aws_encryption_sdk test/ -m local --ignore test/mpl/ [testenv:mplcoverage-mpl] -commands = {[testenv:base-command]commands} --cov --cov-config=.coveragercmpl aws_encryption_sdk test/ -m local +commands = {[testenv:base-command]commands} --cov-config=.coveragercmpl --cov aws_encryption_sdk test/ -m local # Verify that local tests work without environment variables present [testenv:nocmk] From e2354613a6d9c14c8cd4116a399e27010010d6f4 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 11:29:58 -0800 Subject: [PATCH 115/376] fix cov --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 1cc78c45e..419188b54 100644 --- a/tox.ini +++ b/tox.ini @@ -85,8 +85,8 @@ commands = local: {[testenv:base-command]commands} test/ -m local --ignore test/mpl/ # MPL unit tests require the MPL to be installed mpllocal: {[testenv:base-command]commands} test/ -m local - integ: {[testenv:base-command]commands} test/ -m integ - accept: {[testenv:base-command]commands} test/ -m accept + integ: {[testenv:base-command]commands} test/ -m integ --ignore test/mpl/ + accept: {[testenv:base-command]commands} test/ -m accept --ignore test/mpl/ examples: {[testenv:base-command]commands} examples/test/ -m examples --ignore examples/test/keyrings/ # MPL keyring examples require a special IAM role; run these separately under a separate set of permissions mplexamples: {[testenv:base-command]commands} examples/test/keyrings -m examples From e7c745fbdb8e33d342a9027196519c53833fcba8 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 11:34:58 -0800 Subject: [PATCH 116/376] fix tests --- .../unit/test_material_managers_mpl_cmm.py | 20 ------------------- .../test_material_managers_mpl_materials.py | 20 ------------------- tox.ini | 8 +++++--- 3 files changed, 5 insertions(+), 43 deletions(-) diff --git a/test/mpl/unit/test_material_managers_mpl_cmm.py b/test/mpl/unit/test_material_managers_mpl_cmm.py index b1589b1cf..cae334722 100644 --- a/test/mpl/unit/test_material_managers_mpl_cmm.py +++ b/test/mpl/unit/test_material_managers_mpl_cmm.py @@ -57,26 +57,6 @@ mock_encryption_materials_handler = MagicMock(__class__=MPLEncryptionMaterials) mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) -def test_GIVEN_test_has_mpl_is_False_THEN_cmm_has_mpl_is_False(): - """If the MPL IS NOT installed in the runtime environment, - assert the cmm has _HAS_MPL set to False""" - - assert hasattr(aws_encryption_sdk.materials_managers.mpl.cmm, "_HAS_MPL") - assert aws_encryption_sdk.materials_managers.mpl.cmm._HAS_MPL is False - - -def test_GIVEN_test_has_mpl_is_True_THEN_cmm_has_mpl_is_True(): - """If the MPL IS installed in the runtime environment, - assert the cmm has _HAS_MPL set to True""" - - assert hasattr(aws_encryption_sdk.materials_managers.mpl.cmm, "_HAS_MPL") - assert aws_encryption_sdk.materials_managers.mpl.cmm._HAS_MPL is True - - -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_THEN_raise_ImportError(): - with pytest.raises(ImportError): - MPLCMMHandler(mpl_cmm="doesn't matter") - def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_valid_mpl_cmm_THEN_return_new_MPLCMMHandler(): mpl_cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) diff --git a/test/mpl/unit/test_material_managers_mpl_materials.py b/test/mpl/unit/test_material_managers_mpl_materials.py index dfd6b2769..bb83b89fd 100644 --- a/test/mpl/unit/test_material_managers_mpl_materials.py +++ b/test/mpl/unit/test_material_managers_mpl_materials.py @@ -57,26 +57,6 @@ mock_encryption_materials_handler = MagicMock(__class__=MPLEncryptionMaterials) mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) -def test_GIVEN_test_has_mpl_is_False_THEN_cmm_has_mpl_is_False(): - """If the MPL IS NOT installed in the runtime environment, - assert the cmm has _HAS_MPL set to False""" - - assert hasattr(aws_encryption_sdk.materials_managers.mpl.materials, "_HAS_MPL") - assert aws_encryption_sdk.materials_managers.mpl.materials._HAS_MPL is False - - -def test_GIVEN_test_has_mpl_is_True_THEN_cmm_has_mpl_is_True(): - """If the MPL IS installed in the runtime environment, - assert the cmm has _HAS_MPL set to True""" - - assert hasattr(aws_encryption_sdk.materials_managers.mpl.materials, "_HAS_MPL") - assert aws_encryption_sdk.materials_managers.mpl.materials._HAS_MPL is True - - -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_THEN_raise_ImportError(): - with pytest.raises(ImportError): - MPLEncryptionMaterials(mpl_materials="doesn't matter") - def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_valid_mpl_cmm_THEN_return_new_MPLCMMHandler(): mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) diff --git a/tox.ini b/tox.ini index 419188b54..3daa40e47 100644 --- a/tox.ini +++ b/tox.ini @@ -12,9 +12,11 @@ envlist = # tests in a test environment that also has the MPL. py{311,312}-{local,integ,accept,examples}{,-mpl}, # >=3.11: Run ONLY the MPL-specific tests. - # These must be separate from the above target. - # These require the `-mpl` suffix so tox installs the MPL. - # The `mpl` prefix runs only MPL-specific tests + # These must be separate from the above target, since + # these require the `-mpl` suffix. + # The `mpl` prefix specifies a separate target, + # i.e. `mpllocal` instead of `local`. + # `mplXXX` contains tests using MPL components. py{311,312}-mpl{local,examples}-mpl nocmk, bandit, doc8, readme, docs, From fee4f36f44c94bb357d10272190359fdf5d82e62 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 16:57:03 -0800 Subject: [PATCH 117/376] test cleanup --- .../example_branch_key_id_supplier.py | 5 +- .../materials_managers/mpl/cmm.py | 78 +++---- .../materials_managers/mpl/materials.py | 12 +- src/aws_encryption_sdk/streaming_client.py | 8 +- .../unit/test_material_managers_mpl_cmm.py | 207 +++++++++++------- .../test_material_managers_mpl_materials.py | 139 +++++++----- 6 files changed, 271 insertions(+), 178 deletions(-) diff --git a/examples/src/keyrings/example_branch_key_id_supplier.py b/examples/src/keyrings/example_branch_key_id_supplier.py index a06280fa1..ba9ae060c 100644 --- a/examples/src/keyrings/example_branch_key_id_supplier.py +++ b/examples/src/keyrings/example_branch_key_id_supplier.py @@ -18,11 +18,10 @@ def __init__(self, tenant_1_id, tenant_2_id): def get_branch_key_id( self, - # TODO-MPL: Change this to `native_input` in Smithy-Dafny - input: GetBranchKeyIdInput # noqa pylint: disable=redefined-builtin + param: GetBranchKeyIdInput ) -> GetBranchKeyIdOutput: """Returns branch key ID from the tenant ID in input's encryption context.""" - encryption_context: Dict[str, str] = input.encryption_context + encryption_context: Dict[str, str] = param.encryption_context if b"tenant" not in encryption_context: raise ValueError("EncryptionContext invalid, does not contain expected tenant key value pair.") diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index 1e3e3fb34..c97c070f0 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -2,62 +2,64 @@ from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException from aws_cryptographic_materialproviders.mpl.models import ( - AlgorithmSuiteIdESDK, - CommitmentPolicyESDK, - DecryptMaterialsInput, - DecryptMaterialsOutput, + AlgorithmSuiteIdESDK as MPL_AlgorithmSuiteIdESDK, + CommitmentPolicyESDK as MPL_CommitmentPolicyESDK, + DecryptMaterialsInput as MPL_DecryptMaterialsInput, + DecryptMaterialsOutput as MPL_DecryptMaterialsOutput, EncryptedDataKey as MPL_EncryptedDataKey, - GetEncryptionMaterialsInput, - GetEncryptionMaterialsOutput, + GetEncryptionMaterialsInput as MPL_GetEncryptionMaterialsInput, + GetEncryptionMaterialsOutput as MPL_GetEncryptionMaterialsOutput, +) +from aws_cryptographic_materialproviders.mpl.references import ( + ICryptographicMaterialsManager as MPL_ICryptographicMaterialsManager ) -from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager from typing import List from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError from aws_encryption_sdk.identifiers import CommitmentPolicy -from aws_encryption_sdk.materials_managers.mpl.materials import MPLEncryptionMaterials, MPLDecryptionMaterials +from aws_encryption_sdk.materials_managers.mpl.materials import EncryptionMaterialsFromMPL, DecryptionMaterialsFromMPL from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager from aws_encryption_sdk.structures import EncryptedDataKey as Native_EncryptedDataKey -class MPLCMMHandler(CryptoMaterialsManager): +class CryptoMaterialsManagerFromMPL(CryptoMaterialsManager): """ In instances where encryption materials are provided by an implementation of the MPL's - `aws_cryptographic_materialproviders.mpl.references.ICryptographicMaterialsManager`, + `aws_cryptographic_materialproviders.mpl.references.MPL_ICryptographicMaterialsManager`, this maps the ESDK CMM interfaces to the MPL CMM. """ - mpl_cmm: 'ICryptographicMaterialsManager' + mpl_cmm: 'MPL_ICryptographicMaterialsManager' def __init__( self, - mpl_cmm: 'ICryptographicMaterialsManager' + mpl_cmm: 'MPL_ICryptographicMaterialsManager' ): """ - Create MPLCMMHandler. + Create CryptoMaterialsManagerFromMPL. :param mpl_cmm: Underlying MPL cryptographic materials manager """ - if isinstance(mpl_cmm, ICryptographicMaterialsManager): + if isinstance(mpl_cmm, MPL_ICryptographicMaterialsManager): self.mpl_cmm = mpl_cmm else: - raise ValueError(f"Invalid CMM passed to MPLCMMHandler. cmm: {mpl_cmm}") + raise ValueError(f"Invalid CMM passed to CryptoMaterialsManagerFromMPL. cmm: {mpl_cmm}") def get_encryption_materials( self, request: EncryptionMaterialsRequest - ) -> MPLEncryptionMaterials: + ) -> EncryptionMaterialsFromMPL: """ Returns an EncryptionMaterialsHandler for the configured CMM. :param request: Request for encryption materials """ try: - mpl_input: GetEncryptionMaterialsInput = MPLCMMHandler._native_to_mpl_get_encryption_materials( + mpl_input: MPL_GetEncryptionMaterialsInput = CryptoMaterialsManagerFromMPL._native_to_mpl_get_encryption_materials( request ) - mpl_output: GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) - return MPLEncryptionMaterials(mpl_output.encryption_materials) + mpl_output: MPL_GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) + return EncryptionMaterialsFromMPL(mpl_output.encryption_materials) except AwsCryptographicMaterialProvidersException as mpl_exception: # Wrap MPL error into the ESDK error type # so customers only have to catch ESDK error types. @@ -66,11 +68,11 @@ def get_encryption_materials( @staticmethod def _native_to_mpl_get_encryption_materials( request: EncryptionMaterialsRequest - ) -> 'GetEncryptionMaterialsInput': - commitment_policy = MPLCMMHandler._native_to_mpl_commmitment_policy( + ) -> 'MPL_GetEncryptionMaterialsInput': + commitment_policy = CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy( request.commitment_policy ) - output: GetEncryptionMaterialsInput = GetEncryptionMaterialsInput( + output: MPL_GetEncryptionMaterialsInput = MPL_GetEncryptionMaterialsInput( encryption_context=request.encryption_context, commitment_policy=commitment_policy, max_plaintext_length=request.plaintext_length, @@ -80,54 +82,54 @@ def _native_to_mpl_get_encryption_materials( @staticmethod def _native_to_mpl_commmitment_policy( native_commitment_policy: CommitmentPolicy - ) -> 'CommitmentPolicyESDK': + ) -> 'MPL_CommitmentPolicyESDK': if native_commitment_policy == CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT: - return CommitmentPolicyESDK(value="FORBID_ENCRYPT_ALLOW_DECRYPT") + return MPL_CommitmentPolicyESDK(value="FORBID_ENCRYPT_ALLOW_DECRYPT") elif native_commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT: - return CommitmentPolicyESDK(value="REQUIRE_ENCRYPT_ALLOW_DECRYPT") + return MPL_CommitmentPolicyESDK(value="REQUIRE_ENCRYPT_ALLOW_DECRYPT") elif native_commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT: - return CommitmentPolicyESDK(value="REQUIRE_ENCRYPT_REQUIRE_DECRYPT") + return MPL_CommitmentPolicyESDK(value="REQUIRE_ENCRYPT_REQUIRE_DECRYPT") else: raise ValueError(f"Invalid native_commitment_policy: {native_commitment_policy}") def decrypt_materials( self, request: DecryptionMaterialsRequest - ) -> MPLDecryptionMaterials: + ) -> DecryptionMaterialsFromMPL: """ - Returns a MPLDecryptionMaterials for the configured CMM. + Returns a DecryptionMaterialsFromMPL for the configured CMM. :param request: Request for decryption materials """ try: - mpl_input: 'DecryptMaterialsInput' = \ - MPLCMMHandler._create_mpl_decrypt_materials_input_from_request(request) - mpl_output: 'DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(mpl_input) - return MPLDecryptionMaterials(mpl_output.decryption_materials) + mpl_input: 'MPL_DecryptMaterialsInput' = \ + CryptoMaterialsManagerFromMPL._create_mpl_decrypt_materials_input_from_request(request) + mpl_output: 'MPL_DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(mpl_input) + return DecryptionMaterialsFromMPL(mpl_output.decryption_materials) except AwsCryptographicMaterialProvidersException as mpl_exception: # Wrap MPL error into the ESDK error type # so customers only have to catch ESDK error types. raise AWSEncryptionSDKClientError(mpl_exception) @staticmethod - def _native_algorithm_id_to_mpl_algorithm_id(native_algorithm_id: str) -> 'AlgorithmSuiteIdESDK': + def _native_algorithm_id_to_mpl_algorithm_id(native_algorithm_id: str) -> 'MPL_AlgorithmSuiteIdESDK': # MPL algorithm suite ID = hexstr(native_algorithm_id) padded to 4 digits post-`x`. - return AlgorithmSuiteIdESDK(f"{native_algorithm_id:#0{6}x}") + return MPL_AlgorithmSuiteIdESDK(f"{native_algorithm_id:#0{6}x}") @staticmethod def _create_mpl_decrypt_materials_input_from_request( request: DecryptionMaterialsRequest - ) -> 'DecryptMaterialsInput': + ) -> 'MPL_DecryptMaterialsInput': key_blob_list: List[Native_EncryptedDataKey] = request.encrypted_data_keys list_edks = [MPL_EncryptedDataKey( key_provider_id=key_blob.key_provider.provider_id, key_provider_info=key_blob.key_provider.key_info, ciphertext=key_blob.encrypted_data_key, ) for key_blob in key_blob_list] - output: DecryptMaterialsInput = DecryptMaterialsInput( - algorithm_suite_id=MPLCMMHandler._native_algorithm_id_to_mpl_algorithm_id( + output: MPL_DecryptMaterialsInput = MPL_DecryptMaterialsInput( + algorithm_suite_id=CryptoMaterialsManagerFromMPL._native_algorithm_id_to_mpl_algorithm_id( request.algorithm.algorithm_id ), - commitment_policy=MPLCMMHandler._native_to_mpl_commmitment_policy( + commitment_policy=CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy( request.commitment_policy ), encrypted_data_keys=list_edks, diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index 1ea2a199d..2bdf3f810 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -21,7 +21,7 @@ def _mpl_algorithm_id_to_native_algorithm_id(mpl_algorithm_id: str) -> int: return int(mpl_algorithm_id, 16) -class MPLEncryptionMaterials(Native_EncryptionMaterials): +class EncryptionMaterialsFromMPL(Native_EncryptionMaterials): """ In instances where encryption materials are be provided by the MPL's `aws_cryptographic_materialproviders.mpl.models.EncryptionMaterials`, @@ -35,13 +35,13 @@ def __init__( mpl_materials: 'MPL_EncryptionMaterials' ): """ - Create MPLEncryptionMaterials. + Create EncryptionMaterialsFromMPL. :param materials: Underlying encryption materials """ if isinstance(mpl_materials, MPL_EncryptionMaterials): self.mpl_materials = mpl_materials else: - raise ValueError("Invalid EncryptionMaterials passed to MPLEncryptionMaterials. " + raise ValueError("Invalid EncryptionMaterials passed to EncryptionMaterialsFromMPL. " f"materials: {mpl_materials}") @property @@ -91,7 +91,7 @@ def signing_key(self) -> bytes: return self.mpl_materials.signing_key -class MPLDecryptionMaterials(Native_DecryptionMaterials): +class DecryptionMaterialsFromMPL(Native_DecryptionMaterials): """ In instances where decryption materials are be provided by the MPL's `aws_cryptographic_materialproviders.mpl.models.DecryptionMaterials`, @@ -105,13 +105,13 @@ def __init__( mpl_materials: 'MPL_DecryptionMaterials' ): """ - Create MPLDecryptionMaterials. + Create DecryptionMaterialsFromMPL. :param materials: Underlying decryption materials """ if isinstance(mpl_materials, MPL_DecryptionMaterials): self.mpl_materials = mpl_materials else: - raise ValueError(f"Invalid DecryptionMaterials passed to MPLDecryptionMaterials.\ + raise ValueError(f"Invalid DecryptionMaterials passed to DecryptionMaterialsFromMPL.\ materials: {mpl_materials}") @property diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index a3c05bbb7..72ed4efb7 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -80,7 +80,7 @@ _HAS_MPL = True # Import internal ESDK modules that depend on the MPL - from aws_encryption_sdk.materials_managers.mpl.cmm import MPLCMMHandler + from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL except ImportError: _HAS_MPL = False @@ -172,7 +172,7 @@ def _has_mpl_attrs_post_init(self): keyring=self.keyring ) ) - cmm_handler: CryptoMaterialsManager = MPLCMMHandler(cmm) + cmm_handler: CryptoMaterialsManager = CryptoMaterialsManagerFromMPL(cmm) self.materials_manager = cmm_handler def _no_mpl_attrs_post_init(self): @@ -555,7 +555,7 @@ def _prep_message(self): # MPL verification key is PEM bytes, not DER bytes. # If the underlying CMM is from the MPL, load PEM bytes. if (_HAS_MPL - and isinstance(self.config.materials_manager, MPLCMMHandler)): + and isinstance(self.config.materials_manager, CryptoMaterialsManagerFromMPL)): self.signer = Signer.from_key_bytes( algorithm=self._encryption_materials.algorithm, key_bytes=self._encryption_materials.signing_key, encoding=serialization.Encoding.PEM, @@ -923,7 +923,7 @@ def _read_header(self): # MPL verification key is NOT key bytes; it is bytes of the compressed point. # If the underlying CMM is from the MPL, load bytes from encoded point. if (_HAS_MPL - and isinstance(self.config.materials_manager, MPLCMMHandler)): + and isinstance(self.config.materials_manager, CryptoMaterialsManagerFromMPL)): self.verifier = Verifier.from_encoded_point( algorithm=header.algorithm, encoded_point=base64.b64encode(decryption_materials.verification_key) diff --git a/test/mpl/unit/test_material_managers_mpl_cmm.py b/test/mpl/unit/test_material_managers_mpl_cmm.py index cae334722..22f8bf63e 100644 --- a/test/mpl/unit/test_material_managers_mpl_cmm.py +++ b/test/mpl/unit/test_material_managers_mpl_cmm.py @@ -17,11 +17,10 @@ from aws_encryption_sdk.identifiers import CommitmentPolicy -import aws_encryption_sdk.materials_managers.mpl.cmm -from aws_encryption_sdk.materials_managers.mpl.cmm import MPLCMMHandler +from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL from aws_encryption_sdk.materials_managers.mpl.materials import ( - MPLEncryptionMaterials, - MPLDecryptionMaterials, + EncryptionMaterialsFromMPL, + DecryptionMaterialsFromMPL, ) pytestmark = [pytest.mark.unit, pytest.mark.local] @@ -51,180 +50,234 @@ EncryptionMaterialsRequest, DecryptionMaterialsRequest, ) +from aws_encryption_sdk.structures import EncryptedDataKey as Native_EncryptedDataKey mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) -mock_encryption_materials_handler = MagicMock(__class__=MPLEncryptionMaterials) +mock_encryption_materials_handler = MagicMock(__class__=EncryptionMaterialsFromMPL) mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) +mock_edk = MagicMock(__class__=Native_EncryptedDataKey) +mock_mpl_key_provider_id = MagicMock(__class__=str) +mock_edk.key_provider.provider_id = mock_mpl_key_provider_id +mock_mpl_key_provider_info = MagicMock(__class__=bytes) +mock_edk.key_provider.key_info = mock_mpl_key_provider_info +mock_mpl_encrypted_data_key = MagicMock(__class__=bytes) +mock_edk.encrypted_data_key = mock_mpl_encrypted_data_key -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_valid_mpl_cmm_THEN_return_new_MPLCMMHandler(): - mpl_cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) - - assert mpl_cmm_handler.mpl_cmm == mock_mpl_cmm + +def test_GIVEN_valid_mpl_cmm_WHEN_create_CryptoMaterialsManagerFromMPL_THEN_return_new_CryptoMaterialsManagerFromMPL(): + # Given: valid mpl_cmm + # When: create new CryptoMaterialsManagerFromMPL + mpl_cmm = CryptoMaterialsManagerFromMPL(mpl_cmm=mock_mpl_cmm) + # Then: CryptoMaterialsManagerFromMPL is valid + assert mpl_cmm.mpl_cmm == mock_mpl_cmm -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_invalid_mpl_cmm_THEN_raise_ValueError(): +def test_GIVEN_invalid_mpl_cmm_WHEN_create_CryptoMaterialsManagerFromMPL_THEN_raise_ValueError(): + # Then: raises ValueError with pytest.raises(ValueError): - MPLCMMHandler(mpl_cmm="not a valid mpl_cmm") + # Given: invalid mpl_cmm + # When: create new CryptoMaterialsManagerFromMPL + CryptoMaterialsManagerFromMPL(mpl_cmm="not a valid mpl_cmm") @patch.object(mock_mpl_cmm, "get_encryption_materials") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_get_encryption_materials") -def test_GIVEN_valid_request_WHEN_call_get_encryption_materials_THEN_return_MPLEncryptionMaterials( +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._native_to_mpl_get_encryption_materials") +def test_GIVEN_valid_request_WHEN_get_encryption_materials_THEN_return_EncryptionMaterialsFromMPL( mock_native_to_mpl_get_encryption_materials, mock_get_encryption_materials, ): - # Mock: mpl_cmm.get_encryption_materials returns mock MPL encryption materials + # Given: _native_to_mpl_get_encryption_materials creates a GetEncryptionMaterialsInput + mock_get_encryption_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) + mock_native_to_mpl_get_encryption_materials.return_value = mock_get_encryption_materials_input + + # Given: mpl_cmm.get_encryption_materials returns mock MPL encryption materials mock_get_encryption_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) mock_get_encryption_materials_output.encryption_materials = mock_mpl_encryption_materials mock_get_encryption_materials.return_value = mock_get_encryption_materials_output - # Mock: CMMHandler._native_to_mpl_get_encryption_materials creates a GetEncryptionMaterialsInput - mock_get_encryption_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) - mock_native_to_mpl_get_encryption_materials.return_value = mock_get_encryption_materials_input - - cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) - test = cmm_handler.get_encryption_materials(mock_encryption_materials_request) + # When: get_encryption_materials + cmm = CryptoMaterialsManagerFromMPL(mpl_cmm=mock_mpl_cmm) + output = cmm.get_encryption_materials(mock_encryption_materials_request) - # Verify cmm_handler returns MPLEncryptionMaterials - assert isinstance(test, MPLEncryptionMaterials) + # Then: + # Verify cmm returns EncryptionMaterialsFromMPL + assert isinstance(output, EncryptionMaterialsFromMPL) # Verify returned EncryptionMaterialsHandler uses the output of `get_encryption_materials` - assert test.mpl_materials == mock_mpl_encryption_materials + assert output.mpl_materials == mock_mpl_encryption_materials # Verify we actually called `get_encryption_materials` mock_mpl_cmm.get_encryption_materials.assert_called_once_with(mock_get_encryption_materials_input) -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") -def test_GIVEN_get_encryption_materials_raises_MPL_Exception_WHEN_call_get_encryption_materials_THEN_raise_ESDK_Exception( +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy") +def test_GIVEN_mpl_cmm_raises_MPLException_WHEN_get_encryption_materials_THEN_raise_ESDKException( _ ): + # Then: Raises AWSEncryptionSDKClientError with pytest.raises(AWSEncryptionSDKClientError): + # Given: mpl_cmm.get_encryption_materials raises MPL exception with patch.object(mock_mpl_cmm, "get_encryption_materials", side_effect=AwsCryptographicMaterialProvidersException("any")): - - cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) - cmm_handler.get_encryption_materials(mock_encryption_materials_request) + # When: get_encryption_materials + cmm = CryptoMaterialsManagerFromMPL(mpl_cmm=mock_mpl_cmm) + cmm.get_encryption_materials(mock_encryption_materials_request) -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") -def test_GIVEN_native_to_mpl_commmitment_policy_returns_valid_policy_WHEN_call_native_to_mpl_get_encryption_materials_THEN_returns_GetEncryptionMaterialsInput( +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy") +def test_GIVEN_valid_mpl_commitment_policy_WHEN_native_to_mpl_get_encryption_materials_THEN_returns_GetEncryptionMaterialsInput( mock_mpl_commitment_policy ): + # Given: commitment policy is some MPL ESDK commitment policy mock_commitment_policy = MagicMock(__class__=CommitmentPolicyESDK) mock_mpl_commitment_policy.return_value = mock_commitment_policy - output = MPLCMMHandler._native_to_mpl_get_encryption_materials(mock_encryption_materials_request) + # When: _native_to_mpl_get_encryption_materials + output = CryptoMaterialsManagerFromMPL._native_to_mpl_get_encryption_materials(mock_encryption_materials_request) - # verify correctness of returned value + # Then: returned GetEncryptionMaterialsInput is correct assert isinstance(output, GetEncryptionMaterialsInput) assert output.encryption_context == mock_encryption_materials_request.encryption_context assert output.commitment_policy == mock_commitment_policy assert output.max_plaintext_length == mock_encryption_materials_request.plaintext_length -def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_FORBID_ENCRYPT_ALLOW_DECRYPT(): +def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_CommitmentPolicyESDK_FORBID_ENCRYPT_ALLOW_DECRYPT(): + # Given: native FORBID_ENCRYPT_ALLOW_DECRYPT native_commitment_policy = CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT - output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) + # When: _native_to_mpl_commmitment_policy + output = CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy(native_commitment_policy) + # Then: Returns MPL FORBID_ENCRYPT_ALLOW_DECRYPT assert isinstance(output, CommitmentPolicyESDK) assert output.value == "FORBID_ENCRYPT_ALLOW_DECRYPT" -def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_ALLOW_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_REQUIRE_ENCRYPT_ALLOW_DECRYPT(): +def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_CommitmentPolicyESDK_REQUIRE_ENCRYPT_ALLOW_DECRYPT(): + # Given: native REQUIRE_ENCRYPT_ALLOW_DECRYPT native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT - output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) + # When: _native_to_mpl_commmitment_policy + output = CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy(native_commitment_policy) + # Then: Returns MPL REQUIRE_ENCRYPT_ALLOW_DECRYPT assert isinstance(output, CommitmentPolicyESDK) assert output.value == "REQUIRE_ENCRYPT_ALLOW_DECRYPT" -def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_REQUIRE_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_REQUIRE_ENCRYPT_REQUIRE_DECRYPT(): +def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_REQUIRE_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_CommitmentPolicyESDK_REQUIRE_ENCRYPT_REQUIRE_DECRYPT(): + # Given: native REQUIRE_ENCRYPT_REQUIRE_DECRYPT native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT - output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) + # When: _native_to_mpl_commmitment_policy + output = CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy(native_commitment_policy) + # Then: Returns MPL REQUIRE_ENCRYPT_REQUIRE_DECRYPT assert isinstance(output, CommitmentPolicyESDK) assert output.value == "REQUIRE_ENCRYPT_REQUIRE_DECRYPT" -def test_GIVEN_CommitmentPolicy_unrecognized_WHEN_call_native_to_mpl_commmitment_policyTHEN_raise_ValueError(): +def test_GIVEN_CommitmentPolicy_unrecognized_WHEN_native_to_mpl_commmitment_policy_THEN_raise_ValueError(): + # Given: invalid native commitment policy native_commitment_policy = "not a commitment policy" + # Then: Raises ValueError with pytest.raises(ValueError): - MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) + # When: _native_to_mpl_commmitment_policy + CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy(native_commitment_policy) @patch.object(mock_mpl_cmm, "decrypt_materials") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._create_mpl_decrypt_materials_input_from_request") -def test_GIVEN_valid_request_WHEN_call_decrypt_materials_THEN_return_MPLDecryptionMaterials( +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._create_mpl_decrypt_materials_input_from_request") +def test_GIVEN_valid_request_WHEN_decrypt_materials_THEN_return_DecryptionMaterialsFromMPL( mock_native_to_mpl_decrypt_materials, mock_get_encryption_materials, ): - # Mock: mpl_cmm.get_decryption_materials returns mock MPL decryption materials + # Given: mpl_cmm.get_decryption_materials returns mock MPL decryption materials mock_decrypt_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) mock_decrypt_materials_output.decryption_materials = mock_mpl_decrypt_materials mock_get_encryption_materials.return_value = mock_decrypt_materials_output - # Mock: CMMHandler._create_mpl_decrypt_materials_input_from_request creates a DecryptMaterialsInput + # Given: CMMHandler._create_mpl_decrypt_materials_input_from_request creates a DecryptMaterialsInput mock_decrypt_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) mock_native_to_mpl_decrypt_materials.return_value = mock_decrypt_materials_input - cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) - output = cmm_handler.decrypt_materials(mock_decryption_materials_request) + # When: decrypt_materials + cmm = CryptoMaterialsManagerFromMPL(mpl_cmm=mock_mpl_cmm) + output = cmm.decrypt_materials(mock_decryption_materials_request) - # Verify cmm_handler returns MPLDecryptionMaterials - assert isinstance(output, MPLDecryptionMaterials) - # Verify returned MPLDecryptionMaterials uses the output of `decrypt_materials` + # Then: + # Verify cmm returns DecryptionMaterialsFromMPL + assert isinstance(output, DecryptionMaterialsFromMPL) + # Verify returned DecryptionMaterialsFromMPL uses the output of `decrypt_materials` assert output.mpl_materials == mock_mpl_decrypt_materials # Verify we actually called `decrypt_materials` mock_mpl_cmm.decrypt_materials.assert_called_once_with(mock_decrypt_materials_input) -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._create_mpl_decrypt_materials_input_from_request") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._create_mpl_decrypt_materials_input_from_request") def test_GIVEN_decrypt_materials_raises_MPL_Exception_WHEN_call_decrypt_materials_THEN_raise_ESDK_Exception( _ ): + # Then: Raises AWSEncryptionSDKClientError with pytest.raises(AWSEncryptionSDKClientError): + # Given: mpl_cmm.decrypt_materials raises MPL exception with patch.object(mock_mpl_cmm, "decrypt_materials", side_effect=AwsCryptographicMaterialProvidersException("any")): - - cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) - cmm_handler.decrypt_materials(mock_decryption_materials_request) + # When: decrypt_materials + cmm = CryptoMaterialsManagerFromMPL(mpl_cmm=mock_mpl_cmm) + cmm.decrypt_materials(mock_decryption_materials_request) -def test_WHEN_call_native_algorithm_id_to_mpl_algorithm_id_THEN_returns_valid_AlgorithmSuiteIdESDK(): - some_native_algorithm_id = 0x0000 # Not a real algorithm ID, but fits the format +def test_GIVEN_valid_native_algorithm_id_WHEN_native_algorithm_id_to_mpl_algorithm_id_THEN_returns_valid_AlgorithmSuiteIdESDK(): + # Given: any native algorithm ID + some_native_algorithm_id = 0x1234 # Not a real algorithm ID, but fits the format - mpl_output = MPLCMMHandler._native_algorithm_id_to_mpl_algorithm_id( + # When: _native_algorithm_id_to_mpl_algorithm_id + mpl_output = CryptoMaterialsManagerFromMPL._native_algorithm_id_to_mpl_algorithm_id( some_native_algorithm_id ) + # Then: returns valid MPL algorithm ID assert isinstance(mpl_output, AlgorithmSuiteIdESDK) - assert mpl_output.value == "0x0000" + assert mpl_output.value == "0x1234" -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_algorithm_id_to_mpl_algorithm_id") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") -def test__create_mpl_decrypt_materials_input_from_request( +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._native_algorithm_id_to_mpl_algorithm_id") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy") +def test_GIVEN_valid_request_WHEN_create_mpl_decrypt_materials_input_from_request_THEN_returns_MPL_DecryptMaterialsInput( mock_mpl_commitment_policy, mock_mpl_algorithm_id, ): + # Given: _native_algorithm_id_to_mpl_algorithm_id returns a valid MPL algorithm ID mock_algorithm_id = "0x1234" # Some fake algorithm ID that fits the format mock_mpl_algorithm_id.return_value = mock_algorithm_id + + # Given: _native_to_mpl_commmitment_policy returns some MPL commitment policy mock_commitment_policy = MagicMock(__class__=CommitmentPolicyESDK) mock_mpl_commitment_policy.return_value = mock_commitment_policy - # mock_decryption_materials_request.algorithm = - - output = MPLCMMHandler._create_mpl_decrypt_materials_input_from_request(mock_decryption_materials_request) - - assert isinstance(output, DecryptMaterialsInput) - assert output.algorithm_suite_id == mock_algorithm_id - assert output.commitment_policy == mock_commitment_policy - assert output.encryption_context == mock_decryption_materials_request.encryption_context - - assert len(output.encrypted_data_keys) == len(mock_decryption_materials_request.encrypted_data_keys) - for i in range(len(output.encrypted_data_keys)): - # Assume input[i] == output[i], seems to work - output_edk = output.encrypted_data_keys[i] - input_edk = mock_decryption_materials_request[i] - assert output_edk.key_provider_id == input_edk.key_provider.provider_id - assert output_edk.key_provider_info == input_edk.key_provider.key_info - assert output_edk.ciphertext == input_edk.encrypted_data_key + no_mock_edks = [ mock_edk ] + one_mock_edk = [ mock_edk ] + two_mock_edks = [ mock_edk, mock_edk ] + + # Given: ESK lists of various lengths + for mock_edks in [ no_mock_edks, one_mock_edk, two_mock_edks ]: + + mock_decryption_materials_request.encrypted_data_keys = mock_edks + + # When: _create_mpl_decrypt_materials_input_from_request + output = CryptoMaterialsManagerFromMPL._create_mpl_decrypt_materials_input_from_request(mock_decryption_materials_request) + + # Then: + # Verify general correctness of output structure + assert isinstance(output, DecryptMaterialsInput) + assert output.algorithm_suite_id == mock_algorithm_id + assert output.commitment_policy == mock_commitment_policy + assert output.encryption_context == mock_decryption_materials_request.encryption_context + + assert len(output.encrypted_data_keys) == len(mock_edks) + for i in range(len(output.encrypted_data_keys)): + # Assume input[i] == output[i] to make validation easier + # This is how the src is implemented but is not a requirement. + # If this assumption breaks, we should enhance this test. + output_edk = output.encrypted_data_keys[i] + input_edk = mock_edks[i] + assert output_edk.key_provider_id == input_edk.key_provider.provider_id + assert output_edk.key_provider_info == input_edk.key_provider.key_info + assert output_edk.ciphertext == input_edk.encrypted_data_key \ No newline at end of file diff --git a/test/mpl/unit/test_material_managers_mpl_materials.py b/test/mpl/unit/test_material_managers_mpl_materials.py index bb83b89fd..b39e9bc8d 100644 --- a/test/mpl/unit/test_material_managers_mpl_materials.py +++ b/test/mpl/unit/test_material_managers_mpl_materials.py @@ -14,13 +14,12 @@ import pytest from mock import MagicMock, patch, PropertyMock -from typing import Dict, List +from typing import Dict, List, Set -from aws_encryption_sdk.identifiers import CommitmentPolicy import aws_encryption_sdk.materials_managers.mpl.materials from aws_encryption_sdk.materials_managers.mpl.materials import ( - MPLEncryptionMaterials, - MPLDecryptionMaterials, + EncryptionMaterialsFromMPL, + DecryptionMaterialsFromMPL, ) from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite @@ -54,124 +53,164 @@ mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) -mock_encryption_materials_handler = MagicMock(__class__=MPLEncryptionMaterials) +mock_encryption_materials_handler = MagicMock(__class__=EncryptionMaterialsFromMPL) mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) +mock_edk = MagicMock(__class__=MPL_EncryptedDataKey) +mock_mpl_key_provider_id = MagicMock(__class__=str) +mock_edk.key_provider_id = mock_mpl_key_provider_id +mock_mpl_key_provider_info = MagicMock(__class__=bytes) +mock_edk.key_provider_info = mock_mpl_key_provider_info +mock_mpl_ciphertext = MagicMock(__class__=bytes) +mock_edk.ciphertext = mock_mpl_ciphertext -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_valid_mpl_cmm_THEN_return_new_MPLCMMHandler(): - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + +def test_GIVEN_valid_mpl_materials_WHEN_create_EncryptionMaterialsFromMPL_THEN_return_new_CryptoMaterialsManagerFromMPL(): + # Given: valid mpl_materials + # When: create EncryptionMaterialsFromMPL + mpl_encryption_materials = EncryptionMaterialsFromMPL(mpl_materials=mock_mpl_encryption_materials) + # Then: EncryptionMaterialsFromMPL is valid assert mpl_encryption_materials.mpl_materials == mock_mpl_encryption_materials -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_invalid_mpl_cmm_THEN_raise_ValueError(): +def test_GIVEN_invalid_mpl_materials_WHEN_create_EncryptionMaterialsFromMPL_THEN_raise_ValueError(): + # Then: Raise ValueError with pytest.raises(ValueError): - MPLEncryptionMaterials(mpl_materials="not a valid mpl_materials") + # Given: invalid mpl_materials + # When: create EncryptionMaterialsFromMPL + EncryptionMaterialsFromMPL(mpl_materials="not a valid mpl_materials") + -def test_mpl_to_native(): +def test_GIVEN_valid_mpl_algorithm_id_WHEN_mpl_algorithm_id_to_native_algorithm_id_THEN_valid_native_output(): + # Given: any valid MPL algorithm ID some_mpl_algorithm_id = "0x1234" # Not a real algorithm ID, but fits the format + # When: _mpl_algorithm_id_to_native_algorithm_id native_output = aws_encryption_sdk.materials_managers.mpl.materials._mpl_algorithm_id_to_native_algorithm_id( some_mpl_algorithm_id ) + # Then: valid native algorithm ID assert native_output == 0x1234 @patch("aws_encryption_sdk.materials_managers.mpl.materials._mpl_algorithm_id_to_native_algorithm_id") @patch("aws_encryption_sdk.materials_managers.mpl.materials.AlgorithmSuite.get_by_id") -def test_GIVEN_valid_mpl_algorithm_id_WHEN_get_algorithm_THEN_valid_native_algorithm_id( +def test_GIVEN_valid_mpl_algorithm_id_WHEN_EncryptionMaterials_get_algorithm_THEN_valid_native_algorithm_id( mock_algorithm, mock_native_algorithm_id, ): - # Mock valid conversion from MPL to native algorithm ID + # Given: _mpl_algorithm_id_to_native_algorithm_id returns a valid native algorithm ID mock_native_algorithm_id.return_value = 0x1234 - # Mock valid lookup in native AlgorithmSuite lookup + # Given: get_by_id returns a valid native AlgorithmSuite by looking up an ID mock_algorithm.return_value = MagicMock(__class__=AlgorithmSuite) - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + # When: Get algorithm + mpl_encryption_materials = EncryptionMaterialsFromMPL(mpl_materials=mock_mpl_encryption_materials) output = mpl_encryption_materials.algorithm + + # Then: output is valid assert output == mock_algorithm() # property calls automatically, we need to call the mock -def test_GecTHEN_valid_native_algorithm_id(): +def test_GIVEN_valid_encryption_context_WHEN_EncryptionMaterials_get_encryption_context_THEN_valid_encryption_context(): + # Given: valid encryption context mock_encryption_context = MagicMock(__class__=Dict[str, str]) mock_mpl_encryption_materials.encryption_context = mock_encryption_context - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + # When: get encryption context + mpl_encryption_materials = EncryptionMaterialsFromMPL(mpl_materials=mock_mpl_encryption_materials) output = mpl_encryption_materials.encryption_context + # Then: returns valid encryption context assert output == mock_encryption_context -def test_GecTHEN_valid_nativefadsf_algorithm_id(): - mock_edk = MagicMock(__class__=MPL_EncryptedDataKey) - mock_mpl_key_provider_id = MagicMock(__class__=str) - mock_edk.key_provider_id = mock_mpl_key_provider_id - mock_mpl_key_provider_info = MagicMock(__class__=bytes) - mock_edk.key_provider_info = mock_mpl_key_provider_info - mock_mpl_ciphertext = MagicMock(__class__=bytes) - mock_edk.ciphertext = mock_mpl_ciphertext - - mock_edks = [ mock_edk ] - mock_mpl_encryption_materials.encrypted_data_keys = mock_edks - - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) - output = mpl_encryption_materials.encrypted_data_keys - output_as_list = list(output) - - assert len(output_as_list) == len(mock_edks) - for i in range(len(output_as_list)): - # assume output[i] corresponds to input[i] - native_edk = output_as_list[i] - mpl_edk = mock_edks[i] - - assert native_edk.encrypted_data_key == mpl_edk.ciphertext - assert native_edk.key_provider.provider_id == mpl_edk.key_provider_id - assert native_edk.key_provider.key_info == mpl_edk.key_provider_info - -def test_GecTHEN_valid_nativefadsffadsfa_algorithm_id(): +def test_GIVEN_valid_edks_WHEN_EncryptionMaterials_get_edks_THEN_returns_edks(): + + # Given: lists of mocked EDKs of various lengths + no_mock_edks = [] + one_mock_edk = [ mock_edk ] + two_mocked_edks = [ mock_edk, mock_edk ] + for mock_edks in [ no_mock_edks, one_mock_edk, two_mocked_edks ]: + mock_mpl_encryption_materials.encrypted_data_keys = mock_edks + + # When: get EDKs + mpl_encryption_materials = EncryptionMaterialsFromMPL(mpl_materials=mock_mpl_encryption_materials) + output = mpl_encryption_materials.encrypted_data_keys + + # Then: returns EDKs + output_as_list = list(output) + # Native ESDK Python types the EDKs as a set; + # Ensure the MPL's list is collapsed into a set correctly + assert len(output_as_list) == len(set(mock_edks)) + for i in range(len(output_as_list)): + # Assume input[i] == output[i] to make validation easier + # This is how the src is implemented but is not a requirement. + # If this assumption breaks, we should enhance this test. + native_edk = output_as_list[i] + mpl_edk = mock_edks[i] + + assert native_edk.encrypted_data_key == mpl_edk.ciphertext + assert native_edk.key_provider.provider_id == mpl_edk.key_provider_id + assert native_edk.key_provider.key_info == mpl_edk.key_provider_info + + +def test_GIVEN_valid_data_key_WHEN_EncryptionMaterials_get_data_key_THEN_returns_data_key(): + # Given: Valid MPL data key mock_data_key = MagicMock(__class__=bytes) mock_mpl_encryption_materials.plaintext_data_key = mock_data_key - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + # When: get data key + mpl_encryption_materials = EncryptionMaterialsFromMPL(mpl_materials=mock_mpl_encryption_materials) output = mpl_encryption_materials.data_encryption_key + # Then: Returns native data key assert output.key_provider.provider_id == "" assert output.key_provider.key_info == b"" assert output.data_key == mock_data_key assert output.encrypted_data_key == b"" -def test_GecTHEN_valid_nativefasdfasdffadsf_algorithm_id(): +def test_GIVEN_valid_signing_key_WHEN_EncryptionMaterials_get_signing_key_THEN_returns_signing_key(): + # Given: valid signing key mock_signing_key = MagicMock(__class__=bytes) mock_mpl_encryption_materials.signing_key = mock_signing_key - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + # When: get signing key + mpl_encryption_materials = EncryptionMaterialsFromMPL(mpl_materials=mock_mpl_encryption_materials) output = mpl_encryption_materials.signing_key + # Then: returns signing key assert output == mock_signing_key -def test_GecTHEN_valid_nativeffasdfasdadsffadsfa_algorithm_id(): +def test_GIVEN_valid_data_key_WHEN_DecryptionMaterials_get_data_key_THEN_returns_data_key(): + # Given: valid MPL data key mock_data_key = MagicMock(__class__=bytes) mock_mpl_decrypt_materials.plaintext_data_key = mock_data_key - mpl_decryption_materials = MPLDecryptionMaterials(mpl_materials=mock_mpl_decrypt_materials) + # When: get data key + mpl_decryption_materials = DecryptionMaterialsFromMPL(mpl_materials=mock_mpl_decrypt_materials) output = mpl_decryption_materials.data_key + # Then: returns valid native data key assert output.key_provider.provider_id == "" assert output.key_provider.key_info == b"" assert output.data_key == mock_data_key assert output.encrypted_data_key == b"" -def test_GecTHEN_validadsfasdf_nativefasdfasdffadsf_algorithm_id(): +def test_GIVEN_valid_verification_key_WHEN_DecryptionMaterials_get_verification_key_THEN_returns_verification_key(): + # Given: valid verification key mock_verification_key = MagicMock(__class__=bytes) mock_mpl_decrypt_materials.verification_key = mock_verification_key - mpl_decryption_materials = MPLDecryptionMaterials(mpl_materials=mock_mpl_decrypt_materials) + # When: get verification key + mpl_decryption_materials = DecryptionMaterialsFromMPL(mpl_materials=mock_mpl_decrypt_materials) output = mpl_decryption_materials.verification_key + # Then: returns verification key assert output == mock_verification_key From ac6471a921407df409bceb9dc1dafbb5e697544c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 17:00:31 -0800 Subject: [PATCH 118/376] test cleanup --- examples/src/keyrings/hierarchical_keyring.py | 6 +- .../unit/test_material_managers_mpl_cmm.py | 56 +++++++++---------- .../test_material_managers_mpl_materials.py | 11 ---- 3 files changed, 31 insertions(+), 42 deletions(-) diff --git a/examples/src/keyrings/hierarchical_keyring.py b/examples/src/keyrings/hierarchical_keyring.py index c71719346..aa87485f9 100644 --- a/examples/src/keyrings/hierarchical_keyring.py +++ b/examples/src/keyrings/hierarchical_keyring.py @@ -4,12 +4,12 @@ import sys import boto3 -# ignore missing MPL for pylint, but the MPL is required for this example +# Ignore missing MPL for pylint, but the MPL is required for this example # noqa pylint: disable=import-error -from aws_cryptographic_materialproviders.keystore.client import KeyStore +from aws_cryptographic_materialproviders.keystore import KeyStore from aws_cryptographic_materialproviders.keystore.config import KeyStoreConfig from aws_cryptographic_materialproviders.keystore.models import CreateKeyInput, KMSConfigurationKmsKeyArn -from aws_cryptographic_materialproviders.mpl.client import AwsCryptographicMaterialProviders +from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig from aws_cryptographic_materialproviders.mpl.models import ( CacheTypeDefault, diff --git a/test/mpl/unit/test_material_managers_mpl_cmm.py b/test/mpl/unit/test_material_managers_mpl_cmm.py index 22f8bf63e..52a4b333c 100644 --- a/test/mpl/unit/test_material_managers_mpl_cmm.py +++ b/test/mpl/unit/test_material_managers_mpl_cmm.py @@ -28,19 +28,19 @@ from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException from aws_cryptographic_materialproviders.mpl.models import ( - AlgorithmSuiteIdESDK, - CommitmentPolicyESDK, - DecryptMaterialsInput, + AlgorithmSuiteIdESDK as MPL_AlgorithmSuiteIdESDK, + CommitmentPolicyESDK as MPL_CommitmentPolicyESDK, + DecryptMaterialsInput as MPL_DecryptMaterialsInput, DecryptionMaterials as MPL_DecryptionMaterials, EncryptionMaterials as MPL_EncryptionMaterials, - GetEncryptionMaterialsInput, - GetEncryptionMaterialsOutput, + GetEncryptionMaterialsInput as MPL_GetEncryptionMaterialsInput, + GetEncryptionMaterialsOutput as MPL_GetEncryptionMaterialsOutput, ) from aws_cryptographic_materialproviders.mpl.references import ( - ICryptographicMaterialsManager + ICryptographicMaterialsManager as MPL_ICryptographicMaterialsManager, ) -mock_mpl_cmm = MagicMock(__class__=ICryptographicMaterialsManager) +mock_mpl_cmm = MagicMock(__class__=MPL_ICryptographicMaterialsManager) mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) @@ -89,12 +89,12 @@ def test_GIVEN_valid_request_WHEN_get_encryption_materials_THEN_return_Encryptio mock_get_encryption_materials, ): - # Given: _native_to_mpl_get_encryption_materials creates a GetEncryptionMaterialsInput - mock_get_encryption_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) + # Given: _native_to_mpl_get_encryption_materials creates a MPL_GetEncryptionMaterialsInput + mock_get_encryption_materials_input = MagicMock(__class__=MPL_GetEncryptionMaterialsInput) mock_native_to_mpl_get_encryption_materials.return_value = mock_get_encryption_materials_input # Given: mpl_cmm.get_encryption_materials returns mock MPL encryption materials - mock_get_encryption_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) + mock_get_encryption_materials_output = MagicMock(__class__=MPL_GetEncryptionMaterialsOutput) mock_get_encryption_materials_output.encryption_materials = mock_mpl_encryption_materials mock_get_encryption_materials.return_value = mock_get_encryption_materials_output @@ -125,24 +125,24 @@ def test_GIVEN_mpl_cmm_raises_MPLException_WHEN_get_encryption_materials_THEN_ra cmm.get_encryption_materials(mock_encryption_materials_request) @patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy") -def test_GIVEN_valid_mpl_commitment_policy_WHEN_native_to_mpl_get_encryption_materials_THEN_returns_GetEncryptionMaterialsInput( +def test_GIVEN_valid_mpl_commitment_policy_WHEN_native_to_mpl_get_encryption_materials_THEN_returns_MPL_GetEncryptionMaterialsInput( mock_mpl_commitment_policy ): # Given: commitment policy is some MPL ESDK commitment policy - mock_commitment_policy = MagicMock(__class__=CommitmentPolicyESDK) + mock_commitment_policy = MagicMock(__class__=MPL_CommitmentPolicyESDK) mock_mpl_commitment_policy.return_value = mock_commitment_policy # When: _native_to_mpl_get_encryption_materials output = CryptoMaterialsManagerFromMPL._native_to_mpl_get_encryption_materials(mock_encryption_materials_request) - # Then: returned GetEncryptionMaterialsInput is correct - assert isinstance(output, GetEncryptionMaterialsInput) + # Then: returned MPL_GetEncryptionMaterialsInput is correct + assert isinstance(output, MPL_GetEncryptionMaterialsInput) assert output.encryption_context == mock_encryption_materials_request.encryption_context assert output.commitment_policy == mock_commitment_policy assert output.max_plaintext_length == mock_encryption_materials_request.plaintext_length -def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_CommitmentPolicyESDK_FORBID_ENCRYPT_ALLOW_DECRYPT(): +def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_MPL_CommitmentPolicyESDK_FORBID_ENCRYPT_ALLOW_DECRYPT(): # Given: native FORBID_ENCRYPT_ALLOW_DECRYPT native_commitment_policy = CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT @@ -150,10 +150,10 @@ def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_ output = CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy(native_commitment_policy) # Then: Returns MPL FORBID_ENCRYPT_ALLOW_DECRYPT - assert isinstance(output, CommitmentPolicyESDK) + assert isinstance(output, MPL_CommitmentPolicyESDK) assert output.value == "FORBID_ENCRYPT_ALLOW_DECRYPT" -def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_CommitmentPolicyESDK_REQUIRE_ENCRYPT_ALLOW_DECRYPT(): +def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_MPL_CommitmentPolicyESDK_REQUIRE_ENCRYPT_ALLOW_DECRYPT(): # Given: native REQUIRE_ENCRYPT_ALLOW_DECRYPT native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT @@ -161,10 +161,10 @@ def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl output = CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy(native_commitment_policy) # Then: Returns MPL REQUIRE_ENCRYPT_ALLOW_DECRYPT - assert isinstance(output, CommitmentPolicyESDK) + assert isinstance(output, MPL_CommitmentPolicyESDK) assert output.value == "REQUIRE_ENCRYPT_ALLOW_DECRYPT" -def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_REQUIRE_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_CommitmentPolicyESDK_REQUIRE_ENCRYPT_REQUIRE_DECRYPT(): +def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_REQUIRE_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_MPL_CommitmentPolicyESDK_REQUIRE_ENCRYPT_REQUIRE_DECRYPT(): # Given: native REQUIRE_ENCRYPT_REQUIRE_DECRYPT native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT @@ -172,7 +172,7 @@ def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_REQUIRE_DECRYPT_WHEN_native_to_m output = CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy(native_commitment_policy) # Then: Returns MPL REQUIRE_ENCRYPT_REQUIRE_DECRYPT - assert isinstance(output, CommitmentPolicyESDK) + assert isinstance(output, MPL_CommitmentPolicyESDK) assert output.value == "REQUIRE_ENCRYPT_REQUIRE_DECRYPT" def test_GIVEN_CommitmentPolicy_unrecognized_WHEN_native_to_mpl_commmitment_policy_THEN_raise_ValueError(): @@ -192,12 +192,12 @@ def test_GIVEN_valid_request_WHEN_decrypt_materials_THEN_return_DecryptionMateri ): # Given: mpl_cmm.get_decryption_materials returns mock MPL decryption materials - mock_decrypt_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) + mock_decrypt_materials_output = MagicMock(__class__=MPL_GetEncryptionMaterialsOutput) mock_decrypt_materials_output.decryption_materials = mock_mpl_decrypt_materials mock_get_encryption_materials.return_value = mock_decrypt_materials_output - # Given: CMMHandler._create_mpl_decrypt_materials_input_from_request creates a DecryptMaterialsInput - mock_decrypt_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) + # Given: CMMHandler._create_mpl_decrypt_materials_input_from_request creates a MPL_DecryptMaterialsInput + mock_decrypt_materials_input = MagicMock(__class__=MPL_GetEncryptionMaterialsInput) mock_native_to_mpl_decrypt_materials.return_value = mock_decrypt_materials_input # When: decrypt_materials @@ -225,7 +225,7 @@ def test_GIVEN_decrypt_materials_raises_MPL_Exception_WHEN_call_decrypt_material cmm = CryptoMaterialsManagerFromMPL(mpl_cmm=mock_mpl_cmm) cmm.decrypt_materials(mock_decryption_materials_request) -def test_GIVEN_valid_native_algorithm_id_WHEN_native_algorithm_id_to_mpl_algorithm_id_THEN_returns_valid_AlgorithmSuiteIdESDK(): +def test_GIVEN_valid_native_algorithm_id_WHEN_native_algorithm_id_to_mpl_algorithm_id_THEN_returns_valid_MPL_AlgorithmSuiteIdESDK(): # Given: any native algorithm ID some_native_algorithm_id = 0x1234 # Not a real algorithm ID, but fits the format @@ -235,12 +235,12 @@ def test_GIVEN_valid_native_algorithm_id_WHEN_native_algorithm_id_to_mpl_algorit ) # Then: returns valid MPL algorithm ID - assert isinstance(mpl_output, AlgorithmSuiteIdESDK) + assert isinstance(mpl_output, MPL_AlgorithmSuiteIdESDK) assert mpl_output.value == "0x1234" @patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._native_algorithm_id_to_mpl_algorithm_id") @patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy") -def test_GIVEN_valid_request_WHEN_create_mpl_decrypt_materials_input_from_request_THEN_returns_MPL_DecryptMaterialsInput( +def test_GIVEN_valid_request_WHEN_create_mpl_decrypt_materials_input_from_request_THEN_returns_MPL_MPL_DecryptMaterialsInput( mock_mpl_commitment_policy, mock_mpl_algorithm_id, ): @@ -249,7 +249,7 @@ def test_GIVEN_valid_request_WHEN_create_mpl_decrypt_materials_input_from_reques mock_mpl_algorithm_id.return_value = mock_algorithm_id # Given: _native_to_mpl_commmitment_policy returns some MPL commitment policy - mock_commitment_policy = MagicMock(__class__=CommitmentPolicyESDK) + mock_commitment_policy = MagicMock(__class__=MPL_CommitmentPolicyESDK) mock_mpl_commitment_policy.return_value = mock_commitment_policy no_mock_edks = [ mock_edk ] @@ -266,7 +266,7 @@ def test_GIVEN_valid_request_WHEN_create_mpl_decrypt_materials_input_from_reques # Then: # Verify general correctness of output structure - assert isinstance(output, DecryptMaterialsInput) + assert isinstance(output, MPL_DecryptMaterialsInput) assert output.algorithm_suite_id == mock_algorithm_id assert output.commitment_policy == mock_commitment_policy assert output.encryption_context == mock_decryption_materials_request.encryption_context diff --git a/test/mpl/unit/test_material_managers_mpl_materials.py b/test/mpl/unit/test_material_managers_mpl_materials.py index b39e9bc8d..92a8c95df 100644 --- a/test/mpl/unit/test_material_managers_mpl_materials.py +++ b/test/mpl/unit/test_material_managers_mpl_materials.py @@ -26,32 +26,21 @@ pytestmark = [pytest.mark.unit, pytest.mark.local] -from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException from aws_cryptographic_materialproviders.mpl.models import ( - AlgorithmSuiteIdESDK, - CommitmentPolicyESDK, - DecryptMaterialsInput, DecryptionMaterials as MPL_DecryptionMaterials, EncryptedDataKey as MPL_EncryptedDataKey, EncryptionMaterials as MPL_EncryptionMaterials, - GetEncryptionMaterialsInput, - GetEncryptionMaterialsOutput, -) -from aws_cryptographic_materialproviders.mpl.references import ( - ICryptographicMaterialsManager ) mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) -from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError from aws_encryption_sdk.materials_managers import ( EncryptionMaterialsRequest, DecryptionMaterialsRequest, ) - mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) mock_encryption_materials_handler = MagicMock(__class__=EncryptionMaterialsFromMPL) mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) From a5ebc19c479f4b6de3874dc4735ca0e27ffbbc38 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 17:18:50 -0800 Subject: [PATCH 119/376] isort --- .../materials_managers/mpl/cmm.py | 5 ++--- .../materials_managers/mpl/materials.py | 1 - test/mpl/unit/test_material_managers_mpl_cmm.py | 14 +++----------- .../unit/test_material_managers_mpl_materials.py | 12 +++--------- 4 files changed, 8 insertions(+), 24 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index c97c070f0..1bbd7c89a 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -11,16 +11,15 @@ GetEncryptionMaterialsOutput as MPL_GetEncryptionMaterialsOutput, ) from aws_cryptographic_materialproviders.mpl.references import ( - ICryptographicMaterialsManager as MPL_ICryptographicMaterialsManager + ICryptographicMaterialsManager as MPL_ICryptographicMaterialsManager, ) - from typing import List from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError from aws_encryption_sdk.identifiers import CommitmentPolicy -from aws_encryption_sdk.materials_managers.mpl.materials import EncryptionMaterialsFromMPL, DecryptionMaterialsFromMPL from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager +from aws_encryption_sdk.materials_managers.mpl.materials import DecryptionMaterialsFromMPL, EncryptionMaterialsFromMPL from aws_encryption_sdk.structures import EncryptedDataKey as Native_EncryptedDataKey diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index 2bdf3f810..31f7d2a65 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -5,7 +5,6 @@ EncryptedDataKey as MPL_EncryptedDataKey, EncryptionMaterials as MPL_EncryptionMaterials, ) - from typing import Dict, List, Set from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite diff --git a/test/mpl/unit/test_material_managers_mpl_cmm.py b/test/mpl/unit/test_material_managers_mpl_cmm.py index 52a4b333c..eb795b7c2 100644 --- a/test/mpl/unit/test_material_managers_mpl_cmm.py +++ b/test/mpl/unit/test_material_managers_mpl_cmm.py @@ -15,13 +15,9 @@ import pytest from mock import MagicMock, patch - from aws_encryption_sdk.identifiers import CommitmentPolicy from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL -from aws_encryption_sdk.materials_managers.mpl.materials import ( - EncryptionMaterialsFromMPL, - DecryptionMaterialsFromMPL, -) +from aws_encryption_sdk.materials_managers.mpl.materials import DecryptionMaterialsFromMPL, EncryptionMaterialsFromMPL pytestmark = [pytest.mark.unit, pytest.mark.local] @@ -30,8 +26,8 @@ from aws_cryptographic_materialproviders.mpl.models import ( AlgorithmSuiteIdESDK as MPL_AlgorithmSuiteIdESDK, CommitmentPolicyESDK as MPL_CommitmentPolicyESDK, - DecryptMaterialsInput as MPL_DecryptMaterialsInput, DecryptionMaterials as MPL_DecryptionMaterials, + DecryptMaterialsInput as MPL_DecryptMaterialsInput, EncryptionMaterials as MPL_EncryptionMaterials, GetEncryptionMaterialsInput as MPL_GetEncryptionMaterialsInput, GetEncryptionMaterialsOutput as MPL_GetEncryptionMaterialsOutput, @@ -46,13 +42,9 @@ from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError -from aws_encryption_sdk.materials_managers import ( - EncryptionMaterialsRequest, - DecryptionMaterialsRequest, -) +from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest from aws_encryption_sdk.structures import EncryptedDataKey as Native_EncryptedDataKey - mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) mock_encryption_materials_handler = MagicMock(__class__=EncryptionMaterialsFromMPL) mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) diff --git a/test/mpl/unit/test_material_managers_mpl_materials.py b/test/mpl/unit/test_material_managers_mpl_materials.py index 92a8c95df..96237998a 100644 --- a/test/mpl/unit/test_material_managers_mpl_materials.py +++ b/test/mpl/unit/test_material_managers_mpl_materials.py @@ -13,15 +13,12 @@ """Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" import pytest -from mock import MagicMock, patch, PropertyMock +from mock import MagicMock, PropertyMock, patch from typing import Dict, List, Set import aws_encryption_sdk.materials_managers.mpl.materials -from aws_encryption_sdk.materials_managers.mpl.materials import ( - EncryptionMaterialsFromMPL, - DecryptionMaterialsFromMPL, -) from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite +from aws_encryption_sdk.materials_managers.mpl.materials import DecryptionMaterialsFromMPL, EncryptionMaterialsFromMPL pytestmark = [pytest.mark.unit, pytest.mark.local] @@ -36,10 +33,7 @@ mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) -from aws_encryption_sdk.materials_managers import ( - EncryptionMaterialsRequest, - DecryptionMaterialsRequest, -) +from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) mock_encryption_materials_handler = MagicMock(__class__=EncryptionMaterialsFromMPL) From 21f361462ec2542056ffcd25ae08bffdb21a7a8d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 17:21:52 -0800 Subject: [PATCH 120/376] fixes --- test/mpl/__init__.py | 2 +- .../unit/test_material_managers_mpl_cmm.py | 28 +++++++++---------- .../test_material_managers_mpl_materials.py | 15 ++++------ 3 files changed, 20 insertions(+), 25 deletions(-) diff --git a/test/mpl/__init__.py b/test/mpl/__init__.py index b976c1308..2a6c71715 100644 --- a/test/mpl/__init__.py +++ b/test/mpl/__init__.py @@ -10,4 +10,4 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Module containing tests that REQUIRE the aws-cryptographic-material-providers library to run.""" \ No newline at end of file +"""Module containing tests that REQUIRE the aws-cryptographic-material-providers library to run.""" diff --git a/test/mpl/unit/test_material_managers_mpl_cmm.py b/test/mpl/unit/test_material_managers_mpl_cmm.py index eb795b7c2..a67c3e5c5 100644 --- a/test/mpl/unit/test_material_managers_mpl_cmm.py +++ b/test/mpl/unit/test_material_managers_mpl_cmm.py @@ -13,15 +13,6 @@ """Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" import pytest -from mock import MagicMock, patch - -from aws_encryption_sdk.identifiers import CommitmentPolicy -from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL -from aws_encryption_sdk.materials_managers.mpl.materials import DecryptionMaterialsFromMPL, EncryptionMaterialsFromMPL - -pytestmark = [pytest.mark.unit, pytest.mark.local] - - from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException from aws_cryptographic_materialproviders.mpl.models import ( AlgorithmSuiteIdESDK as MPL_AlgorithmSuiteIdESDK, @@ -35,20 +26,27 @@ from aws_cryptographic_materialproviders.mpl.references import ( ICryptographicMaterialsManager as MPL_ICryptographicMaterialsManager, ) - -mock_mpl_cmm = MagicMock(__class__=MPL_ICryptographicMaterialsManager) -mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) -mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) - +from mock import MagicMock, patch from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError +from aws_encryption_sdk.identifiers import CommitmentPolicy from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest +from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL +from aws_encryption_sdk.materials_managers.mpl.materials import DecryptionMaterialsFromMPL, EncryptionMaterialsFromMPL from aws_encryption_sdk.structures import EncryptedDataKey as Native_EncryptedDataKey +pytestmark = [pytest.mark.unit, pytest.mark.local] + + mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) -mock_encryption_materials_handler = MagicMock(__class__=EncryptionMaterialsFromMPL) mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) + +mock_mpl_cmm = MagicMock(__class__=MPL_ICryptographicMaterialsManager) +mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) +mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) + + mock_edk = MagicMock(__class__=Native_EncryptedDataKey) mock_mpl_key_provider_id = MagicMock(__class__=str) mock_edk.key_provider.provider_id = mock_mpl_key_provider_id diff --git a/test/mpl/unit/test_material_managers_mpl_materials.py b/test/mpl/unit/test_material_managers_mpl_materials.py index 96237998a..cb3ca7397 100644 --- a/test/mpl/unit/test_material_managers_mpl_materials.py +++ b/test/mpl/unit/test_material_managers_mpl_materials.py @@ -13,28 +13,25 @@ """Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" import pytest +from aws_cryptographic_materialproviders.mpl.models import ( + DecryptionMaterials as MPL_DecryptionMaterials, + EncryptedDataKey as MPL_EncryptedDataKey, + EncryptionMaterials as MPL_EncryptionMaterials, +) from mock import MagicMock, PropertyMock, patch from typing import Dict, List, Set import aws_encryption_sdk.materials_managers.mpl.materials from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite +from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest from aws_encryption_sdk.materials_managers.mpl.materials import DecryptionMaterialsFromMPL, EncryptionMaterialsFromMPL pytestmark = [pytest.mark.unit, pytest.mark.local] -from aws_cryptographic_materialproviders.mpl.models import ( - DecryptionMaterials as MPL_DecryptionMaterials, - EncryptedDataKey as MPL_EncryptedDataKey, - EncryptionMaterials as MPL_EncryptionMaterials, -) - mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) - -from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest - mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) mock_encryption_materials_handler = MagicMock(__class__=EncryptionMaterialsFromMPL) mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) From 22eabb64dec94e8eaf6ccfb7a3fb14a29fcd09eb Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 17:27:38 -0800 Subject: [PATCH 121/376] fix --- decrypt_oracle/.chalice/pipeline.py | 2 +- .../src/aws_encryption_sdk_decrypt_oracle/app.py | 3 ++- .../test/integration/integration_test_utils.py | 3 ++- decrypt_oracle/test/test_n_generate_test_vectors.py | 7 ++++--- examples/test/examples_test_utils.py | 2 +- examples/test/test_i_basic_encryption.py | 1 - ...basic_file_encryption_with_multiple_providers.py | 4 +--- ...i_basic_file_encryption_with_raw_key_provider.py | 1 - examples/test/test_i_data_key_caching_basic.py | 1 - examples/test/test_i_discovery_kms_provider.py | 4 +--- examples/test/test_i_mrk_aware_kms_provider.py | 4 +--- examples/test/test_i_multiple_kms_cmk.py | 4 +--- examples/test/test_i_one_kms_cmk.py | 4 +--- examples/test/test_i_one_kms_cmk_streaming_data.py | 1 - examples/test/test_i_one_kms_cmk_unsigned.py | 4 +--- examples/test/test_i_set_commitment.py | 4 +--- .../materials_managers/mpl/__init__.py | 5 ++++- .../materials_managers/mpl/cmm.py | 12 ++++++++---- .../materials_managers/mpl/materials.py | 5 ++++- src/aws_encryption_sdk/streaming_client.py | 4 ++-- test/mpl/__init__.py | 5 ++++- test/mpl/unit/test_material_managers_mpl_cmm.py | 5 ++++- .../unit/test_material_managers_mpl_materials.py | 7 +++++-- .../src/awses_test_vectors/internal/aws_kms.py | 5 +++-- .../src/awses_test_vectors/internal/util.py | 3 +-- .../manifests/full_message/decrypt.py | 9 ++++----- .../manifests/full_message/decrypt_generation.py | 13 ++++++------- .../manifests/full_message/encrypt.py | 7 +++---- .../src/awses_test_vectors/manifests/keys.py | 4 +--- .../src/awses_test_vectors/manifests/master_key.py | 11 +++++------ .../commands/test_i_full_message_encrypt.py | 1 - 31 files changed, 71 insertions(+), 74 deletions(-) diff --git a/decrypt_oracle/.chalice/pipeline.py b/decrypt_oracle/.chalice/pipeline.py index 9d5573646..c05df6739 100644 --- a/decrypt_oracle/.chalice/pipeline.py +++ b/decrypt_oracle/.chalice/pipeline.py @@ -2,7 +2,6 @@ import argparse import getpass import logging -from typing import Iterable import boto3 import troposphere @@ -20,6 +19,7 @@ ) from botocore.exceptions import ClientError from troposphere import GetAtt, Ref, Sub, Template, codebuild, codepipeline, iam, s3 +from typing import Iterable APPLICATION_NAME = "AwsEncryptionSdkDecryptOraclePython" PIPELINE_STACK_NAME = "{}DeployPipeline".format(APPLICATION_NAME) diff --git a/decrypt_oracle/src/aws_encryption_sdk_decrypt_oracle/app.py b/decrypt_oracle/src/aws_encryption_sdk_decrypt_oracle/app.py index 820b9e015..e250bb3c8 100644 --- a/decrypt_oracle/src/aws_encryption_sdk_decrypt_oracle/app.py +++ b/decrypt_oracle/src/aws_encryption_sdk_decrypt_oracle/app.py @@ -15,10 +15,11 @@ import logging import os +from chalice import Chalice, Response + import aws_encryption_sdk from aws_encryption_sdk.identifiers import CommitmentPolicy from aws_encryption_sdk.key_providers.kms import DiscoveryAwsKmsMasterKeyProvider -from chalice import Chalice, Response from .key_providers.counting import CountingMasterKey from .key_providers.null import NullMasterKey diff --git a/decrypt_oracle/test/integration/integration_test_utils.py b/decrypt_oracle/test/integration/integration_test_utils.py index c03b7f440..9849f1ecc 100644 --- a/decrypt_oracle/test/integration/integration_test_utils.py +++ b/decrypt_oracle/test/integration/integration_test_utils.py @@ -15,10 +15,11 @@ import json import os from collections import namedtuple + +import pytest from typing import Any, Callable, Iterable, Optional, Text import aws_encryption_sdk -import pytest from aws_encryption_sdk.identifiers import CommitmentPolicy from aws_encryption_sdk.key_providers.kms import StrictAwsKmsMasterKeyProvider diff --git a/decrypt_oracle/test/test_n_generate_test_vectors.py b/decrypt_oracle/test/test_n_generate_test_vectors.py index deb3f7c4d..ae9bb7d7d 100644 --- a/decrypt_oracle/test/test_n_generate_test_vectors.py +++ b/decrypt_oracle/test/test_n_generate_test_vectors.py @@ -15,14 +15,15 @@ import binascii import json import os + +import pytest +from aws_encryption_sdk_decrypt_oracle.key_providers.counting import CountingMasterKey +from aws_encryption_sdk_decrypt_oracle.key_providers.null import NullMasterKey from typing import Dict, Iterable, Text import aws_encryption_sdk -import pytest from aws_encryption_sdk.key_providers.base import MasterKeyProvider from aws_encryption_sdk.key_providers.kms import KMSMasterKey -from aws_encryption_sdk_decrypt_oracle.key_providers.counting import CountingMasterKey -from aws_encryption_sdk_decrypt_oracle.key_providers.null import NullMasterKey from .integration.integration_test_utils import test_vectors_filename diff --git a/examples/test/examples_test_utils.py b/examples/test/examples_test_utils.py index 8a51f21c8..08e8cf2f5 100644 --- a/examples/test/examples_test_utils.py +++ b/examples/test/examples_test_utils.py @@ -49,7 +49,7 @@ from integration_test_utils import ( # noqa pylint: disable=unused-import,import-error get_cmk_arn, - get_second_cmk_arn, get_mrk_arn, + get_second_cmk_arn, get_second_mrk_arn, ) diff --git a/examples/test/test_i_basic_encryption.py b/examples/test/test_i_basic_encryption.py index f2a4fab51..aa32d61fa 100644 --- a/examples/test/test_i_basic_encryption.py +++ b/examples/test/test_i_basic_encryption.py @@ -17,7 +17,6 @@ from ..src.basic_encryption import cycle_string from .examples_test_utils import get_cmk_arn, static_plaintext - pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_basic_file_encryption_with_multiple_providers.py b/examples/test/test_i_basic_file_encryption_with_multiple_providers.py index 282a272ab..0792f4958 100644 --- a/examples/test/test_i_basic_file_encryption_with_multiple_providers.py +++ b/examples/test/test_i_basic_file_encryption_with_multiple_providers.py @@ -18,9 +18,7 @@ import pytest from ..src.basic_file_encryption_with_multiple_providers import cycle_file -from .examples_test_utils import get_cmk_arn -from .examples_test_utils import static_plaintext - +from .examples_test_utils import get_cmk_arn, static_plaintext pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_basic_file_encryption_with_raw_key_provider.py b/examples/test/test_i_basic_file_encryption_with_raw_key_provider.py index 710c0ccac..046b7f964 100644 --- a/examples/test/test_i_basic_file_encryption_with_raw_key_provider.py +++ b/examples/test/test_i_basic_file_encryption_with_raw_key_provider.py @@ -19,7 +19,6 @@ from ..src.basic_file_encryption_with_raw_key_provider import cycle_file from .examples_test_utils import static_plaintext - pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_data_key_caching_basic.py b/examples/test/test_i_data_key_caching_basic.py index 734c35692..7a30f4e53 100644 --- a/examples/test/test_i_data_key_caching_basic.py +++ b/examples/test/test_i_data_key_caching_basic.py @@ -16,7 +16,6 @@ from ..src.data_key_caching_basic import encrypt_with_caching from .examples_test_utils import get_cmk_arn - pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_discovery_kms_provider.py b/examples/test/test_i_discovery_kms_provider.py index e9a1c6e71..0f64cbf59 100644 --- a/examples/test/test_i_discovery_kms_provider.py +++ b/examples/test/test_i_discovery_kms_provider.py @@ -16,9 +16,7 @@ import pytest from ..src.discovery_kms_provider import encrypt_decrypt -from .examples_test_utils import get_cmk_arn -from .examples_test_utils import static_plaintext - +from .examples_test_utils import get_cmk_arn, static_plaintext pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_mrk_aware_kms_provider.py b/examples/test/test_i_mrk_aware_kms_provider.py index 8e7a003f8..a90101fa8 100644 --- a/examples/test/test_i_mrk_aware_kms_provider.py +++ b/examples/test/test_i_mrk_aware_kms_provider.py @@ -15,9 +15,7 @@ import pytest from ..src.mrk_aware_kms_provider import encrypt_decrypt -from .examples_test_utils import get_mrk_arn, get_second_mrk_arn -from .examples_test_utils import static_plaintext - +from .examples_test_utils import get_mrk_arn, get_second_mrk_arn, static_plaintext pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_multiple_kms_cmk.py b/examples/test/test_i_multiple_kms_cmk.py index 39369cbc6..2915a0fd7 100644 --- a/examples/test/test_i_multiple_kms_cmk.py +++ b/examples/test/test_i_multiple_kms_cmk.py @@ -16,9 +16,7 @@ import pytest from ..src.multiple_kms_cmk import encrypt_decrypt -from .examples_test_utils import get_cmk_arn, get_second_cmk_arn -from .examples_test_utils import static_plaintext - +from .examples_test_utils import get_cmk_arn, get_second_cmk_arn, static_plaintext pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_one_kms_cmk.py b/examples/test/test_i_one_kms_cmk.py index 71ce74d3d..96dd48dae 100644 --- a/examples/test/test_i_one_kms_cmk.py +++ b/examples/test/test_i_one_kms_cmk.py @@ -16,9 +16,7 @@ import pytest from ..src.one_kms_cmk import encrypt_decrypt -from .examples_test_utils import get_cmk_arn -from .examples_test_utils import static_plaintext - +from .examples_test_utils import get_cmk_arn, static_plaintext pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_one_kms_cmk_streaming_data.py b/examples/test/test_i_one_kms_cmk_streaming_data.py index b22fa4232..f0a3094d0 100644 --- a/examples/test/test_i_one_kms_cmk_streaming_data.py +++ b/examples/test/test_i_one_kms_cmk_streaming_data.py @@ -20,7 +20,6 @@ from ..src.one_kms_cmk_streaming_data import encrypt_decrypt_stream from .examples_test_utils import get_cmk_arn, static_plaintext - pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_one_kms_cmk_unsigned.py b/examples/test/test_i_one_kms_cmk_unsigned.py index 8a2758c96..41f16473d 100644 --- a/examples/test/test_i_one_kms_cmk_unsigned.py +++ b/examples/test/test_i_one_kms_cmk_unsigned.py @@ -16,9 +16,7 @@ import pytest from ..src.one_kms_cmk_unsigned import encrypt_decrypt -from .examples_test_utils import get_cmk_arn -from .examples_test_utils import static_plaintext - +from .examples_test_utils import get_cmk_arn, static_plaintext pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_set_commitment.py b/examples/test/test_i_set_commitment.py index 96247334b..c14a379bf 100644 --- a/examples/test/test_i_set_commitment.py +++ b/examples/test/test_i_set_commitment.py @@ -16,9 +16,7 @@ import pytest from ..src.set_commitment import encrypt_decrypt -from .examples_test_utils import get_cmk_arn -from .examples_test_utils import static_plaintext - +from .examples_test_utils import get_cmk_arn, static_plaintext pytestmark = [pytest.mark.examples] diff --git a/src/aws_encryption_sdk/materials_managers/mpl/__init__.py b/src/aws_encryption_sdk/materials_managers/mpl/__init__.py index 295400d76..7593a3300 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/__init__.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/__init__.py @@ -10,4 +10,7 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Modules related to the MPL's materials managers interfaces.""" +"""Modules related to the MPL's materials managers interfaces. + +The aws-cryptographic-materials-library MUST be installed to use these modules. +""" diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index 1bbd7c89a..24a10139f 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -1,4 +1,7 @@ -"""Retrieves encryption/decryption materials from the MPL.""" +"""Retrieves encryption/decryption materials from the MPL and interfaces them to EDK components. + +The aws-cryptographic-materials-library MUST be installed to use this module. +""" from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException from aws_cryptographic_materialproviders.mpl.models import ( @@ -54,9 +57,10 @@ def get_encryption_materials( :param request: Request for encryption materials """ try: - mpl_input: MPL_GetEncryptionMaterialsInput = CryptoMaterialsManagerFromMPL._native_to_mpl_get_encryption_materials( - request - ) + mpl_input: MPL_GetEncryptionMaterialsInput = \ + CryptoMaterialsManagerFromMPL._native_to_mpl_get_encryption_materials( + request + ) mpl_output: MPL_GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) return EncryptionMaterialsFromMPL(mpl_output.encryption_materials) except AwsCryptographicMaterialProvidersException as mpl_exception: diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index 31f7d2a65..c23e2b038 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -1,4 +1,7 @@ -"""Provides encryption/decryption materials from an underlying materials provider.""" +"""Provides encryption/decryption materials from an underlying materials provider from the MPL. + +The aws-cryptographic-materials-library MUST be installed to use this module. +""" from aws_cryptographic_materialproviders.mpl.models import ( DecryptionMaterials as MPL_DecryptionMaterials, diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 72ed4efb7..959b5ff0b 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -555,7 +555,7 @@ def _prep_message(self): # MPL verification key is PEM bytes, not DER bytes. # If the underlying CMM is from the MPL, load PEM bytes. if (_HAS_MPL - and isinstance(self.config.materials_manager, CryptoMaterialsManagerFromMPL)): + and isinstance(self.config.materials_manager, CryptoMaterialsManagerFromMPL)): self.signer = Signer.from_key_bytes( algorithm=self._encryption_materials.algorithm, key_bytes=self._encryption_materials.signing_key, encoding=serialization.Encoding.PEM, @@ -923,7 +923,7 @@ def _read_header(self): # MPL verification key is NOT key bytes; it is bytes of the compressed point. # If the underlying CMM is from the MPL, load bytes from encoded point. if (_HAS_MPL - and isinstance(self.config.materials_manager, CryptoMaterialsManagerFromMPL)): + and isinstance(self.config.materials_manager, CryptoMaterialsManagerFromMPL)): self.verifier = Verifier.from_encoded_point( algorithm=header.algorithm, encoded_point=base64.b64encode(decryption_materials.verification_key) diff --git a/test/mpl/__init__.py b/test/mpl/__init__.py index 2a6c71715..37f482e0b 100644 --- a/test/mpl/__init__.py +++ b/test/mpl/__init__.py @@ -10,4 +10,7 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Module containing tests that REQUIRE the aws-cryptographic-material-providers library to run.""" +"""Module testing components that use the MPL. + +The aws-cryptographic-materials-library MUST be installed to run tests in this module. +""" diff --git a/test/mpl/unit/test_material_managers_mpl_cmm.py b/test/mpl/unit/test_material_managers_mpl_cmm.py index a67c3e5c5..fa8f76410 100644 --- a/test/mpl/unit/test_material_managers_mpl_cmm.py +++ b/test/mpl/unit/test_material_managers_mpl_cmm.py @@ -10,7 +10,10 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" +"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic. + +The aws-cryptographic-materials-library MUST be installed to run tests in this module. +""" import pytest from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException diff --git a/test/mpl/unit/test_material_managers_mpl_materials.py b/test/mpl/unit/test_material_managers_mpl_materials.py index cb3ca7397..60e12c634 100644 --- a/test/mpl/unit/test_material_managers_mpl_materials.py +++ b/test/mpl/unit/test_material_managers_mpl_materials.py @@ -10,7 +10,10 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" +"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic. + +The aws-cryptographic-materials-library MUST be installed to run tests in this module. +""" import pytest from aws_cryptographic_materialproviders.mpl.models import ( @@ -18,7 +21,7 @@ EncryptedDataKey as MPL_EncryptedDataKey, EncryptionMaterials as MPL_EncryptionMaterials, ) -from mock import MagicMock, PropertyMock, patch +from mock import MagicMock, patch from typing import Dict, List, Set import aws_encryption_sdk.materials_managers.mpl.materials diff --git a/test_vector_handlers/src/awses_test_vectors/internal/aws_kms.py b/test_vector_handlers/src/awses_test_vectors/internal/aws_kms.py index 14c109e7d..3d2088a73 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/aws_kms.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/aws_kms.py @@ -15,14 +15,15 @@ from aws_encryption_sdk.identifiers import AlgorithmSuite except ImportError: from aws_encryption_sdk.identifiers import Algorithm as AlgorithmSuite + +from awses_test_vectors.internal.defaults import ENCODING + from aws_encryption_sdk.key_providers.kms import ( DiscoveryAwsKmsMasterKeyProvider, MRKAwareDiscoveryAwsKmsMasterKeyProvider, StrictAwsKmsMasterKeyProvider, ) -from awses_test_vectors.internal.defaults import ENCODING - # This lets us easily use a single boto3 client per region for all KMS master keys. KMS_MASTER_KEY_PROVIDER = DiscoveryAwsKmsMasterKeyProvider() KMS_MRK_AWARE_MASTER_KEY_PROVIDER = MRKAwareDiscoveryAwsKmsMasterKeyProvider(discovery_region="us-west-2") diff --git a/test_vector_handlers/src/awses_test_vectors/internal/util.py b/test_vector_handlers/src/awses_test_vectors/internal/util.py index da5552f13..67d4ec67f 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/util.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/util.py @@ -24,12 +24,11 @@ from aws_encryption_sdk.identifiers import Algorithm as AlgorithmSuite try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Any, Callable, Dict, Iterable, Type # noqa pylint: disable=unused-import - from awses_test_vectors.internal.mypy_types import ( # noqa pylint: disable=unused-import ISINSTANCE, MANIFEST_VERSION, ) + from typing import Any, Callable, Dict, Iterable, Type # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index c94fd1452..a53f6cc5d 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -20,11 +20,8 @@ from enum import Enum import attr -import aws_encryption_sdk import pytest import six -from aws_encryption_sdk.identifiers import CommitmentPolicy - from awses_test_vectors.internal.defaults import ENCODING from awses_test_vectors.internal.util import ( dictionary_validator, @@ -35,14 +32,16 @@ from awses_test_vectors.manifests.keys import KeysManifest from awses_test_vectors.manifests.master_key import MasterKeySpec, master_key_provider_from_master_key_specs -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import IO, Callable, Dict, Iterable, Optional # noqa pylint: disable=unused-import +import aws_encryption_sdk +from aws_encryption_sdk.identifiers import CommitmentPolicy +try: # Python 3.5.0 and 3.5.1 have incompatible typing modules from awses_test_vectors.internal.mypy_types import ( # noqa pylint: disable=unused-import DECRYPT_SCENARIO_SPEC, FULL_MESSAGE_DECRYPT_MANIFEST, MASTER_KEY_SPEC, ) + from typing import IO, Callable, Dict, Iterable, Optional # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index e407a1b65..48fc1a6b3 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -22,11 +22,6 @@ import attr import six -from aws_encryption_sdk.caches.local import LocalCryptoMaterialsCache -from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager -from aws_encryption_sdk.materials_managers.caching import CachingCryptoMaterialsManager -from aws_encryption_sdk.materials_managers.default import DefaultCryptoMaterialsManager - from awses_test_vectors.internal.defaults import ENCODING from awses_test_vectors.internal.util import ( dictionary_validator, @@ -45,6 +40,11 @@ from awses_test_vectors.manifests.full_message.encrypt import MessageEncryptionTestScenario from awses_test_vectors.manifests.keys import KeysManifest +from aws_encryption_sdk.caches.local import LocalCryptoMaterialsCache +from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager +from aws_encryption_sdk.materials_managers.caching import CachingCryptoMaterialsManager +from aws_encryption_sdk.materials_managers.default import DefaultCryptoMaterialsManager + try: from aws_encryption_sdk.identifiers import AlgorithmSuite except ImportError: @@ -53,12 +53,11 @@ from awses_test_vectors.manifests.master_key import MasterKeySpec, master_key_provider_from_master_key_specs try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import IO, Callable, Dict, Iterable, Optional # noqa pylint: disable=unused-import - from awses_test_vectors.internal.mypy_types import ( # noqa pylint: disable=unused-import ENCRYPT_SCENARIO_SPEC, PLAINTEXTS_SPEC, ) + from typing import IO, Callable, Dict, Iterable, Optional # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index c77fed1ce..2e88c8a52 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -19,9 +19,7 @@ import os import attr -import aws_encryption_sdk import six - from awses_test_vectors.internal.defaults import ENCODING from awses_test_vectors.internal.util import ( algorithm_suite_from_string_id, @@ -34,6 +32,8 @@ from awses_test_vectors.manifests.keys import KeysManifest from awses_test_vectors.manifests.master_key import MasterKeySpec, master_key_provider_from_master_key_specs +import aws_encryption_sdk + try: from aws_encryption_sdk.identifiers import AlgorithmSuite, CommitmentPolicy except ImportError: @@ -41,12 +41,11 @@ try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import IO, Callable, Dict, Iterable, Optional # noqa pylint: disable=unused-import - from awses_test_vectors.internal.mypy_types import ( # noqa pylint: disable=unused-import ENCRYPT_SCENARIO_SPEC, PLAINTEXTS_SPEC, ) + from typing import IO, Callable, Dict, Iterable, Optional # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/keys.py b/test_vector_handlers/src/awses_test_vectors/manifests/keys.py index cba6b7e25..546dbb489 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/keys.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/keys.py @@ -19,14 +19,11 @@ import attr import six - from awses_test_vectors.internal.aws_kms import arn_from_key_id from awses_test_vectors.internal.defaults import ENCODING from awses_test_vectors.internal.util import dictionary_validator, membership_validator, validate_manifest_type try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Dict, Iterable, Optional, cast # noqa pylint: disable=unused-import - from awses_test_vectors.internal.mypy_types import ( # noqa pylint: disable=unused-import AWS_KMS_KEY_SPEC, KEY_SPEC, @@ -34,6 +31,7 @@ MANIFEST_VERSION, MANUAL_KEY_SPEC, ) + from typing import Dict, Iterable, Optional, cast # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/master_key.py b/test_vector_handlers/src/awses_test_vectors/manifests/master_key.py index a1a7ae4af..8b00870f1 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/master_key.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/master_key.py @@ -17,6 +17,10 @@ """ import attr import six +from awses_test_vectors.internal.aws_kms import KMS_MASTER_KEY_PROVIDER, KMS_MRK_AWARE_MASTER_KEY_PROVIDER +from awses_test_vectors.internal.util import membership_validator +from awses_test_vectors.manifests.keys import KeysManifest, KeySpec # noqa pylint: disable=unused-import + from aws_encryption_sdk.identifiers import EncryptionKeyType, WrappingAlgorithm from aws_encryption_sdk.key_providers.base import MasterKeyProvider # noqa pylint: disable=unused-import from aws_encryption_sdk.key_providers.kms import ( # noqa pylint: disable=unused-import @@ -26,10 +30,6 @@ ) from aws_encryption_sdk.key_providers.raw import RawMasterKey -from awses_test_vectors.internal.aws_kms import KMS_MASTER_KEY_PROVIDER, KMS_MRK_AWARE_MASTER_KEY_PROVIDER -from awses_test_vectors.internal.util import membership_validator -from awses_test_vectors.manifests.keys import KeysManifest, KeySpec # noqa pylint: disable=unused-import - try: from aws_encryption_sdk.internal.crypto.wrapping_keys import WrappingKey except ImportError: @@ -37,9 +37,8 @@ try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Iterable # noqa pylint: disable=unused-import - from awses_test_vectors.internal.mypy_types import MASTER_KEY_SPEC # noqa pylint: disable=unused-import + from typing import Iterable # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass diff --git a/test_vector_handlers/test/integration/commands/test_i_full_message_encrypt.py b/test_vector_handlers/test/integration/commands/test_i_full_message_encrypt.py index 6305a15da..6928caeba 100644 --- a/test_vector_handlers/test/integration/commands/test_i_full_message_encrypt.py +++ b/test_vector_handlers/test/integration/commands/test_i_full_message_encrypt.py @@ -14,7 +14,6 @@ Integration tests for ``awses_test_vectors.commands``. """ import pytest - from awses_test_vectors.commands import full_message_decrypt, full_message_decrypt_generate, full_message_encrypt from ..integration_test_utils import ( # noqa pylint: disable=unused-import From ac0ceb3e60d9b4b8ff3c3ae44b7ff2b9b0a50af2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 17:29:19 -0800 Subject: [PATCH 122/376] fix --- src/aws_encryption_sdk/materials_managers/mpl/cmm.py | 4 +++- src/aws_encryption_sdk/materials_managers/mpl/materials.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index 24a10139f..1913505c4 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -2,7 +2,8 @@ The aws-cryptographic-materials-library MUST be installed to use this module. """ - +# pylint should pass even if the MPL isn't installed +# noqa pylint: disable=import-error from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException from aws_cryptographic_materialproviders.mpl.models import ( AlgorithmSuiteIdESDK as MPL_AlgorithmSuiteIdESDK, @@ -16,6 +17,7 @@ from aws_cryptographic_materialproviders.mpl.references import ( ICryptographicMaterialsManager as MPL_ICryptographicMaterialsManager, ) +# noqa pylint: enable=import-error from typing import List from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index c23e2b038..faa47cb46 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -2,12 +2,14 @@ The aws-cryptographic-materials-library MUST be installed to use this module. """ - +# pylint should pass even if the MPL isn't installed +# noqa pylint: disable=import-error from aws_cryptographic_materialproviders.mpl.models import ( DecryptionMaterials as MPL_DecryptionMaterials, EncryptedDataKey as MPL_EncryptedDataKey, EncryptionMaterials as MPL_EncryptionMaterials, ) +# noqa pylint: enable=import-error from typing import Dict, List, Set from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite From 2fd88584be77ed1be902e56ff473208e674b2d1d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 17:32:01 -0800 Subject: [PATCH 123/376] oops --- src/aws_encryption_sdk/materials_managers/mpl/cmm.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index 1913505c4..760808bfe 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -3,7 +3,8 @@ The aws-cryptographic-materials-library MUST be installed to use this module. """ # pylint should pass even if the MPL isn't installed -# noqa pylint: disable=import-error +# Also thinks these imports aren't used if it can't import them +# noqa pylint: disable=import-error,unused-import from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException from aws_cryptographic_materialproviders.mpl.models import ( AlgorithmSuiteIdESDK as MPL_AlgorithmSuiteIdESDK, @@ -17,7 +18,7 @@ from aws_cryptographic_materialproviders.mpl.references import ( ICryptographicMaterialsManager as MPL_ICryptographicMaterialsManager, ) -# noqa pylint: enable=import-error +# noqa pylint: enable=import-error,unused-import from typing import List from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError From 51c6a9caad271e1288ae06e07355c8b2cc6c0b85 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 17:33:06 -0800 Subject: [PATCH 124/376] revert --- decrypt_oracle/.chalice/pipeline.py | 2 +- .../src/aws_encryption_sdk_decrypt_oracle/app.py | 3 +-- .../test/integration/integration_test_utils.py | 3 +-- decrypt_oracle/test/test_n_generate_test_vectors.py | 7 +++---- examples/test/examples_test_utils.py | 2 +- examples/test/test_i_basic_encryption.py | 1 + ...basic_file_encryption_with_multiple_providers.py | 4 +++- ...i_basic_file_encryption_with_raw_key_provider.py | 1 + examples/test/test_i_data_key_caching_basic.py | 1 + examples/test/test_i_discovery_kms_provider.py | 4 +++- examples/test/test_i_mrk_aware_kms_provider.py | 4 +++- examples/test/test_i_multiple_kms_cmk.py | 4 +++- examples/test/test_i_one_kms_cmk.py | 4 +++- examples/test/test_i_one_kms_cmk_streaming_data.py | 1 + examples/test/test_i_one_kms_cmk_unsigned.py | 4 +++- examples/test/test_i_set_commitment.py | 4 +++- .../materials_managers/mpl/__init__.py | 5 +---- .../materials_managers/mpl/cmm.py | 7 +++---- .../materials_managers/mpl/materials.py | 5 +++++ src/aws_encryption_sdk/streaming_client.py | 4 ++-- test/mpl/__init__.py | 5 +---- test/mpl/unit/test_material_managers_mpl_cmm.py | 5 +---- .../unit/test_material_managers_mpl_materials.py | 7 ++----- .../src/awses_test_vectors/internal/aws_kms.py | 5 ++--- .../src/awses_test_vectors/internal/util.py | 3 ++- .../manifests/full_message/decrypt.py | 9 +++++---- .../manifests/full_message/decrypt_generation.py | 13 +++++++------ .../manifests/full_message/encrypt.py | 7 ++++--- .../src/awses_test_vectors/manifests/keys.py | 4 +++- .../src/awses_test_vectors/manifests/master_key.py | 11 ++++++----- .../commands/test_i_full_message_encrypt.py | 1 + 31 files changed, 77 insertions(+), 63 deletions(-) diff --git a/decrypt_oracle/.chalice/pipeline.py b/decrypt_oracle/.chalice/pipeline.py index c05df6739..9d5573646 100644 --- a/decrypt_oracle/.chalice/pipeline.py +++ b/decrypt_oracle/.chalice/pipeline.py @@ -2,6 +2,7 @@ import argparse import getpass import logging +from typing import Iterable import boto3 import troposphere @@ -19,7 +20,6 @@ ) from botocore.exceptions import ClientError from troposphere import GetAtt, Ref, Sub, Template, codebuild, codepipeline, iam, s3 -from typing import Iterable APPLICATION_NAME = "AwsEncryptionSdkDecryptOraclePython" PIPELINE_STACK_NAME = "{}DeployPipeline".format(APPLICATION_NAME) diff --git a/decrypt_oracle/src/aws_encryption_sdk_decrypt_oracle/app.py b/decrypt_oracle/src/aws_encryption_sdk_decrypt_oracle/app.py index e250bb3c8..820b9e015 100644 --- a/decrypt_oracle/src/aws_encryption_sdk_decrypt_oracle/app.py +++ b/decrypt_oracle/src/aws_encryption_sdk_decrypt_oracle/app.py @@ -15,11 +15,10 @@ import logging import os -from chalice import Chalice, Response - import aws_encryption_sdk from aws_encryption_sdk.identifiers import CommitmentPolicy from aws_encryption_sdk.key_providers.kms import DiscoveryAwsKmsMasterKeyProvider +from chalice import Chalice, Response from .key_providers.counting import CountingMasterKey from .key_providers.null import NullMasterKey diff --git a/decrypt_oracle/test/integration/integration_test_utils.py b/decrypt_oracle/test/integration/integration_test_utils.py index 9849f1ecc..c03b7f440 100644 --- a/decrypt_oracle/test/integration/integration_test_utils.py +++ b/decrypt_oracle/test/integration/integration_test_utils.py @@ -15,11 +15,10 @@ import json import os from collections import namedtuple - -import pytest from typing import Any, Callable, Iterable, Optional, Text import aws_encryption_sdk +import pytest from aws_encryption_sdk.identifiers import CommitmentPolicy from aws_encryption_sdk.key_providers.kms import StrictAwsKmsMasterKeyProvider diff --git a/decrypt_oracle/test/test_n_generate_test_vectors.py b/decrypt_oracle/test/test_n_generate_test_vectors.py index ae9bb7d7d..deb3f7c4d 100644 --- a/decrypt_oracle/test/test_n_generate_test_vectors.py +++ b/decrypt_oracle/test/test_n_generate_test_vectors.py @@ -15,15 +15,14 @@ import binascii import json import os - -import pytest -from aws_encryption_sdk_decrypt_oracle.key_providers.counting import CountingMasterKey -from aws_encryption_sdk_decrypt_oracle.key_providers.null import NullMasterKey from typing import Dict, Iterable, Text import aws_encryption_sdk +import pytest from aws_encryption_sdk.key_providers.base import MasterKeyProvider from aws_encryption_sdk.key_providers.kms import KMSMasterKey +from aws_encryption_sdk_decrypt_oracle.key_providers.counting import CountingMasterKey +from aws_encryption_sdk_decrypt_oracle.key_providers.null import NullMasterKey from .integration.integration_test_utils import test_vectors_filename diff --git a/examples/test/examples_test_utils.py b/examples/test/examples_test_utils.py index 08e8cf2f5..8a51f21c8 100644 --- a/examples/test/examples_test_utils.py +++ b/examples/test/examples_test_utils.py @@ -49,7 +49,7 @@ from integration_test_utils import ( # noqa pylint: disable=unused-import,import-error get_cmk_arn, - get_mrk_arn, get_second_cmk_arn, + get_mrk_arn, get_second_mrk_arn, ) diff --git a/examples/test/test_i_basic_encryption.py b/examples/test/test_i_basic_encryption.py index aa32d61fa..f2a4fab51 100644 --- a/examples/test/test_i_basic_encryption.py +++ b/examples/test/test_i_basic_encryption.py @@ -17,6 +17,7 @@ from ..src.basic_encryption import cycle_string from .examples_test_utils import get_cmk_arn, static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_basic_file_encryption_with_multiple_providers.py b/examples/test/test_i_basic_file_encryption_with_multiple_providers.py index 0792f4958..282a272ab 100644 --- a/examples/test/test_i_basic_file_encryption_with_multiple_providers.py +++ b/examples/test/test_i_basic_file_encryption_with_multiple_providers.py @@ -18,7 +18,9 @@ import pytest from ..src.basic_file_encryption_with_multiple_providers import cycle_file -from .examples_test_utils import get_cmk_arn, static_plaintext +from .examples_test_utils import get_cmk_arn +from .examples_test_utils import static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_basic_file_encryption_with_raw_key_provider.py b/examples/test/test_i_basic_file_encryption_with_raw_key_provider.py index 046b7f964..710c0ccac 100644 --- a/examples/test/test_i_basic_file_encryption_with_raw_key_provider.py +++ b/examples/test/test_i_basic_file_encryption_with_raw_key_provider.py @@ -19,6 +19,7 @@ from ..src.basic_file_encryption_with_raw_key_provider import cycle_file from .examples_test_utils import static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_data_key_caching_basic.py b/examples/test/test_i_data_key_caching_basic.py index 7a30f4e53..734c35692 100644 --- a/examples/test/test_i_data_key_caching_basic.py +++ b/examples/test/test_i_data_key_caching_basic.py @@ -16,6 +16,7 @@ from ..src.data_key_caching_basic import encrypt_with_caching from .examples_test_utils import get_cmk_arn + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_discovery_kms_provider.py b/examples/test/test_i_discovery_kms_provider.py index 0f64cbf59..e9a1c6e71 100644 --- a/examples/test/test_i_discovery_kms_provider.py +++ b/examples/test/test_i_discovery_kms_provider.py @@ -16,7 +16,9 @@ import pytest from ..src.discovery_kms_provider import encrypt_decrypt -from .examples_test_utils import get_cmk_arn, static_plaintext +from .examples_test_utils import get_cmk_arn +from .examples_test_utils import static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_mrk_aware_kms_provider.py b/examples/test/test_i_mrk_aware_kms_provider.py index a90101fa8..8e7a003f8 100644 --- a/examples/test/test_i_mrk_aware_kms_provider.py +++ b/examples/test/test_i_mrk_aware_kms_provider.py @@ -15,7 +15,9 @@ import pytest from ..src.mrk_aware_kms_provider import encrypt_decrypt -from .examples_test_utils import get_mrk_arn, get_second_mrk_arn, static_plaintext +from .examples_test_utils import get_mrk_arn, get_second_mrk_arn +from .examples_test_utils import static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_multiple_kms_cmk.py b/examples/test/test_i_multiple_kms_cmk.py index 2915a0fd7..39369cbc6 100644 --- a/examples/test/test_i_multiple_kms_cmk.py +++ b/examples/test/test_i_multiple_kms_cmk.py @@ -16,7 +16,9 @@ import pytest from ..src.multiple_kms_cmk import encrypt_decrypt -from .examples_test_utils import get_cmk_arn, get_second_cmk_arn, static_plaintext +from .examples_test_utils import get_cmk_arn, get_second_cmk_arn +from .examples_test_utils import static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_one_kms_cmk.py b/examples/test/test_i_one_kms_cmk.py index 96dd48dae..71ce74d3d 100644 --- a/examples/test/test_i_one_kms_cmk.py +++ b/examples/test/test_i_one_kms_cmk.py @@ -16,7 +16,9 @@ import pytest from ..src.one_kms_cmk import encrypt_decrypt -from .examples_test_utils import get_cmk_arn, static_plaintext +from .examples_test_utils import get_cmk_arn +from .examples_test_utils import static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_one_kms_cmk_streaming_data.py b/examples/test/test_i_one_kms_cmk_streaming_data.py index f0a3094d0..b22fa4232 100644 --- a/examples/test/test_i_one_kms_cmk_streaming_data.py +++ b/examples/test/test_i_one_kms_cmk_streaming_data.py @@ -20,6 +20,7 @@ from ..src.one_kms_cmk_streaming_data import encrypt_decrypt_stream from .examples_test_utils import get_cmk_arn, static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_one_kms_cmk_unsigned.py b/examples/test/test_i_one_kms_cmk_unsigned.py index 41f16473d..8a2758c96 100644 --- a/examples/test/test_i_one_kms_cmk_unsigned.py +++ b/examples/test/test_i_one_kms_cmk_unsigned.py @@ -16,7 +16,9 @@ import pytest from ..src.one_kms_cmk_unsigned import encrypt_decrypt -from .examples_test_utils import get_cmk_arn, static_plaintext +from .examples_test_utils import get_cmk_arn +from .examples_test_utils import static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_set_commitment.py b/examples/test/test_i_set_commitment.py index c14a379bf..96247334b 100644 --- a/examples/test/test_i_set_commitment.py +++ b/examples/test/test_i_set_commitment.py @@ -16,7 +16,9 @@ import pytest from ..src.set_commitment import encrypt_decrypt -from .examples_test_utils import get_cmk_arn, static_plaintext +from .examples_test_utils import get_cmk_arn +from .examples_test_utils import static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/src/aws_encryption_sdk/materials_managers/mpl/__init__.py b/src/aws_encryption_sdk/materials_managers/mpl/__init__.py index 7593a3300..295400d76 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/__init__.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/__init__.py @@ -10,7 +10,4 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Modules related to the MPL's materials managers interfaces. - -The aws-cryptographic-materials-library MUST be installed to use these modules. -""" +"""Modules related to the MPL's materials managers interfaces.""" diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index 760808bfe..ead7c48f1 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -60,10 +60,9 @@ def get_encryption_materials( :param request: Request for encryption materials """ try: - mpl_input: MPL_GetEncryptionMaterialsInput = \ - CryptoMaterialsManagerFromMPL._native_to_mpl_get_encryption_materials( - request - ) + mpl_input: MPL_GetEncryptionMaterialsInput = CryptoMaterialsManagerFromMPL._native_to_mpl_get_encryption_materials( + request + ) mpl_output: MPL_GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) return EncryptionMaterialsFromMPL(mpl_output.encryption_materials) except AwsCryptographicMaterialProvidersException as mpl_exception: diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index faa47cb46..a82a3c372 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -1,9 +1,14 @@ +<<<<<<< HEAD """Provides encryption/decryption materials from an underlying materials provider from the MPL. The aws-cryptographic-materials-library MUST be installed to use this module. """ # pylint should pass even if the MPL isn't installed # noqa pylint: disable=import-error +======= +"""Provides encryption/decryption materials from an underlying materials provider.""" + +>>>>>>> parent of 22eabb6 (fix) from aws_cryptographic_materialproviders.mpl.models import ( DecryptionMaterials as MPL_DecryptionMaterials, EncryptedDataKey as MPL_EncryptedDataKey, diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 959b5ff0b..72ed4efb7 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -555,7 +555,7 @@ def _prep_message(self): # MPL verification key is PEM bytes, not DER bytes. # If the underlying CMM is from the MPL, load PEM bytes. if (_HAS_MPL - and isinstance(self.config.materials_manager, CryptoMaterialsManagerFromMPL)): + and isinstance(self.config.materials_manager, CryptoMaterialsManagerFromMPL)): self.signer = Signer.from_key_bytes( algorithm=self._encryption_materials.algorithm, key_bytes=self._encryption_materials.signing_key, encoding=serialization.Encoding.PEM, @@ -923,7 +923,7 @@ def _read_header(self): # MPL verification key is NOT key bytes; it is bytes of the compressed point. # If the underlying CMM is from the MPL, load bytes from encoded point. if (_HAS_MPL - and isinstance(self.config.materials_manager, CryptoMaterialsManagerFromMPL)): + and isinstance(self.config.materials_manager, CryptoMaterialsManagerFromMPL)): self.verifier = Verifier.from_encoded_point( algorithm=header.algorithm, encoded_point=base64.b64encode(decryption_materials.verification_key) diff --git a/test/mpl/__init__.py b/test/mpl/__init__.py index 37f482e0b..2a6c71715 100644 --- a/test/mpl/__init__.py +++ b/test/mpl/__init__.py @@ -10,7 +10,4 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Module testing components that use the MPL. - -The aws-cryptographic-materials-library MUST be installed to run tests in this module. -""" +"""Module containing tests that REQUIRE the aws-cryptographic-material-providers library to run.""" diff --git a/test/mpl/unit/test_material_managers_mpl_cmm.py b/test/mpl/unit/test_material_managers_mpl_cmm.py index fa8f76410..a67c3e5c5 100644 --- a/test/mpl/unit/test_material_managers_mpl_cmm.py +++ b/test/mpl/unit/test_material_managers_mpl_cmm.py @@ -10,10 +10,7 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic. - -The aws-cryptographic-materials-library MUST be installed to run tests in this module. -""" +"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" import pytest from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException diff --git a/test/mpl/unit/test_material_managers_mpl_materials.py b/test/mpl/unit/test_material_managers_mpl_materials.py index 60e12c634..cb3ca7397 100644 --- a/test/mpl/unit/test_material_managers_mpl_materials.py +++ b/test/mpl/unit/test_material_managers_mpl_materials.py @@ -10,10 +10,7 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic. - -The aws-cryptographic-materials-library MUST be installed to run tests in this module. -""" +"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" import pytest from aws_cryptographic_materialproviders.mpl.models import ( @@ -21,7 +18,7 @@ EncryptedDataKey as MPL_EncryptedDataKey, EncryptionMaterials as MPL_EncryptionMaterials, ) -from mock import MagicMock, patch +from mock import MagicMock, PropertyMock, patch from typing import Dict, List, Set import aws_encryption_sdk.materials_managers.mpl.materials diff --git a/test_vector_handlers/src/awses_test_vectors/internal/aws_kms.py b/test_vector_handlers/src/awses_test_vectors/internal/aws_kms.py index 3d2088a73..14c109e7d 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/aws_kms.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/aws_kms.py @@ -15,15 +15,14 @@ from aws_encryption_sdk.identifiers import AlgorithmSuite except ImportError: from aws_encryption_sdk.identifiers import Algorithm as AlgorithmSuite - -from awses_test_vectors.internal.defaults import ENCODING - from aws_encryption_sdk.key_providers.kms import ( DiscoveryAwsKmsMasterKeyProvider, MRKAwareDiscoveryAwsKmsMasterKeyProvider, StrictAwsKmsMasterKeyProvider, ) +from awses_test_vectors.internal.defaults import ENCODING + # This lets us easily use a single boto3 client per region for all KMS master keys. KMS_MASTER_KEY_PROVIDER = DiscoveryAwsKmsMasterKeyProvider() KMS_MRK_AWARE_MASTER_KEY_PROVIDER = MRKAwareDiscoveryAwsKmsMasterKeyProvider(discovery_region="us-west-2") diff --git a/test_vector_handlers/src/awses_test_vectors/internal/util.py b/test_vector_handlers/src/awses_test_vectors/internal/util.py index 67d4ec67f..da5552f13 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/util.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/util.py @@ -24,11 +24,12 @@ from aws_encryption_sdk.identifiers import Algorithm as AlgorithmSuite try: # Python 3.5.0 and 3.5.1 have incompatible typing modules + from typing import Any, Callable, Dict, Iterable, Type # noqa pylint: disable=unused-import + from awses_test_vectors.internal.mypy_types import ( # noqa pylint: disable=unused-import ISINSTANCE, MANIFEST_VERSION, ) - from typing import Any, Callable, Dict, Iterable, Type # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index a53f6cc5d..c94fd1452 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -20,8 +20,11 @@ from enum import Enum import attr +import aws_encryption_sdk import pytest import six +from aws_encryption_sdk.identifiers import CommitmentPolicy + from awses_test_vectors.internal.defaults import ENCODING from awses_test_vectors.internal.util import ( dictionary_validator, @@ -32,16 +35,14 @@ from awses_test_vectors.manifests.keys import KeysManifest from awses_test_vectors.manifests.master_key import MasterKeySpec, master_key_provider_from_master_key_specs -import aws_encryption_sdk -from aws_encryption_sdk.identifiers import CommitmentPolicy - try: # Python 3.5.0 and 3.5.1 have incompatible typing modules + from typing import IO, Callable, Dict, Iterable, Optional # noqa pylint: disable=unused-import + from awses_test_vectors.internal.mypy_types import ( # noqa pylint: disable=unused-import DECRYPT_SCENARIO_SPEC, FULL_MESSAGE_DECRYPT_MANIFEST, MASTER_KEY_SPEC, ) - from typing import IO, Callable, Dict, Iterable, Optional # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 48fc1a6b3..e407a1b65 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -22,6 +22,11 @@ import attr import six +from aws_encryption_sdk.caches.local import LocalCryptoMaterialsCache +from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager +from aws_encryption_sdk.materials_managers.caching import CachingCryptoMaterialsManager +from aws_encryption_sdk.materials_managers.default import DefaultCryptoMaterialsManager + from awses_test_vectors.internal.defaults import ENCODING from awses_test_vectors.internal.util import ( dictionary_validator, @@ -40,11 +45,6 @@ from awses_test_vectors.manifests.full_message.encrypt import MessageEncryptionTestScenario from awses_test_vectors.manifests.keys import KeysManifest -from aws_encryption_sdk.caches.local import LocalCryptoMaterialsCache -from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager -from aws_encryption_sdk.materials_managers.caching import CachingCryptoMaterialsManager -from aws_encryption_sdk.materials_managers.default import DefaultCryptoMaterialsManager - try: from aws_encryption_sdk.identifiers import AlgorithmSuite except ImportError: @@ -53,11 +53,12 @@ from awses_test_vectors.manifests.master_key import MasterKeySpec, master_key_provider_from_master_key_specs try: # Python 3.5.0 and 3.5.1 have incompatible typing modules + from typing import IO, Callable, Dict, Iterable, Optional # noqa pylint: disable=unused-import + from awses_test_vectors.internal.mypy_types import ( # noqa pylint: disable=unused-import ENCRYPT_SCENARIO_SPEC, PLAINTEXTS_SPEC, ) - from typing import IO, Callable, Dict, Iterable, Optional # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 2e88c8a52..c77fed1ce 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -19,7 +19,9 @@ import os import attr +import aws_encryption_sdk import six + from awses_test_vectors.internal.defaults import ENCODING from awses_test_vectors.internal.util import ( algorithm_suite_from_string_id, @@ -32,8 +34,6 @@ from awses_test_vectors.manifests.keys import KeysManifest from awses_test_vectors.manifests.master_key import MasterKeySpec, master_key_provider_from_master_key_specs -import aws_encryption_sdk - try: from aws_encryption_sdk.identifiers import AlgorithmSuite, CommitmentPolicy except ImportError: @@ -41,11 +41,12 @@ try: # Python 3.5.0 and 3.5.1 have incompatible typing modules + from typing import IO, Callable, Dict, Iterable, Optional # noqa pylint: disable=unused-import + from awses_test_vectors.internal.mypy_types import ( # noqa pylint: disable=unused-import ENCRYPT_SCENARIO_SPEC, PLAINTEXTS_SPEC, ) - from typing import IO, Callable, Dict, Iterable, Optional # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/keys.py b/test_vector_handlers/src/awses_test_vectors/manifests/keys.py index 546dbb489..cba6b7e25 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/keys.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/keys.py @@ -19,11 +19,14 @@ import attr import six + from awses_test_vectors.internal.aws_kms import arn_from_key_id from awses_test_vectors.internal.defaults import ENCODING from awses_test_vectors.internal.util import dictionary_validator, membership_validator, validate_manifest_type try: # Python 3.5.0 and 3.5.1 have incompatible typing modules + from typing import Dict, Iterable, Optional, cast # noqa pylint: disable=unused-import + from awses_test_vectors.internal.mypy_types import ( # noqa pylint: disable=unused-import AWS_KMS_KEY_SPEC, KEY_SPEC, @@ -31,7 +34,6 @@ MANIFEST_VERSION, MANUAL_KEY_SPEC, ) - from typing import Dict, Iterable, Optional, cast # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/master_key.py b/test_vector_handlers/src/awses_test_vectors/manifests/master_key.py index 8b00870f1..a1a7ae4af 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/master_key.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/master_key.py @@ -17,10 +17,6 @@ """ import attr import six -from awses_test_vectors.internal.aws_kms import KMS_MASTER_KEY_PROVIDER, KMS_MRK_AWARE_MASTER_KEY_PROVIDER -from awses_test_vectors.internal.util import membership_validator -from awses_test_vectors.manifests.keys import KeysManifest, KeySpec # noqa pylint: disable=unused-import - from aws_encryption_sdk.identifiers import EncryptionKeyType, WrappingAlgorithm from aws_encryption_sdk.key_providers.base import MasterKeyProvider # noqa pylint: disable=unused-import from aws_encryption_sdk.key_providers.kms import ( # noqa pylint: disable=unused-import @@ -30,6 +26,10 @@ ) from aws_encryption_sdk.key_providers.raw import RawMasterKey +from awses_test_vectors.internal.aws_kms import KMS_MASTER_KEY_PROVIDER, KMS_MRK_AWARE_MASTER_KEY_PROVIDER +from awses_test_vectors.internal.util import membership_validator +from awses_test_vectors.manifests.keys import KeysManifest, KeySpec # noqa pylint: disable=unused-import + try: from aws_encryption_sdk.internal.crypto.wrapping_keys import WrappingKey except ImportError: @@ -37,8 +37,9 @@ try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from awses_test_vectors.internal.mypy_types import MASTER_KEY_SPEC # noqa pylint: disable=unused-import from typing import Iterable # noqa pylint: disable=unused-import + + from awses_test_vectors.internal.mypy_types import MASTER_KEY_SPEC # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass diff --git a/test_vector_handlers/test/integration/commands/test_i_full_message_encrypt.py b/test_vector_handlers/test/integration/commands/test_i_full_message_encrypt.py index 6928caeba..6305a15da 100644 --- a/test_vector_handlers/test/integration/commands/test_i_full_message_encrypt.py +++ b/test_vector_handlers/test/integration/commands/test_i_full_message_encrypt.py @@ -14,6 +14,7 @@ Integration tests for ``awses_test_vectors.commands``. """ import pytest + from awses_test_vectors.commands import full_message_decrypt, full_message_decrypt_generate, full_message_encrypt from ..integration_test_utils import ( # noqa pylint: disable=unused-import From 800f9de0de601bb647ba2ab57e7df3d4eedfa795 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 17:33:18 -0800 Subject: [PATCH 125/376] revert --- src/aws_encryption_sdk/materials_managers/mpl/materials.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index a82a3c372..faa47cb46 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -1,14 +1,9 @@ -<<<<<<< HEAD """Provides encryption/decryption materials from an underlying materials provider from the MPL. The aws-cryptographic-materials-library MUST be installed to use this module. """ # pylint should pass even if the MPL isn't installed # noqa pylint: disable=import-error -======= -"""Provides encryption/decryption materials from an underlying materials provider.""" - ->>>>>>> parent of 22eabb6 (fix) from aws_cryptographic_materialproviders.mpl.models import ( DecryptionMaterials as MPL_DecryptionMaterials, EncryptedDataKey as MPL_EncryptedDataKey, From ebcb7590c472f82affa86ecc793a5e8a3e494a8b Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 17:35:14 -0800 Subject: [PATCH 126/376] fix --- src/aws_encryption_sdk/materials_managers/mpl/__init__.py | 5 ++++- src/aws_encryption_sdk/materials_managers/mpl/cmm.py | 7 ++++--- src/aws_encryption_sdk/streaming_client.py | 4 ++-- test/mpl/__init__.py | 5 ++++- test/mpl/unit/test_material_managers_mpl_cmm.py | 5 ++++- test/mpl/unit/test_material_managers_mpl_materials.py | 7 +++++-- 6 files changed, 23 insertions(+), 10 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/__init__.py b/src/aws_encryption_sdk/materials_managers/mpl/__init__.py index 295400d76..7593a3300 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/__init__.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/__init__.py @@ -10,4 +10,7 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Modules related to the MPL's materials managers interfaces.""" +"""Modules related to the MPL's materials managers interfaces. + +The aws-cryptographic-materials-library MUST be installed to use these modules. +""" diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index ead7c48f1..760808bfe 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -60,9 +60,10 @@ def get_encryption_materials( :param request: Request for encryption materials """ try: - mpl_input: MPL_GetEncryptionMaterialsInput = CryptoMaterialsManagerFromMPL._native_to_mpl_get_encryption_materials( - request - ) + mpl_input: MPL_GetEncryptionMaterialsInput = \ + CryptoMaterialsManagerFromMPL._native_to_mpl_get_encryption_materials( + request + ) mpl_output: MPL_GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) return EncryptionMaterialsFromMPL(mpl_output.encryption_materials) except AwsCryptographicMaterialProvidersException as mpl_exception: diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 72ed4efb7..959b5ff0b 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -555,7 +555,7 @@ def _prep_message(self): # MPL verification key is PEM bytes, not DER bytes. # If the underlying CMM is from the MPL, load PEM bytes. if (_HAS_MPL - and isinstance(self.config.materials_manager, CryptoMaterialsManagerFromMPL)): + and isinstance(self.config.materials_manager, CryptoMaterialsManagerFromMPL)): self.signer = Signer.from_key_bytes( algorithm=self._encryption_materials.algorithm, key_bytes=self._encryption_materials.signing_key, encoding=serialization.Encoding.PEM, @@ -923,7 +923,7 @@ def _read_header(self): # MPL verification key is NOT key bytes; it is bytes of the compressed point. # If the underlying CMM is from the MPL, load bytes from encoded point. if (_HAS_MPL - and isinstance(self.config.materials_manager, CryptoMaterialsManagerFromMPL)): + and isinstance(self.config.materials_manager, CryptoMaterialsManagerFromMPL)): self.verifier = Verifier.from_encoded_point( algorithm=header.algorithm, encoded_point=base64.b64encode(decryption_materials.verification_key) diff --git a/test/mpl/__init__.py b/test/mpl/__init__.py index 2a6c71715..d3f78d0bf 100644 --- a/test/mpl/__init__.py +++ b/test/mpl/__init__.py @@ -10,4 +10,7 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Module containing tests that REQUIRE the aws-cryptographic-material-providers library to run.""" +"""Module testing components that use the MPL. + +The aws-cryptographic-materials-library MUST be installed to run tests in this module. +""" \ No newline at end of file diff --git a/test/mpl/unit/test_material_managers_mpl_cmm.py b/test/mpl/unit/test_material_managers_mpl_cmm.py index a67c3e5c5..fa8f76410 100644 --- a/test/mpl/unit/test_material_managers_mpl_cmm.py +++ b/test/mpl/unit/test_material_managers_mpl_cmm.py @@ -10,7 +10,10 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" +"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic. + +The aws-cryptographic-materials-library MUST be installed to run tests in this module. +""" import pytest from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException diff --git a/test/mpl/unit/test_material_managers_mpl_materials.py b/test/mpl/unit/test_material_managers_mpl_materials.py index cb3ca7397..6c992ff24 100644 --- a/test/mpl/unit/test_material_managers_mpl_materials.py +++ b/test/mpl/unit/test_material_managers_mpl_materials.py @@ -10,7 +10,10 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" +"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.materials logic. + +The aws-cryptographic-materials-library MUST be installed to run tests in this module. +""" import pytest from aws_cryptographic_materialproviders.mpl.models import ( @@ -18,7 +21,7 @@ EncryptedDataKey as MPL_EncryptedDataKey, EncryptionMaterials as MPL_EncryptionMaterials, ) -from mock import MagicMock, PropertyMock, patch +from mock import MagicMock, patch from typing import Dict, List, Set import aws_encryption_sdk.materials_managers.mpl.materials From cf26ca3fdb0b2cccbe011d8c2071453fe06bbb09 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 17:45:15 -0800 Subject: [PATCH 127/376] fix --- .../materials_managers/mpl/cmm.py | 2 +- .../unit/test_material_managers_mpl_cmm.py | 68 ++++++++++++------- .../test_material_managers_mpl_materials.py | 18 ++--- 3 files changed, 53 insertions(+), 35 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index 760808bfe..e0879f3fb 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -2,6 +2,7 @@ The aws-cryptographic-materials-library MUST be installed to use this module. """ +from typing import List # pylint should pass even if the MPL isn't installed # Also thinks these imports aren't used if it can't import them # noqa pylint: disable=import-error,unused-import @@ -19,7 +20,6 @@ ICryptographicMaterialsManager as MPL_ICryptographicMaterialsManager, ) # noqa pylint: enable=import-error,unused-import -from typing import List from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError from aws_encryption_sdk.identifiers import CommitmentPolicy diff --git a/test/mpl/unit/test_material_managers_mpl_cmm.py b/test/mpl/unit/test_material_managers_mpl_cmm.py index fa8f76410..060f19f95 100644 --- a/test/mpl/unit/test_material_managers_mpl_cmm.py +++ b/test/mpl/unit/test_material_managers_mpl_cmm.py @@ -76,16 +76,17 @@ def test_GIVEN_invalid_mpl_cmm_WHEN_create_CryptoMaterialsManagerFromMPL_THEN_ra @patch.object(mock_mpl_cmm, "get_encryption_materials") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._native_to_mpl_get_encryption_materials") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL" + "._native_to_mpl_get_encryption_materials") def test_GIVEN_valid_request_WHEN_get_encryption_materials_THEN_return_EncryptionMaterialsFromMPL( mock_native_to_mpl_get_encryption_materials, mock_get_encryption_materials, ): - + # Given: _native_to_mpl_get_encryption_materials creates a MPL_GetEncryptionMaterialsInput mock_get_encryption_materials_input = MagicMock(__class__=MPL_GetEncryptionMaterialsInput) mock_native_to_mpl_get_encryption_materials.return_value = mock_get_encryption_materials_input - + # Given: mpl_cmm.get_encryption_materials returns mock MPL encryption materials mock_get_encryption_materials_output = MagicMock(__class__=MPL_GetEncryptionMaterialsOutput) mock_get_encryption_materials_output.encryption_materials = mock_mpl_encryption_materials @@ -104,7 +105,8 @@ def test_GIVEN_valid_request_WHEN_get_encryption_materials_THEN_return_Encryptio mock_mpl_cmm.get_encryption_materials.assert_called_once_with(mock_get_encryption_materials_input) -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL" + "._native_to_mpl_commmitment_policy") def test_GIVEN_mpl_cmm_raises_MPLException_WHEN_get_encryption_materials_THEN_raise_ESDKException( _ ): @@ -112,13 +114,15 @@ def test_GIVEN_mpl_cmm_raises_MPLException_WHEN_get_encryption_materials_THEN_ra with pytest.raises(AWSEncryptionSDKClientError): # Given: mpl_cmm.get_encryption_materials raises MPL exception with patch.object(mock_mpl_cmm, "get_encryption_materials", - side_effect=AwsCryptographicMaterialProvidersException("any")): + side_effect=AwsCryptographicMaterialProvidersException("any")): # When: get_encryption_materials cmm = CryptoMaterialsManagerFromMPL(mpl_cmm=mock_mpl_cmm) cmm.get_encryption_materials(mock_encryption_materials_request) -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy") -def test_GIVEN_valid_mpl_commitment_policy_WHEN_native_to_mpl_get_encryption_materials_THEN_returns_MPL_GetEncryptionMaterialsInput( + +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL" + "._native_to_mpl_commmitment_policy") +def test_GIVEN_valid_mpl_commitment_policy_WHEN_native_to_mpl_get_encryption_materials_THEN_returns_MPL_GetEncryptionMaterialsInput( # noqa: E501 mock_mpl_commitment_policy ): # Given: commitment policy is some MPL ESDK commitment policy @@ -126,7 +130,9 @@ def test_GIVEN_valid_mpl_commitment_policy_WHEN_native_to_mpl_get_encryption_mat mock_mpl_commitment_policy.return_value = mock_commitment_policy # When: _native_to_mpl_get_encryption_materials - output = CryptoMaterialsManagerFromMPL._native_to_mpl_get_encryption_materials(mock_encryption_materials_request) + output = CryptoMaterialsManagerFromMPL._native_to_mpl_get_encryption_materials( + mock_encryption_materials_request + ) # Then: returned MPL_GetEncryptionMaterialsInput is correct assert isinstance(output, MPL_GetEncryptionMaterialsInput) @@ -135,7 +141,7 @@ def test_GIVEN_valid_mpl_commitment_policy_WHEN_native_to_mpl_get_encryption_mat assert output.max_plaintext_length == mock_encryption_materials_request.plaintext_length -def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_MPL_CommitmentPolicyESDK_FORBID_ENCRYPT_ALLOW_DECRYPT(): +def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_MPL_CommitmentPolicyESDK_FORBID_ENCRYPT_ALLOW_DECRYPT(): # noqa: E501 # Given: native FORBID_ENCRYPT_ALLOW_DECRYPT native_commitment_policy = CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT @@ -146,7 +152,8 @@ def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_ assert isinstance(output, MPL_CommitmentPolicyESDK) assert output.value == "FORBID_ENCRYPT_ALLOW_DECRYPT" -def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_MPL_CommitmentPolicyESDK_REQUIRE_ENCRYPT_ALLOW_DECRYPT(): + +def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_MPL_CommitmentPolicyESDK_REQUIRE_ENCRYPT_ALLOW_DECRYPT(): # noqa: E501 # Given: native REQUIRE_ENCRYPT_ALLOW_DECRYPT native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT @@ -157,7 +164,8 @@ def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl assert isinstance(output, MPL_CommitmentPolicyESDK) assert output.value == "REQUIRE_ENCRYPT_ALLOW_DECRYPT" -def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_REQUIRE_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_MPL_CommitmentPolicyESDK_REQUIRE_ENCRYPT_REQUIRE_DECRYPT(): + +def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_REQUIRE_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_MPL_CommitmentPolicyESDK_REQUIRE_ENCRYPT_REQUIRE_DECRYPT(): # noqa: E501 # Given: native REQUIRE_ENCRYPT_REQUIRE_DECRYPT native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT @@ -168,6 +176,7 @@ def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_REQUIRE_DECRYPT_WHEN_native_to_m assert isinstance(output, MPL_CommitmentPolicyESDK) assert output.value == "REQUIRE_ENCRYPT_REQUIRE_DECRYPT" + def test_GIVEN_CommitmentPolicy_unrecognized_WHEN_native_to_mpl_commmitment_policy_THEN_raise_ValueError(): # Given: invalid native commitment policy native_commitment_policy = "not a commitment policy" @@ -177,13 +186,14 @@ def test_GIVEN_CommitmentPolicy_unrecognized_WHEN_native_to_mpl_commmitment_poli # When: _native_to_mpl_commmitment_policy CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy(native_commitment_policy) + @patch.object(mock_mpl_cmm, "decrypt_materials") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._create_mpl_decrypt_materials_input_from_request") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL" + "._create_mpl_decrypt_materials_input_from_request") def test_GIVEN_valid_request_WHEN_decrypt_materials_THEN_return_DecryptionMaterialsFromMPL( mock_native_to_mpl_decrypt_materials, mock_get_encryption_materials, ): - # Given: mpl_cmm.get_decryption_materials returns mock MPL decryption materials mock_decrypt_materials_output = MagicMock(__class__=MPL_GetEncryptionMaterialsOutput) mock_decrypt_materials_output.decryption_materials = mock_mpl_decrypt_materials @@ -205,7 +215,9 @@ def test_GIVEN_valid_request_WHEN_decrypt_materials_THEN_return_DecryptionMateri # Verify we actually called `decrypt_materials` mock_mpl_cmm.decrypt_materials.assert_called_once_with(mock_decrypt_materials_input) -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._create_mpl_decrypt_materials_input_from_request") + +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL" + "._create_mpl_decrypt_materials_input_from_request") def test_GIVEN_decrypt_materials_raises_MPL_Exception_WHEN_call_decrypt_materials_THEN_raise_ESDK_Exception( _ ): @@ -213,12 +225,13 @@ def test_GIVEN_decrypt_materials_raises_MPL_Exception_WHEN_call_decrypt_material with pytest.raises(AWSEncryptionSDKClientError): # Given: mpl_cmm.decrypt_materials raises MPL exception with patch.object(mock_mpl_cmm, "decrypt_materials", - side_effect=AwsCryptographicMaterialProvidersException("any")): + side_effect=AwsCryptographicMaterialProvidersException("any")): # When: decrypt_materials cmm = CryptoMaterialsManagerFromMPL(mpl_cmm=mock_mpl_cmm) cmm.decrypt_materials(mock_decryption_materials_request) -def test_GIVEN_valid_native_algorithm_id_WHEN_native_algorithm_id_to_mpl_algorithm_id_THEN_returns_valid_MPL_AlgorithmSuiteIdESDK(): + +def test_GIVEN_valid_native_algorithm_id_WHEN_native_algorithm_id_to_mpl_algorithm_id_THEN_returns_valid_MPL_AlgorithmSuiteIdESDK(): # noqa: E501 # Given: any native algorithm ID some_native_algorithm_id = 0x1234 # Not a real algorithm ID, but fits the format @@ -231,9 +244,12 @@ def test_GIVEN_valid_native_algorithm_id_WHEN_native_algorithm_id_to_mpl_algorit assert isinstance(mpl_output, MPL_AlgorithmSuiteIdESDK) assert mpl_output.value == "0x1234" -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._native_algorithm_id_to_mpl_algorithm_id") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy") -def test_GIVEN_valid_request_WHEN_create_mpl_decrypt_materials_input_from_request_THEN_returns_MPL_MPL_DecryptMaterialsInput( + +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL" + "._native_algorithm_id_to_mpl_algorithm_id") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL" + "._native_to_mpl_commmitment_policy") +def test_GIVEN_valid_request_WHEN_create_mpl_decrypt_materials_input_from_request_THEN_returns_MPL_MPL_DecryptMaterialsInput( # noqa: E501 mock_mpl_commitment_policy, mock_mpl_algorithm_id, ): @@ -245,17 +261,19 @@ def test_GIVEN_valid_request_WHEN_create_mpl_decrypt_materials_input_from_reques mock_commitment_policy = MagicMock(__class__=MPL_CommitmentPolicyESDK) mock_mpl_commitment_policy.return_value = mock_commitment_policy - no_mock_edks = [ mock_edk ] - one_mock_edk = [ mock_edk ] - two_mock_edks = [ mock_edk, mock_edk ] + no_mock_edks = [mock_edk] + one_mock_edk = [mock_edk] + two_mock_edks = [mock_edk, mock_edk] # Given: ESK lists of various lengths - for mock_edks in [ no_mock_edks, one_mock_edk, two_mock_edks ]: + for mock_edks in [no_mock_edks, one_mock_edk, two_mock_edks]: mock_decryption_materials_request.encrypted_data_keys = mock_edks # When: _create_mpl_decrypt_materials_input_from_request - output = CryptoMaterialsManagerFromMPL._create_mpl_decrypt_materials_input_from_request(mock_decryption_materials_request) + output = CryptoMaterialsManagerFromMPL._create_mpl_decrypt_materials_input_from_request( + mock_decryption_materials_request + ) # Then: # Verify general correctness of output structure @@ -273,4 +291,4 @@ def test_GIVEN_valid_request_WHEN_create_mpl_decrypt_materials_input_from_reques input_edk = mock_edks[i] assert output_edk.key_provider_id == input_edk.key_provider.provider_id assert output_edk.key_provider_info == input_edk.key_provider.key_info - assert output_edk.ciphertext == input_edk.encrypted_data_key \ No newline at end of file + assert output_edk.ciphertext == input_edk.encrypted_data_key diff --git a/test/mpl/unit/test_material_managers_mpl_materials.py b/test/mpl/unit/test_material_managers_mpl_materials.py index 6c992ff24..a2333f267 100644 --- a/test/mpl/unit/test_material_managers_mpl_materials.py +++ b/test/mpl/unit/test_material_managers_mpl_materials.py @@ -22,10 +22,10 @@ EncryptionMaterials as MPL_EncryptionMaterials, ) from mock import MagicMock, patch -from typing import Dict, List, Set +from typing import Dict import aws_encryption_sdk.materials_managers.mpl.materials -from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite +from aws_encryption_sdk.identifiers import AlgorithmSuite from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest from aws_encryption_sdk.materials_managers.mpl.materials import DecryptionMaterialsFromMPL, EncryptionMaterialsFromMPL @@ -48,11 +48,11 @@ mock_edk.ciphertext = mock_mpl_ciphertext -def test_GIVEN_valid_mpl_materials_WHEN_create_EncryptionMaterialsFromMPL_THEN_return_new_CryptoMaterialsManagerFromMPL(): +def test_GIVEN_mpl_materials_WHEN_create_EncryptionMaterialsFromMPL_THEN_return_new_CryptoMaterialsManagerFromMPL(): # Given: valid mpl_materials # When: create EncryptionMaterialsFromMPL mpl_encryption_materials = EncryptionMaterialsFromMPL(mpl_materials=mock_mpl_encryption_materials) - + # Then: EncryptionMaterialsFromMPL is valid assert mpl_encryption_materials.mpl_materials == mock_mpl_encryption_materials @@ -93,7 +93,7 @@ def test_GIVEN_valid_mpl_algorithm_id_WHEN_EncryptionMaterials_get_algorithm_THE # When: Get algorithm mpl_encryption_materials = EncryptionMaterialsFromMPL(mpl_materials=mock_mpl_encryption_materials) output = mpl_encryption_materials.algorithm - + # Then: output is valid assert output == mock_algorithm() # property calls automatically, we need to call the mock @@ -112,12 +112,12 @@ def test_GIVEN_valid_encryption_context_WHEN_EncryptionMaterials_get_encryption_ def test_GIVEN_valid_edks_WHEN_EncryptionMaterials_get_edks_THEN_returns_edks(): - + # Given: lists of mocked EDKs of various lengths no_mock_edks = [] - one_mock_edk = [ mock_edk ] - two_mocked_edks = [ mock_edk, mock_edk ] - for mock_edks in [ no_mock_edks, one_mock_edk, two_mocked_edks ]: + one_mock_edk = [mock_edk] + two_mocked_edks = [mock_edk, mock_edk] + for mock_edks in [no_mock_edks, one_mock_edk, two_mocked_edks]: mock_mpl_encryption_materials.encrypted_data_keys = mock_edks # When: get EDKs From 7f27ebdb1cbe0a7b7039d8912ffb8deed032a5e2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 17:46:38 -0800 Subject: [PATCH 128/376] fix --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 3daa40e47..952d3c24f 100644 --- a/tox.ini +++ b/tox.ini @@ -112,7 +112,7 @@ passenv = setenv = ######################################################### deps = -rdev_requirements/test-requirements.txt -commands = {[testenv:base-command]commands} test/ -m local --ignore test/unit/mpl/ +commands = {[testenv:base-command]commands} test/ -m local --ignore test/mpl/ # Collect requirements for use in upstream tests [testenv:freeze-upstream-requirements-base] @@ -144,7 +144,7 @@ commands = {[testenv:freeze-upstream-requirements-base]commands} test/upstream-r [testenv:test-upstream-requirements-base] sitepackages = False recreate = True -commands = {[testenv:base-command]commands} test/ -m local --ignore test/unit/mpl/ +commands = {[testenv:base-command]commands} test/ -m local --ignore --ignore test/mpl/ # Test frozen upstream requirements for Python 3.7 [testenv:test-upstream-requirements-py37] From 00f4721542d8529eb2ca6b0ac621b440940055be Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 17:48:24 -0800 Subject: [PATCH 129/376] fix --- src/aws_encryption_sdk/materials_managers/mpl/materials.py | 2 +- test/mpl/__init__.py | 2 +- tox.ini | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index faa47cb46..39aff2c3c 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -2,6 +2,7 @@ The aws-cryptographic-materials-library MUST be installed to use this module. """ +from typing import Dict, List, Set # pylint should pass even if the MPL isn't installed # noqa pylint: disable=import-error from aws_cryptographic_materialproviders.mpl.models import ( @@ -10,7 +11,6 @@ EncryptionMaterials as MPL_EncryptionMaterials, ) # noqa pylint: enable=import-error -from typing import Dict, List, Set from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite from aws_encryption_sdk.materials_managers import ( diff --git a/test/mpl/__init__.py b/test/mpl/__init__.py index d3f78d0bf..37f482e0b 100644 --- a/test/mpl/__init__.py +++ b/test/mpl/__init__.py @@ -13,4 +13,4 @@ """Module testing components that use the MPL. The aws-cryptographic-materials-library MUST be installed to run tests in this module. -""" \ No newline at end of file +""" diff --git a/tox.ini b/tox.ini index 952d3c24f..3644c973a 100644 --- a/tox.ini +++ b/tox.ini @@ -144,7 +144,7 @@ commands = {[testenv:freeze-upstream-requirements-base]commands} test/upstream-r [testenv:test-upstream-requirements-base] sitepackages = False recreate = True -commands = {[testenv:base-command]commands} test/ -m local --ignore --ignore test/mpl/ +commands = {[testenv:base-command]commands} test/ -m local --ignore test/mpl/ # Test frozen upstream requirements for Python 3.7 [testenv:test-upstream-requirements-py37] From 018b93f3ed2b5eae5bf6c2e4dc5d7837e38a34a3 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 17:55:46 -0800 Subject: [PATCH 130/376] fix --- src/aws_encryption_sdk/materials_managers/mpl/cmm.py | 7 ++++++- src/aws_encryption_sdk/materials_managers/mpl/materials.py | 4 +++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index e0879f3fb..c262cf7ce 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -2,7 +2,6 @@ The aws-cryptographic-materials-library MUST be installed to use this module. """ -from typing import List # pylint should pass even if the MPL isn't installed # Also thinks these imports aren't used if it can't import them # noqa pylint: disable=import-error,unused-import @@ -21,6 +20,9 @@ ) # noqa pylint: enable=import-error,unused-import +# pylint and isort disagree on where this should go. Choose isort and disable pylint for this. +from typing import List # noqa pylint: disable=wrong-import-order + from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError from aws_encryption_sdk.identifiers import CommitmentPolicy from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest @@ -28,6 +30,9 @@ from aws_encryption_sdk.materials_managers.mpl.materials import DecryptionMaterialsFromMPL, EncryptionMaterialsFromMPL from aws_encryption_sdk.structures import EncryptedDataKey as Native_EncryptedDataKey +# noqa pylint: enable=import-error,unused-import + + class CryptoMaterialsManagerFromMPL(CryptoMaterialsManager): """ diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index 39aff2c3c..43579fac6 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -2,7 +2,6 @@ The aws-cryptographic-materials-library MUST be installed to use this module. """ -from typing import Dict, List, Set # pylint should pass even if the MPL isn't installed # noqa pylint: disable=import-error from aws_cryptographic_materialproviders.mpl.models import ( @@ -12,6 +11,9 @@ ) # noqa pylint: enable=import-error +# pylint and isort disagree on where this should go. Choose isort and disable pylint for this. +from typing import Dict, List, Set # noqa pylint: disable=wrong-import-order + from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite from aws_encryption_sdk.materials_managers import ( DecryptionMaterials as Native_DecryptionMaterials, From d413b65024d398510f1811c8638109ae7c886336 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 18:00:04 -0800 Subject: [PATCH 131/376] fix --- src/aws_encryption_sdk/materials_managers/mpl/cmm.py | 4 ---- src/aws_encryption_sdk/materials_managers/mpl/materials.py | 2 -- 2 files changed, 6 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index c262cf7ce..a0119a588 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -19,7 +19,6 @@ ICryptographicMaterialsManager as MPL_ICryptographicMaterialsManager, ) # noqa pylint: enable=import-error,unused-import - # pylint and isort disagree on where this should go. Choose isort and disable pylint for this. from typing import List # noqa pylint: disable=wrong-import-order @@ -30,9 +29,6 @@ from aws_encryption_sdk.materials_managers.mpl.materials import DecryptionMaterialsFromMPL, EncryptionMaterialsFromMPL from aws_encryption_sdk.structures import EncryptedDataKey as Native_EncryptedDataKey -# noqa pylint: enable=import-error,unused-import - - class CryptoMaterialsManagerFromMPL(CryptoMaterialsManager): """ diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index 43579fac6..4508d5545 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -9,8 +9,6 @@ EncryptedDataKey as MPL_EncryptedDataKey, EncryptionMaterials as MPL_EncryptionMaterials, ) -# noqa pylint: enable=import-error - # pylint and isort disagree on where this should go. Choose isort and disable pylint for this. from typing import Dict, List, Set # noqa pylint: disable=wrong-import-order From c4ca658d08efdac51f85c04b9aa9cd5cf37a3a60 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 18:09:29 -0800 Subject: [PATCH 132/376] copyright --- examples/src/keyrings/__init__.py | 14 ++------------ .../src/keyrings/example_branch_key_id_supplier.py | 2 ++ examples/test/keyrings/__init__.py | 14 ++------------ .../test/keyrings/test_i_hierarchical_keyring.py | 2 ++ .../materials_managers/mpl/__init__.py | 14 ++------------ .../materials_managers/mpl/cmm.py | 2 ++ .../materials_managers/mpl/materials.py | 2 ++ test/mpl/__init__.py | 14 ++------------ test/mpl/unit/test_material_managers_mpl_cmm.py | 14 ++------------ .../unit/test_material_managers_mpl_materials.py | 14 ++------------ test/unit/test_streaming_client_mpl_import.py | 14 ++------------ 11 files changed, 22 insertions(+), 84 deletions(-) diff --git a/examples/src/keyrings/__init__.py b/examples/src/keyrings/__init__.py index e8fd618b1..120179eda 100644 --- a/examples/src/keyrings/__init__.py +++ b/examples/src/keyrings/__init__.py @@ -1,13 +1,3 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 """Stub module indicator to make linter configuration simpler.""" diff --git a/examples/src/keyrings/example_branch_key_id_supplier.py b/examples/src/keyrings/example_branch_key_id_supplier.py index ba9ae060c..7b390cdda 100644 --- a/examples/src/keyrings/example_branch_key_id_supplier.py +++ b/examples/src/keyrings/example_branch_key_id_supplier.py @@ -1,3 +1,5 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 """Example implementation of a branch key ID supplier.""" from aws_cryptographic_materialproviders.mpl.models import GetBranchKeyIdInput, GetBranchKeyIdOutput diff --git a/examples/test/keyrings/__init__.py b/examples/test/keyrings/__init__.py index e8fd618b1..120179eda 100644 --- a/examples/test/keyrings/__init__.py +++ b/examples/test/keyrings/__init__.py @@ -1,13 +1,3 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 """Stub module indicator to make linter configuration simpler.""" diff --git a/examples/test/keyrings/test_i_hierarchical_keyring.py b/examples/test/keyrings/test_i_hierarchical_keyring.py index d80bb565d..4cae478d7 100644 --- a/examples/test/keyrings/test_i_hierarchical_keyring.py +++ b/examples/test/keyrings/test_i_hierarchical_keyring.py @@ -1,3 +1,5 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 """Unit test suite for the hierarchical keyring example.""" import pytest diff --git a/src/aws_encryption_sdk/materials_managers/mpl/__init__.py b/src/aws_encryption_sdk/materials_managers/mpl/__init__.py index 7593a3300..be75f3566 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/__init__.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/__init__.py @@ -1,15 +1,5 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 """Modules related to the MPL's materials managers interfaces. The aws-cryptographic-materials-library MUST be installed to use these modules. diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index a0119a588..53a4b3505 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -1,3 +1,5 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 """Retrieves encryption/decryption materials from the MPL and interfaces them to EDK components. The aws-cryptographic-materials-library MUST be installed to use this module. diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index 4508d5545..dfd1bd6fc 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -1,3 +1,5 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 """Provides encryption/decryption materials from an underlying materials provider from the MPL. The aws-cryptographic-materials-library MUST be installed to use this module. diff --git a/test/mpl/__init__.py b/test/mpl/__init__.py index 37f482e0b..79522d342 100644 --- a/test/mpl/__init__.py +++ b/test/mpl/__init__.py @@ -1,15 +1,5 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 """Module testing components that use the MPL. The aws-cryptographic-materials-library MUST be installed to run tests in this module. diff --git a/test/mpl/unit/test_material_managers_mpl_cmm.py b/test/mpl/unit/test_material_managers_mpl_cmm.py index 060f19f95..80d6f00ee 100644 --- a/test/mpl/unit/test_material_managers_mpl_cmm.py +++ b/test/mpl/unit/test_material_managers_mpl_cmm.py @@ -1,15 +1,5 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 """Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic. The aws-cryptographic-materials-library MUST be installed to run tests in this module. diff --git a/test/mpl/unit/test_material_managers_mpl_materials.py b/test/mpl/unit/test_material_managers_mpl_materials.py index a2333f267..9e76556a2 100644 --- a/test/mpl/unit/test_material_managers_mpl_materials.py +++ b/test/mpl/unit/test_material_managers_mpl_materials.py @@ -1,15 +1,5 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 """Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.materials logic. The aws-cryptographic-materials-library MUST be installed to run tests in this module. diff --git a/test/unit/test_streaming_client_mpl_import.py b/test/unit/test_streaming_client_mpl_import.py index a4ca87e2a..638b04fd6 100644 --- a/test/unit/test_streaming_client_mpl_import.py +++ b/test/unit/test_streaming_client_mpl_import.py @@ -1,15 +1,5 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 """Unit test suite to validate aws_encryption_sdk.streaming_client MPL import logic.""" import pytest From d99b6667bb1a7d65a36af598889edeab2beecfc6 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 26 Feb 2024 12:40:24 -0800 Subject: [PATCH 133/376] more unit tests --- src/aws_encryption_sdk/streaming_client.py | 34 +++-- .../unit/test_crypto_authentication_signer.py | 63 +++++++-- test/unit/test_streaming_client_configs.py | 96 +++++++++++++ .../test_streaming_client_stream_decryptor.py | 132 +++++++++++++++++- .../test_streaming_client_stream_encryptor.py | 79 +++++++++++ test/unit/test_utils.py | 25 ++++ 6 files changed, 401 insertions(+), 28 deletions(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 959b5ff0b..2cfcc9a02 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -73,8 +73,9 @@ try: # pylint should pass even if the MPL isn't installed # noqa pylint: disable=import-error - from aws_cryptographic_materialproviders.mpl.client import AwsCryptographicMaterialProviders + from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig + from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException from aws_cryptographic_materialproviders.mpl.models import CreateDefaultCryptographicMaterialsManagerInput from aws_cryptographic_materialproviders.mpl.references import IKeyring _HAS_MPL = True @@ -147,9 +148,6 @@ def _has_mpl_attrs_post_init(self): """If the MPL is present in the runtime, perform MPL-specific post-init logic to validate the new object has a valid state. """ - if not hasattr(self, "keyring"): - self._no_mpl_attrs_post_init() - return if not exactly_one_arg_is_not_none(self.materials_manager, self.key_provider, self.keyring): raise TypeError("Exactly one of keyring, materials_manager, or key_provider must be provided") if self.materials_manager is None: @@ -159,21 +157,21 @@ def _has_mpl_attrs_post_init(self): master_key_provider=self.key_provider ) elif self.keyring is not None: - # No CMM, provided MPL keyring => create MPL's DefaultCryptographicMaterialsManager - if not isinstance(self.keyring, IKeyring): - raise ValueError(f"Argument provided to keyring MUST be a {IKeyring}. \ - Found {self.keyring.__class__.__name__}") - - mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( - config=MaterialProvidersConfig() - ) - cmm = mat_prov.create_default_cryptographic_materials_manager( - CreateDefaultCryptographicMaterialsManagerInput( - keyring=self.keyring + try: + mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + config=MaterialProvidersConfig() ) - ) - cmm_handler: CryptoMaterialsManager = CryptoMaterialsManagerFromMPL(cmm) - self.materials_manager = cmm_handler + cmm = mat_prov.create_default_cryptographic_materials_manager( + CreateDefaultCryptographicMaterialsManagerInput( + keyring=self.keyring + ) + ) + cmm_handler: CryptoMaterialsManager = CryptoMaterialsManagerFromMPL(cmm) + self.materials_manager = cmm_handler + except AwsCryptographicMaterialProvidersException as mpl_exception: + # Wrap MPL error into the ESDK error type + # so customers only have to catch ESDK error types. + raise AWSEncryptionSDKClientError(mpl_exception) def _no_mpl_attrs_post_init(self): """If the MPL is NOT present in the runtime, perform post-init logic diff --git a/test/unit/test_crypto_authentication_signer.py b/test/unit/test_crypto_authentication_signer.py index bd7227fd3..c37c97bde 100644 --- a/test/unit/test_crypto_authentication_signer.py +++ b/test/unit/test_crypto_authentication_signer.py @@ -12,7 +12,8 @@ # language governing permissions and limitations under the License. """Unit test suite for ``aws_encryption_sdk.internal.crypto.authentication.Signer``.""" import pytest -from mock import MagicMock, sentinel +from mock import MagicMock, sentinel, patch +import cryptography.hazmat.primitives.serialization from pytest_mock import mocker # noqa pylint: disable=unused-import import aws_encryption_sdk.internal.crypto.authentication @@ -75,28 +76,72 @@ def test_f_signer_from_key_bytes(): def test_f_signer_key_bytes(): test = Signer(algorithm=ALGORITHM, key=VALUES["ecc_private_key_prime"]) assert test.key_bytes() == VALUES["ecc_private_key_prime_private_bytes"] + +def test_GIVEN_no_encoding_WHEN_signer_from_key_bytes_THEN_load_der_private_key( + patch_default_backend, + patch_build_hasher, + patch_ec +): + mock_algorithm_info = MagicMock(return_value=sentinel.algorithm_info, spec=patch_ec.EllipticCurve) + _algorithm = MagicMock(signing_algorithm_info=mock_algorithm_info) -def test_signer_from_key_bytes(patch_default_backend, patch_serialization, patch_build_hasher, patch_ec): + # Make a new patched serialization module for this test. + # The default patch introduces serialization as `serialization.Encoding.DER` + # from within the src, but is `Encoding.DER` in the test. + # This namespace change causes the src's `isinstance` checks to fail. + # Mock the `serialization.Encoding.DER` + with patch.object(cryptography.hazmat.primitives, "serialization"): + # Mock the `serialization.load_der_private_key` + with patch.object(aws_encryption_sdk.internal.crypto.authentication.serialization, "load_der_private_key") as mock_der: + Signer.from_key_bytes( + algorithm=_algorithm, + key_bytes=sentinel.key_bytes, + ) + + mock_der.assert_called_once_with( + data=sentinel.key_bytes, password=None, backend=patch_default_backend.return_value + ) + + +def test_GIVEN_PEM_encoding_WHEN_signer_from_key_bytes_THEN_load_pem_private_key( + patch_default_backend, + patch_serialization, + patch_build_hasher, + patch_ec +): mock_algorithm_info = MagicMock(return_value=sentinel.algorithm_info, spec=patch_ec.EllipticCurve) _algorithm = MagicMock(signing_algorithm_info=mock_algorithm_info) - # Explicitly pass in patched serialization module. - # Patching the module introduces namespace issues - # which causes the method's `isinstance` checks to fail - # by changing the namespace from `serialization.Encoding.DER` to `Encoding.DER`. signer = Signer.from_key_bytes( algorithm=_algorithm, key_bytes=sentinel.key_bytes, - encoding=patch_serialization.Encoding.DER + encoding=patch_serialization.Encoding.PEM ) - patch_serialization.load_der_private_key.assert_called_once_with( + patch_serialization.load_pem_private_key.assert_called_once_with( data=sentinel.key_bytes, password=None, backend=patch_default_backend.return_value ) assert isinstance(signer, Signer) assert signer.algorithm is _algorithm - assert signer.key is patch_serialization.load_der_private_key.return_value + assert signer.key is patch_serialization.load_pem_private_key.return_value + + +def test_GIVEN_unrecognized_encoding_WHEN_signer_from_key_bytes_THEN_raise_ValueError( + patch_default_backend, + patch_serialization, + patch_build_hasher, + patch_ec +): + mock_algorithm_info = MagicMock(return_value=sentinel.algorithm_info, spec=patch_ec.EllipticCurve) + _algorithm = MagicMock(signing_algorithm_info=mock_algorithm_info) + + with pytest.raises(ValueError): + signer = Signer.from_key_bytes( + algorithm=_algorithm, + key_bytes=sentinel.key_bytes, + encoding="not an encoding" + ) def test_signer_key_bytes(patch_default_backend, patch_serialization, patch_build_hasher, patch_ec): diff --git a/test/unit/test_streaming_client_configs.py b/test/unit/test_streaming_client_configs.py index 426f8f85f..c76a64ea7 100644 --- a/test/unit/test_streaming_client_configs.py +++ b/test/unit/test_streaming_client_configs.py @@ -15,6 +15,7 @@ import pytest import six +from mock import patch from aws_encryption_sdk import CommitmentPolicy from aws_encryption_sdk.internal.defaults import ALGORITHM, FRAME_LENGTH, LINE_LENGTH @@ -28,6 +29,22 @@ pytestmark = [pytest.mark.unit, pytest.mark.local] +# Check if MPL is installed, and skip tests based on its installation status +# Ideally, this logic would be based on mocking imports and testing logic, +# but doing that introduces errors that cause other tests to fail. +try: + from aws_cryptographic_materialproviders.mpl.references import ( + IKeyring, + ) + HAS_MPL = True + + from aws_encryption_sdk.materials_managers.mpl.cmm import ( + CryptoMaterialsManagerFromMPL, + ) +except ImportError: + HAS_MPL = False + + class FakeCryptoMaterialsManager(CryptoMaterialsManager): def get_encryption_materials(self, request): return @@ -42,6 +59,14 @@ class FakeMasterKeyProvider(MasterKeyProvider): def _new_master_key(self, key_id): return + +if HAS_MPL: + class FakeKeyring(IKeyring): + def on_encrypt(self, param): + return + + def on_decrypt(self, param): + return BASE_KWARGS = dict( @@ -126,6 +151,18 @@ def test_client_config_defaults(): assert test.max_encrypted_data_keys is None +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_client_config_with_mpl_attr(): + test = _ClientConfig(**BASE_KWARGS) + assert hasattr(test, "keyring") + + +@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") +def test_client_config_no_mpl(): + test = _ClientConfig(**BASE_KWARGS) + assert not hasattr(test, "keyring") + + def test_encryptor_config_defaults(): test = EncryptorConfig(**BASE_KWARGS) assert test.encryption_context == {} @@ -154,3 +191,62 @@ def test_client_config_converts(kwargs, stream_type): assert isinstance(test.source, stream_type) if test.key_provider is not None: assert isinstance(test.materials_manager, DefaultCryptoMaterialsManager) + + +@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") +@patch.object(_ClientConfig, "_no_mpl_attrs_post_init") +def test_GIVEN_no_mpl_WHEN_attrs_post_init_THEN_calls_no_mpl_method( + mock_no_mpl_attrs_post_init, +): + _ClientConfig(**BASE_KWARGS) + mock_no_mpl_attrs_post_init.assert_called_once_with() + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +@patch.object(_ClientConfig, "_has_mpl_attrs_post_init") +def test_GIVEN_has_mpl_WHEN_attrs_post_init_THEN_calls_no_mpl_method( + _has_mpl_attrs_post_init, +): + _ClientConfig(**BASE_KWARGS) + _has_mpl_attrs_post_init.assert_called_once_with() + + +@pytest.mark.parametrize( + "kwargs, stream_type", + ( + (dict(source=b"", materials_manager=FakeCryptoMaterialsManager()), io.BytesIO), + (dict(source=b"", key_provider=FakeMasterKeyProvider()), io.BytesIO), + (dict(source="", materials_manager=FakeCryptoMaterialsManager()), io.BytesIO), + (dict(source=io.BytesIO(), materials_manager=FakeCryptoMaterialsManager()), io.BytesIO), + (dict(source=six.StringIO(), materials_manager=FakeCryptoMaterialsManager()), six.StringIO), + (dict(source=b"", keyring=FakeKeyring()), io.BytesIO), + ), +) +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_client_configs_with_mpl( + kwargs, + stream_type +): + kwargs["commitment_policy"] = CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT + + test = _ClientConfig(**kwargs) + + # In all cases, config should have a materials manager + assert test.materials_manager is not None + + # If materials manager was provided, it should be directly used + if hasattr(kwargs, "materials_manager"): + assert kwargs["materials_manager"] == test.materials_manager + + # If MPL keyring was provided, it should be wrapped in MPL materials manager + if hasattr(kwargs, "keyring"): + assert test.keyring is not None + assert test.keyring == kwargs["keyring"] + assert isinstance(test.keyring, IKeyring) + assert isinstance(test.materials_manager, CryptoMaterialsManagerFromMPL) + + # If native key_provider was provided, it should be wrapped in native materials manager + if hasattr(kwargs, "key_provider"): + assert test.key_provider is not None + assert test.key_provider == kwargs["key_provider"] + assert isinstance(test.materials_manager, DefaultCryptoMaterialsManager) diff --git a/test/unit/test_streaming_client_stream_decryptor.py b/test/unit/test_streaming_client_stream_decryptor.py index 157755094..c8a17e650 100644 --- a/test/unit/test_streaming_client_stream_decryptor.py +++ b/test/unit/test_streaming_client_stream_decryptor.py @@ -33,14 +33,36 @@ pytestmark = [pytest.mark.unit, pytest.mark.local] +# Check if MPL is installed, and skip tests based on its installation status +# Ideally, this logic would be based on mocking imports and testing logic, +# but doing that introduces errors that cause other tests to fail. +try: + from aws_cryptographic_materialproviders.mpl.references import ( + IKeyring, + ) + HAS_MPL = True + + from aws_encryption_sdk.materials_managers.mpl.cmm import ( + CryptoMaterialsManagerFromMPL, + ) +except ImportError: + HAS_MPL = False + + class TestStreamDecryptor(object): @pytest.fixture(autouse=True) def apply_fixtures(self): self.mock_key_provider = MagicMock(__class__=MasterKeyProvider) self.mock_materials_manager = MagicMock(__class__=CryptoMaterialsManager) - self.mock_materials_manager.decrypt_materials.return_value = MagicMock( + self.mock_decrypt_materials = MagicMock( data_key=VALUES["data_key_obj"], verification_key=sentinel.verification_key ) + self.mock_materials_manager.decrypt_materials.return_value = self.mock_decrypt_materials + + if HAS_MPL: + self.mock_mpl_materials_manager = MagicMock(__class__=CryptoMaterialsManagerFromMPL) + self.mock_mpl_materials_manager.decrypt_materials.return_value = self.mock_decrypt_materials + self.mock_header = MagicMock() self.mock_header.version = SerializationVersion.V1 self.mock_header.algorithm = MagicMock( @@ -213,6 +235,114 @@ def test_read_header(self, mock_derive_datakey, mock_decrypt_materials_request, assert test_header is self.mock_header assert test_header_auth is sentinel.header_auth + @patch("aws_encryption_sdk.streaming_client.DecryptionMaterialsRequest") + @patch("aws_encryption_sdk.streaming_client.derive_data_encryption_key") + @patch("aws_encryption_sdk.streaming_client.Verifier") + @pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") + def test_GIVEN_verification_key_AND_no_mpl_WHEN_read_header_THEN_calls_from_key_bytes( + self, + mock_verifier, + *_, + ): + mock_verifier_instance = MagicMock() + mock_verifier.from_key_bytes.return_value = mock_verifier_instance + ct_stream = io.BytesIO(VALUES["data_128"]) + mock_commitment_policy = MagicMock(__class__=CommitmentPolicy) + test_decryptor = StreamDecryptor( + materials_manager=self.mock_materials_manager, + source=ct_stream, + commitment_policy=mock_commitment_policy, + ) + test_decryptor.source_stream = ct_stream + test_decryptor._stream_length = len(VALUES["data_128"]) + + test_decryptor._read_header() + + mock_verifier.from_key_bytes.assert_called_once_with( + algorithm=self.mock_header.algorithm, key_bytes=sentinel.verification_key + ) + + @patch("aws_encryption_sdk.streaming_client.DecryptionMaterialsRequest") + @patch("aws_encryption_sdk.streaming_client.derive_data_encryption_key") + @patch("aws_encryption_sdk.streaming_client.Verifier") + @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") + def test_GIVEN_verification_key_AND_has_mpl_AND_not_MPLCMM_WHEN_read_header_THEN_calls_from_key_bytes( + self, + mock_verifier, + *_, + ): + mock_verifier_instance = MagicMock() + mock_verifier.from_key_bytes.return_value = mock_verifier_instance + ct_stream = io.BytesIO(VALUES["data_128"]) + mock_commitment_policy = MagicMock(__class__=CommitmentPolicy) + test_decryptor = StreamDecryptor( + materials_manager=self.mock_materials_manager, + source=ct_stream, + commitment_policy=mock_commitment_policy, + ) + test_decryptor.source_stream = ct_stream + test_decryptor._stream_length = len(VALUES["data_128"]) + + test_decryptor._read_header() + + mock_verifier.from_key_bytes.assert_called_once_with( + algorithm=self.mock_header.algorithm, key_bytes=sentinel.verification_key + ) + + @patch("aws_encryption_sdk.streaming_client.DecryptionMaterialsRequest") + @patch("aws_encryption_sdk.streaming_client.derive_data_encryption_key") + @patch("aws_encryption_sdk.streaming_client.Verifier") + @patch("base64.b64encode") + @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") + def test_GIVEN_verification_key_AND_has_mpl_AND_has_MPLCMM_WHEN_read_header_THEN_calls_from_encoded_point( + self, + mock_b64encoding, + mock_verifier, + *_, + ): + mock_verifier_instance = MagicMock() + mock_verifier.from_key_bytes.return_value = mock_verifier_instance + ct_stream = io.BytesIO(VALUES["data_128"]) + mock_commitment_policy = MagicMock(__class__=CommitmentPolicy) + test_decryptor = StreamDecryptor( + materials_manager=self.mock_mpl_materials_manager, + source=ct_stream, + commitment_policy=mock_commitment_policy, + ) + test_decryptor.source_stream = ct_stream + test_decryptor._stream_length = len(VALUES["data_128"]) + + test_decryptor._read_header() + + mock_verifier.from_encoded_point.assert_called_once_with( + algorithm=self.mock_header.algorithm, encoded_point=mock_b64encoding() + ) + + # @patch("aws_encryption_sdk.streaming_client.Verifier") + # @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") + # def test_GIVEN_verification_key_AND_has_mpl_AND_not_MPLCMM_WHEN_read_header_THEN_calls_from_key_bytes( + # self, + # mock_verifier, + # ): + # mock_verifier_instance = MagicMock() + # mock_verifier.from_key_bytes.return_value = mock_verifier_instance + # ct_stream = io.BytesIO(VALUES["data_128"]) + # mock_commitment_policy = MagicMock(__class__=CommitmentPolicy) + # test_decryptor = StreamDecryptor( + # materials_manager=self.mock_materials_manager, + # source=ct_stream, + # commitment_policy=mock_commitment_policy, + # ) + # test_decryptor.source_stream = ct_stream + # test_decryptor._stream_length = len(VALUES["data_128"]) + + # test_decryptor._read_header() + + # mock_verifier.from_key_bytes.assert_called_once_with( + # algorithm=self.mock_header.algorithm, key_bytes=sentinel.verification_key + # ) + + @patch("aws_encryption_sdk.streaming_client.derive_data_encryption_key") def test_read_header_frame_too_large(self, mock_derive_datakey): self.mock_header.content_type = ContentType.FRAMED_DATA diff --git a/test/unit/test_streaming_client_stream_encryptor.py b/test/unit/test_streaming_client_stream_encryptor.py index 5bfd0c903..11664411a 100644 --- a/test/unit/test_streaming_client_stream_encryptor.py +++ b/test/unit/test_streaming_client_stream_encryptor.py @@ -13,6 +13,7 @@ """Unit test suite for aws_encryption_sdk.streaming_client.StreamEncryptor""" import io +from cryptography.hazmat.primitives import serialization import pytest import six from mock import MagicMock, call, patch, sentinel @@ -37,6 +38,22 @@ pytestmark = [pytest.mark.unit, pytest.mark.local] +# Check if MPL is installed, and skip tests based on its installation status +# Ideally, this logic would be based on mocking imports and testing logic, +# but doing that introduces errors that cause other tests to fail. +try: + from aws_cryptographic_materialproviders.mpl.references import ( + IKeyring, + ) + HAS_MPL = True + + from aws_encryption_sdk.materials_managers.mpl.cmm import ( + CryptoMaterialsManagerFromMPL, + ) +except ImportError: + HAS_MPL = False + + class TestStreamEncryptor(object): @pytest.fixture(autouse=True) def apply_fixtures(self): @@ -60,6 +77,10 @@ def apply_fixtures(self): self.mock_master_keys_set, ) + if HAS_MPL: + self.mock_mpl_materials_manager = MagicMock(__class__=CryptoMaterialsManagerFromMPL) + self.mock_mpl_materials_manager.get_encryption_materials.return_value = self.mock_encryption_materials + self.mock_master_key = MagicMock(__class__=MasterKey) self.mock_frame_length = MagicMock(__class__=int) @@ -366,6 +387,64 @@ def test_prep_message_non_framed_message(self, mock_write_header, mock_prep_non_ test_encryptor._prep_message() mock_prep_non_framed.assert_called_once_with() + @pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") + def test_GIVEN_no_mpl_AND_uses_signer_WHEN_prep_message_THEN_signer_uses_default_encoding(self): + self.mock_encryption_materials.algorithm = Algorithm.AES_128_GCM_IV12_TAG16 + test_encryptor = StreamEncryptor( + source=VALUES["data_128"], + materials_manager=self.mock_materials_manager, + frame_length=self.mock_frame_length, + algorithm=Algorithm.AES_128_GCM_IV12_TAG16, + commitment_policy=self.mock_commitment_policy, + signature_policy=self.mock_signature_policy, + ) + test_encryptor.content_type = ContentType.FRAMED_DATA + with patch.object(self.mock_signer, "from_key_bytes"): + test_encryptor._prep_message() + self.mock_signer.from_key_bytes.assert_called_once_with( + algorithm=self.mock_encryption_materials.algorithm, + key_bytes=self.mock_encryption_materials.signing_key + ) + + @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") + def test_GIVEN_has_mpl_AND_not_MPLCMM_AND_uses_signer_WHEN_prep_message_THEN_signer_uses_default_encoding(self): + self.mock_encryption_materials.algorithm = Algorithm.AES_128_GCM_IV12_TAG16 + test_encryptor = StreamEncryptor( + source=VALUES["data_128"], + materials_manager=self.mock_materials_manager, + frame_length=self.mock_frame_length, + algorithm=Algorithm.AES_128_GCM_IV12_TAG16, + commitment_policy=self.mock_commitment_policy, + signature_policy=self.mock_signature_policy, + ) + test_encryptor.content_type = ContentType.FRAMED_DATA + with patch.object(self.mock_signer, "from_key_bytes"): + test_encryptor._prep_message() + self.mock_signer.from_key_bytes.assert_called_once_with( + algorithm=self.mock_encryption_materials.algorithm, + key_bytes=self.mock_encryption_materials.signing_key + ) + + @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") + def test_GIVEN_has_mpl_AND_has_MPLCMM_AND_uses_signer_WHEN_prep_message_THEN_signer_uses_default_encoding(self): + self.mock_encryption_materials.algorithm = Algorithm.AES_128_GCM_IV12_TAG16 + test_encryptor = StreamEncryptor( + source=VALUES["data_128"], + materials_manager=self.mock_mpl_materials_manager, + frame_length=self.mock_frame_length, + algorithm=Algorithm.AES_128_GCM_IV12_TAG16, + commitment_policy=self.mock_commitment_policy, + signature_policy=self.mock_signature_policy, + ) + test_encryptor.content_type = ContentType.FRAMED_DATA + with patch.object(self.mock_signer, "from_key_bytes"): + test_encryptor._prep_message() + self.mock_signer.from_key_bytes.assert_called_once_with( + algorithm=self.mock_encryption_materials.algorithm, + key_bytes=self.mock_encryption_materials.signing_key, + encoding=serialization.Encoding.PEM + ) + def test_prep_message_no_signer(self): self.mock_encryption_materials.algorithm = Algorithm.AES_128_GCM_IV12_TAG16 test_encryptor = StreamEncryptor( diff --git a/test/unit/test_utils.py b/test/unit/test_utils.py index c6d565108..d717b51c7 100644 --- a/test/unit/test_utils.py +++ b/test/unit/test_utils.py @@ -265,3 +265,28 @@ def test_source_data_key_length_check_invalid(self): source_data_key=mock_data_key, algorithm=mock_algorithm ) excinfo.match("Invalid Source Data Key length 4 for algorithm required: 5") + + def test_exactly_one_arg_is_not_none(self): + # No args => no args are not None + assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none() is False + assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none( + None + ) is False + assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none( + "not None" + ) is True + assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none( + "not None", "also not None" + ) is False + assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none( + "not None", None + ) is True + assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none( + "not None", "also not None" + ) is False + assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none( + None, "not None" + ) is True + assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none( + None, None + ) is False \ No newline at end of file From 49cb7c8d8b57f125c22fcddd206f18a31347e7fc Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 26 Feb 2024 12:45:57 -0800 Subject: [PATCH 134/376] more unit tests --- test/unit/test_streaming_client_configs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/test_streaming_client_configs.py b/test/unit/test_streaming_client_configs.py index c76a64ea7..3e49e6747 100644 --- a/test/unit/test_streaming_client_configs.py +++ b/test/unit/test_streaming_client_configs.py @@ -211,6 +211,7 @@ def test_GIVEN_has_mpl_WHEN_attrs_post_init_THEN_calls_no_mpl_method( _has_mpl_attrs_post_init.assert_called_once_with() +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") @pytest.mark.parametrize( "kwargs, stream_type", ( @@ -222,7 +223,6 @@ def test_GIVEN_has_mpl_WHEN_attrs_post_init_THEN_calls_no_mpl_method( (dict(source=b"", keyring=FakeKeyring()), io.BytesIO), ), ) -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") def test_client_configs_with_mpl( kwargs, stream_type From 705113a3ce2fcb2bda264c3f453125bd20db6a96 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 26 Feb 2024 12:54:24 -0800 Subject: [PATCH 135/376] more unit tests --- test/unit/test_streaming_client_configs.py | 38 ++++++++++++++++++---- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/test/unit/test_streaming_client_configs.py b/test/unit/test_streaming_client_configs.py index 3e49e6747..120931cb8 100644 --- a/test/unit/test_streaming_client_configs.py +++ b/test/unit/test_streaming_client_configs.py @@ -215,17 +215,15 @@ def test_GIVEN_has_mpl_WHEN_attrs_post_init_THEN_calls_no_mpl_method( @pytest.mark.parametrize( "kwargs, stream_type", ( - (dict(source=b"", materials_manager=FakeCryptoMaterialsManager()), io.BytesIO), - (dict(source=b"", key_provider=FakeMasterKeyProvider()), io.BytesIO), - (dict(source="", materials_manager=FakeCryptoMaterialsManager()), io.BytesIO), - (dict(source=io.BytesIO(), materials_manager=FakeCryptoMaterialsManager()), io.BytesIO), - (dict(source=six.StringIO(), materials_manager=FakeCryptoMaterialsManager()), six.StringIO), - (dict(source=b"", keyring=FakeKeyring()), io.BytesIO), + (dict(source=b"", materials_manager=FakeCryptoMaterialsManager())), + (dict(source=b"", key_provider=FakeMasterKeyProvider())), + (dict(source="", materials_manager=FakeCryptoMaterialsManager())), + (dict(source=io.BytesIO(), materials_manager=FakeCryptoMaterialsManager())), + (dict(source=six.StringIO(), materials_manager=FakeCryptoMaterialsManager())), ), ) def test_client_configs_with_mpl( kwargs, - stream_type ): kwargs["commitment_policy"] = CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT @@ -250,3 +248,29 @@ def test_client_configs_with_mpl( assert test.key_provider is not None assert test.key_provider == kwargs["key_provider"] assert isinstance(test.materials_manager, DefaultCryptoMaterialsManager) + + +# This needs its own test; pytest parametrize cannot use a conditionally-loaded type +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_keyring_client_config_with_mpl( +): + kwargs = { + "source": b"", + "keyring": FakeKeyring() + } + + test = _ClientConfig(**kwargs) + + # In all cases, config should have a materials manager + assert test.materials_manager is not None + + # If materials manager was provided, it should be directly used + if hasattr(kwargs, "materials_manager"): + assert kwargs["materials_manager"] == test.materials_manager + + # If MPL keyring was provided, it should be wrapped in MPL materials manager + if hasattr(kwargs, "keyring"): + assert test.keyring is not None + assert test.keyring == kwargs["keyring"] + assert isinstance(test.keyring, IKeyring) + assert isinstance(test.materials_manager, CryptoMaterialsManagerFromMPL) From f76d7f9f76c1eeaaf14aafbeade594680066515b Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 26 Feb 2024 12:56:49 -0800 Subject: [PATCH 136/376] more unit tests --- test/unit/test_streaming_client_configs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/test_streaming_client_configs.py b/test/unit/test_streaming_client_configs.py index 120931cb8..1a3fc89bc 100644 --- a/test/unit/test_streaming_client_configs.py +++ b/test/unit/test_streaming_client_configs.py @@ -213,7 +213,7 @@ def test_GIVEN_has_mpl_WHEN_attrs_post_init_THEN_calls_no_mpl_method( @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") @pytest.mark.parametrize( - "kwargs, stream_type", + "kwargs", ( (dict(source=b"", materials_manager=FakeCryptoMaterialsManager())), (dict(source=b"", key_provider=FakeMasterKeyProvider())), From 0da2a4f2c7dee0c0bd333bb42872b83f322af6e9 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 26 Feb 2024 13:10:59 -0800 Subject: [PATCH 137/376] more unit tests --- .../unit/test_crypto_authentication_signer.py | 9 ++++ test/unit/test_streaming_client_configs.py | 10 +++- .../test_streaming_client_stream_decryptor.py | 46 +++++++------------ .../test_streaming_client_stream_encryptor.py | 20 +++++--- 4 files changed, 47 insertions(+), 38 deletions(-) diff --git a/test/unit/test_crypto_authentication_signer.py b/test/unit/test_crypto_authentication_signer.py index c37c97bde..58cad2a7e 100644 --- a/test/unit/test_crypto_authentication_signer.py +++ b/test/unit/test_crypto_authentication_signer.py @@ -94,11 +94,14 @@ def test_GIVEN_no_encoding_WHEN_signer_from_key_bytes_THEN_load_der_private_key( with patch.object(cryptography.hazmat.primitives, "serialization"): # Mock the `serialization.load_der_private_key` with patch.object(aws_encryption_sdk.internal.crypto.authentication.serialization, "load_der_private_key") as mock_der: + # When: from_key_bytes Signer.from_key_bytes( algorithm=_algorithm, key_bytes=sentinel.key_bytes, + # Given: No encoding provided => default arg ) + # Then: calls load_der_private_key mock_der.assert_called_once_with( data=sentinel.key_bytes, password=None, backend=patch_default_backend.return_value ) @@ -113,12 +116,15 @@ def test_GIVEN_PEM_encoding_WHEN_signer_from_key_bytes_THEN_load_pem_private_key mock_algorithm_info = MagicMock(return_value=sentinel.algorithm_info, spec=patch_ec.EllipticCurve) _algorithm = MagicMock(signing_algorithm_info=mock_algorithm_info) + # When: from_key_bytes signer = Signer.from_key_bytes( algorithm=_algorithm, key_bytes=sentinel.key_bytes, + # Given: PEM encoding encoding=patch_serialization.Encoding.PEM ) + # Then: calls load_pem_private_key patch_serialization.load_pem_private_key.assert_called_once_with( data=sentinel.key_bytes, password=None, backend=patch_default_backend.return_value ) @@ -136,10 +142,13 @@ def test_GIVEN_unrecognized_encoding_WHEN_signer_from_key_bytes_THEN_raise_Value mock_algorithm_info = MagicMock(return_value=sentinel.algorithm_info, spec=patch_ec.EllipticCurve) _algorithm = MagicMock(signing_algorithm_info=mock_algorithm_info) + # Then: Raises ValueError with pytest.raises(ValueError): + # When: from_key_bytes signer = Signer.from_key_bytes( algorithm=_algorithm, key_bytes=sentinel.key_bytes, + # Given: Invalid encoding encoding="not an encoding" ) diff --git a/test/unit/test_streaming_client_configs.py b/test/unit/test_streaming_client_configs.py index 1a3fc89bc..38f6de930 100644 --- a/test/unit/test_streaming_client_configs.py +++ b/test/unit/test_streaming_client_configs.py @@ -193,22 +193,28 @@ def test_client_config_converts(kwargs, stream_type): assert isinstance(test.materials_manager, DefaultCryptoMaterialsManager) +# Given: no MPL @pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") @patch.object(_ClientConfig, "_no_mpl_attrs_post_init") def test_GIVEN_no_mpl_WHEN_attrs_post_init_THEN_calls_no_mpl_method( mock_no_mpl_attrs_post_init, ): + # When: attrs_post_init _ClientConfig(**BASE_KWARGS) + # Then: calls _no_mpl_attrs_post_init mock_no_mpl_attrs_post_init.assert_called_once_with() +# Given: has MPL @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") @patch.object(_ClientConfig, "_has_mpl_attrs_post_init") def test_GIVEN_has_mpl_WHEN_attrs_post_init_THEN_calls_no_mpl_method( - _has_mpl_attrs_post_init, + mock_has_mpl_attrs_post_init, ): + # When: attrs_post_init _ClientConfig(**BASE_KWARGS) - _has_mpl_attrs_post_init.assert_called_once_with() + # Then: calls _has_mpl_attrs_post_init + mock_has_mpl_attrs_post_init.assert_called_once_with() @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") diff --git a/test/unit/test_streaming_client_stream_decryptor.py b/test/unit/test_streaming_client_stream_decryptor.py index c8a17e650..fc45cc393 100644 --- a/test/unit/test_streaming_client_stream_decryptor.py +++ b/test/unit/test_streaming_client_stream_decryptor.py @@ -37,14 +37,11 @@ # Ideally, this logic would be based on mocking imports and testing logic, # but doing that introduces errors that cause other tests to fail. try: - from aws_cryptographic_materialproviders.mpl.references import ( - IKeyring, - ) - HAS_MPL = True - from aws_encryption_sdk.materials_managers.mpl.cmm import ( CryptoMaterialsManagerFromMPL, ) + HAS_MPL = True + except ImportError: HAS_MPL = False @@ -238,12 +235,14 @@ def test_read_header(self, mock_derive_datakey, mock_decrypt_materials_request, @patch("aws_encryption_sdk.streaming_client.DecryptionMaterialsRequest") @patch("aws_encryption_sdk.streaming_client.derive_data_encryption_key") @patch("aws_encryption_sdk.streaming_client.Verifier") + # Given: no MPL @pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") def test_GIVEN_verification_key_AND_no_mpl_WHEN_read_header_THEN_calls_from_key_bytes( self, mock_verifier, *_, ): + # Given: verification key mock_verifier_instance = MagicMock() mock_verifier.from_key_bytes.return_value = mock_verifier_instance ct_stream = io.BytesIO(VALUES["data_128"]) @@ -256,8 +255,10 @@ def test_GIVEN_verification_key_AND_no_mpl_WHEN_read_header_THEN_calls_from_key_ test_decryptor.source_stream = ct_stream test_decryptor._stream_length = len(VALUES["data_128"]) + # When: read header test_decryptor._read_header() + # Then: calls from_key_bytes mock_verifier.from_key_bytes.assert_called_once_with( algorithm=self.mock_header.algorithm, key_bytes=sentinel.verification_key ) @@ -265,17 +266,20 @@ def test_GIVEN_verification_key_AND_no_mpl_WHEN_read_header_THEN_calls_from_key_ @patch("aws_encryption_sdk.streaming_client.DecryptionMaterialsRequest") @patch("aws_encryption_sdk.streaming_client.derive_data_encryption_key") @patch("aws_encryption_sdk.streaming_client.Verifier") + # Given: has MPL @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") def test_GIVEN_verification_key_AND_has_mpl_AND_not_MPLCMM_WHEN_read_header_THEN_calls_from_key_bytes( self, mock_verifier, *_, ): + # Given: verification key mock_verifier_instance = MagicMock() mock_verifier.from_key_bytes.return_value = mock_verifier_instance ct_stream = io.BytesIO(VALUES["data_128"]) mock_commitment_policy = MagicMock(__class__=CommitmentPolicy) test_decryptor = StreamDecryptor( + # Given: native CMM materials_manager=self.mock_materials_manager, source=ct_stream, commitment_policy=mock_commitment_policy, @@ -283,8 +287,10 @@ def test_GIVEN_verification_key_AND_has_mpl_AND_not_MPLCMM_WHEN_read_header_THEN test_decryptor.source_stream = ct_stream test_decryptor._stream_length = len(VALUES["data_128"]) + # When: read_header test_decryptor._read_header() + # Then: calls from_key_bytess mock_verifier.from_key_bytes.assert_called_once_with( algorithm=self.mock_header.algorithm, key_bytes=sentinel.verification_key ) @@ -293,6 +299,7 @@ def test_GIVEN_verification_key_AND_has_mpl_AND_not_MPLCMM_WHEN_read_header_THEN @patch("aws_encryption_sdk.streaming_client.derive_data_encryption_key") @patch("aws_encryption_sdk.streaming_client.Verifier") @patch("base64.b64encode") + # Given: has MPL @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") def test_GIVEN_verification_key_AND_has_mpl_AND_has_MPLCMM_WHEN_read_header_THEN_calls_from_encoded_point( self, @@ -300,11 +307,13 @@ def test_GIVEN_verification_key_AND_has_mpl_AND_has_MPLCMM_WHEN_read_header_THEN mock_verifier, *_, ): + # Given: Verification key mock_verifier_instance = MagicMock() mock_verifier.from_key_bytes.return_value = mock_verifier_instance ct_stream = io.BytesIO(VALUES["data_128"]) mock_commitment_policy = MagicMock(__class__=CommitmentPolicy) test_decryptor = StreamDecryptor( + # Given: MPL CMM materials_manager=self.mock_mpl_materials_manager, source=ct_stream, commitment_policy=mock_commitment_policy, @@ -312,37 +321,14 @@ def test_GIVEN_verification_key_AND_has_mpl_AND_has_MPLCMM_WHEN_read_header_THEN test_decryptor.source_stream = ct_stream test_decryptor._stream_length = len(VALUES["data_128"]) + # When: read header test_decryptor._read_header() + # Then: calls from_encoded_point mock_verifier.from_encoded_point.assert_called_once_with( algorithm=self.mock_header.algorithm, encoded_point=mock_b64encoding() ) - # @patch("aws_encryption_sdk.streaming_client.Verifier") - # @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") - # def test_GIVEN_verification_key_AND_has_mpl_AND_not_MPLCMM_WHEN_read_header_THEN_calls_from_key_bytes( - # self, - # mock_verifier, - # ): - # mock_verifier_instance = MagicMock() - # mock_verifier.from_key_bytes.return_value = mock_verifier_instance - # ct_stream = io.BytesIO(VALUES["data_128"]) - # mock_commitment_policy = MagicMock(__class__=CommitmentPolicy) - # test_decryptor = StreamDecryptor( - # materials_manager=self.mock_materials_manager, - # source=ct_stream, - # commitment_policy=mock_commitment_policy, - # ) - # test_decryptor.source_stream = ct_stream - # test_decryptor._stream_length = len(VALUES["data_128"]) - - # test_decryptor._read_header() - - # mock_verifier.from_key_bytes.assert_called_once_with( - # algorithm=self.mock_header.algorithm, key_bytes=sentinel.verification_key - # ) - - @patch("aws_encryption_sdk.streaming_client.derive_data_encryption_key") def test_read_header_frame_too_large(self, mock_derive_datakey): self.mock_header.content_type = ContentType.FRAMED_DATA diff --git a/test/unit/test_streaming_client_stream_encryptor.py b/test/unit/test_streaming_client_stream_encryptor.py index 11664411a..bb4ba1c5e 100644 --- a/test/unit/test_streaming_client_stream_encryptor.py +++ b/test/unit/test_streaming_client_stream_encryptor.py @@ -42,14 +42,11 @@ # Ideally, this logic would be based on mocking imports and testing logic, # but doing that introduces errors that cause other tests to fail. try: - from aws_cryptographic_materialproviders.mpl.references import ( - IKeyring, - ) - HAS_MPL = True - from aws_encryption_sdk.materials_managers.mpl.cmm import ( CryptoMaterialsManagerFromMPL, ) + HAS_MPL = True + except ImportError: HAS_MPL = False @@ -387,6 +384,7 @@ def test_prep_message_non_framed_message(self, mock_write_header, mock_prep_non_ test_encryptor._prep_message() mock_prep_non_framed.assert_called_once_with() + # Given: no MPL @pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") def test_GIVEN_no_mpl_AND_uses_signer_WHEN_prep_message_THEN_signer_uses_default_encoding(self): self.mock_encryption_materials.algorithm = Algorithm.AES_128_GCM_IV12_TAG16 @@ -400,17 +398,21 @@ def test_GIVEN_no_mpl_AND_uses_signer_WHEN_prep_message_THEN_signer_uses_default ) test_encryptor.content_type = ContentType.FRAMED_DATA with patch.object(self.mock_signer, "from_key_bytes"): + # When: prep message test_encryptor._prep_message() + # Then: calls from_key_bytes with default encoding self.mock_signer.from_key_bytes.assert_called_once_with( algorithm=self.mock_encryption_materials.algorithm, key_bytes=self.mock_encryption_materials.signing_key ) + # Given: has MPL @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") def test_GIVEN_has_mpl_AND_not_MPLCMM_AND_uses_signer_WHEN_prep_message_THEN_signer_uses_default_encoding(self): self.mock_encryption_materials.algorithm = Algorithm.AES_128_GCM_IV12_TAG16 test_encryptor = StreamEncryptor( source=VALUES["data_128"], + # Given: native CMM materials_manager=self.mock_materials_manager, frame_length=self.mock_frame_length, algorithm=Algorithm.AES_128_GCM_IV12_TAG16, @@ -419,17 +421,21 @@ def test_GIVEN_has_mpl_AND_not_MPLCMM_AND_uses_signer_WHEN_prep_message_THEN_sig ) test_encryptor.content_type = ContentType.FRAMED_DATA with patch.object(self.mock_signer, "from_key_bytes"): + # When: prep_message test_encryptor._prep_message() + # Then: calls from_key_bytes with default encoding self.mock_signer.from_key_bytes.assert_called_once_with( algorithm=self.mock_encryption_materials.algorithm, key_bytes=self.mock_encryption_materials.signing_key ) + # Given: has MPL @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") - def test_GIVEN_has_mpl_AND_has_MPLCMM_AND_uses_signer_WHEN_prep_message_THEN_signer_uses_default_encoding(self): + def test_GIVEN_has_mpl_AND_has_MPLCMM_AND_uses_signer_WHEN_prep_message_THEN_signer_uses_PEM_encoding(self): self.mock_encryption_materials.algorithm = Algorithm.AES_128_GCM_IV12_TAG16 test_encryptor = StreamEncryptor( source=VALUES["data_128"], + # Given: MPL CMM materials_manager=self.mock_mpl_materials_manager, frame_length=self.mock_frame_length, algorithm=Algorithm.AES_128_GCM_IV12_TAG16, @@ -438,10 +444,12 @@ def test_GIVEN_has_mpl_AND_has_MPLCMM_AND_uses_signer_WHEN_prep_message_THEN_sig ) test_encryptor.content_type = ContentType.FRAMED_DATA with patch.object(self.mock_signer, "from_key_bytes"): + # When: prep_message test_encryptor._prep_message() self.mock_signer.from_key_bytes.assert_called_once_with( algorithm=self.mock_encryption_materials.algorithm, key_bytes=self.mock_encryption_materials.signing_key, + # Then: calls from_key_bytes with PEM encoding encoding=serialization.Encoding.PEM ) From 0040b2c67af302cf5e624838f2f54222d7aa85f3 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 26 Feb 2024 13:16:37 -0800 Subject: [PATCH 138/376] cleanup --- src/aws_encryption_sdk/streaming_client.py | 2 +- .../unit/test_crypto_authentication_signer.py | 15 ++++++----- test/unit/test_streaming_client_configs.py | 25 ++++++++----------- .../test_streaming_client_stream_decryptor.py | 8 +++--- .../test_streaming_client_stream_encryptor.py | 6 ++--- test/unit/test_utils.py | 16 ++++++------ 6 files changed, 34 insertions(+), 38 deletions(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 2cfcc9a02..cc9a6bb0f 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -171,7 +171,7 @@ def _has_mpl_attrs_post_init(self): except AwsCryptographicMaterialProvidersException as mpl_exception: # Wrap MPL error into the ESDK error type # so customers only have to catch ESDK error types. - raise AWSEncryptionSDKClientError(mpl_exception) + raise AWSEncryptionSDKClientError(mpl_exception) def _no_mpl_attrs_post_init(self): """If the MPL is NOT present in the runtime, perform post-init logic diff --git a/test/unit/test_crypto_authentication_signer.py b/test/unit/test_crypto_authentication_signer.py index 58cad2a7e..425f672ed 100644 --- a/test/unit/test_crypto_authentication_signer.py +++ b/test/unit/test_crypto_authentication_signer.py @@ -11,9 +11,9 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. """Unit test suite for ``aws_encryption_sdk.internal.crypto.authentication.Signer``.""" -import pytest -from mock import MagicMock, sentinel, patch import cryptography.hazmat.primitives.serialization +import pytest +from mock import MagicMock, patch, sentinel from pytest_mock import mocker # noqa pylint: disable=unused-import import aws_encryption_sdk.internal.crypto.authentication @@ -76,7 +76,7 @@ def test_f_signer_from_key_bytes(): def test_f_signer_key_bytes(): test = Signer(algorithm=ALGORITHM, key=VALUES["ecc_private_key_prime"]) assert test.key_bytes() == VALUES["ecc_private_key_prime_private_bytes"] - + def test_GIVEN_no_encoding_WHEN_signer_from_key_bytes_THEN_load_der_private_key( patch_default_backend, @@ -93,7 +93,10 @@ def test_GIVEN_no_encoding_WHEN_signer_from_key_bytes_THEN_load_der_private_key( # Mock the `serialization.Encoding.DER` with patch.object(cryptography.hazmat.primitives, "serialization"): # Mock the `serialization.load_der_private_key` - with patch.object(aws_encryption_sdk.internal.crypto.authentication.serialization, "load_der_private_key") as mock_der: + with patch.object( + aws_encryption_sdk.internal.crypto.authentication.serialization, + "load_der_private_key" + ) as mock_der: # When: from_key_bytes Signer.from_key_bytes( algorithm=_algorithm, @@ -106,7 +109,7 @@ def test_GIVEN_no_encoding_WHEN_signer_from_key_bytes_THEN_load_der_private_key( data=sentinel.key_bytes, password=None, backend=patch_default_backend.return_value ) - + def test_GIVEN_PEM_encoding_WHEN_signer_from_key_bytes_THEN_load_pem_private_key( patch_default_backend, patch_serialization, @@ -145,7 +148,7 @@ def test_GIVEN_unrecognized_encoding_WHEN_signer_from_key_bytes_THEN_raise_Value # Then: Raises ValueError with pytest.raises(ValueError): # When: from_key_bytes - signer = Signer.from_key_bytes( + Signer.from_key_bytes( algorithm=_algorithm, key_bytes=sentinel.key_bytes, # Given: Invalid encoding diff --git a/test/unit/test_streaming_client_configs.py b/test/unit/test_streaming_client_configs.py index 38f6de930..26ef86be8 100644 --- a/test/unit/test_streaming_client_configs.py +++ b/test/unit/test_streaming_client_configs.py @@ -33,14 +33,10 @@ # Ideally, this logic would be based on mocking imports and testing logic, # but doing that introduces errors that cause other tests to fail. try: - from aws_cryptographic_materialproviders.mpl.references import ( - IKeyring, - ) + from aws_cryptographic_materialproviders.mpl.references import IKeyring HAS_MPL = True - from aws_encryption_sdk.materials_managers.mpl.cmm import ( - CryptoMaterialsManagerFromMPL, - ) + from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL except ImportError: HAS_MPL = False @@ -59,14 +55,15 @@ class FakeMasterKeyProvider(MasterKeyProvider): def _new_master_key(self, key_id): return - + + if HAS_MPL: class FakeKeyring(IKeyring): def on_encrypt(self, param): - return - + return + def on_decrypt(self, param): - return + return BASE_KWARGS = dict( @@ -234,10 +231,10 @@ def test_client_configs_with_mpl( kwargs["commitment_policy"] = CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT test = _ClientConfig(**kwargs) - + # In all cases, config should have a materials manager assert test.materials_manager is not None - + # If materials manager was provided, it should be directly used if hasattr(kwargs, "materials_manager"): assert kwargs["materials_manager"] == test.materials_manager @@ -266,10 +263,10 @@ def test_keyring_client_config_with_mpl( } test = _ClientConfig(**kwargs) - + # In all cases, config should have a materials manager assert test.materials_manager is not None - + # If materials manager was provided, it should be directly used if hasattr(kwargs, "materials_manager"): assert kwargs["materials_manager"] == test.materials_manager diff --git a/test/unit/test_streaming_client_stream_decryptor.py b/test/unit/test_streaming_client_stream_decryptor.py index fc45cc393..e06cad308 100644 --- a/test/unit/test_streaming_client_stream_decryptor.py +++ b/test/unit/test_streaming_client_stream_decryptor.py @@ -37,9 +37,7 @@ # Ideally, this logic would be based on mocking imports and testing logic, # but doing that introduces errors that cause other tests to fail. try: - from aws_encryption_sdk.materials_managers.mpl.cmm import ( - CryptoMaterialsManagerFromMPL, - ) + from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL HAS_MPL = True except ImportError: @@ -55,7 +53,7 @@ def apply_fixtures(self): data_key=VALUES["data_key_obj"], verification_key=sentinel.verification_key ) self.mock_materials_manager.decrypt_materials.return_value = self.mock_decrypt_materials - + if HAS_MPL: self.mock_mpl_materials_manager = MagicMock(__class__=CryptoMaterialsManagerFromMPL) self.mock_mpl_materials_manager.decrypt_materials.return_value = self.mock_decrypt_materials @@ -258,7 +256,7 @@ def test_GIVEN_verification_key_AND_no_mpl_WHEN_read_header_THEN_calls_from_key_ # When: read header test_decryptor._read_header() - # Then: calls from_key_bytes + # Then: calls from_key_bytes mock_verifier.from_key_bytes.assert_called_once_with( algorithm=self.mock_header.algorithm, key_bytes=sentinel.verification_key ) diff --git a/test/unit/test_streaming_client_stream_encryptor.py b/test/unit/test_streaming_client_stream_encryptor.py index bb4ba1c5e..e43752689 100644 --- a/test/unit/test_streaming_client_stream_encryptor.py +++ b/test/unit/test_streaming_client_stream_encryptor.py @@ -13,9 +13,9 @@ """Unit test suite for aws_encryption_sdk.streaming_client.StreamEncryptor""" import io -from cryptography.hazmat.primitives import serialization import pytest import six +from cryptography.hazmat.primitives import serialization from mock import MagicMock, call, patch, sentinel import aws_encryption_sdk.internal.defaults @@ -42,9 +42,7 @@ # Ideally, this logic would be based on mocking imports and testing logic, # but doing that introduces errors that cause other tests to fail. try: - from aws_encryption_sdk.materials_managers.mpl.cmm import ( - CryptoMaterialsManagerFromMPL, - ) + from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL HAS_MPL = True except ImportError: diff --git a/test/unit/test_utils.py b/test/unit/test_utils.py index d717b51c7..69f9f060d 100644 --- a/test/unit/test_utils.py +++ b/test/unit/test_utils.py @@ -268,25 +268,25 @@ def test_source_data_key_length_check_invalid(self): def test_exactly_one_arg_is_not_none(self): # No args => no args are not None - assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none() is False + assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none() is False assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none( None - ) is False + ) is False assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none( "not None" - ) is True + ) is True assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none( "not None", "also not None" - ) is False + ) is False assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none( "not None", None - ) is True + ) is True assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none( "not None", "also not None" - ) is False + ) is False assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none( None, "not None" - ) is True + ) is True assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none( None, None - ) is False \ No newline at end of file + ) is False From 9131433f84f2f796a7e29f221e774020008aeeca Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 26 Feb 2024 13:20:17 -0800 Subject: [PATCH 139/376] cleanup --- src/aws_encryption_sdk/streaming_client.py | 2 +- test/unit/test_streaming_client_configs.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index cc9a6bb0f..5bf953244 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -171,7 +171,7 @@ def _has_mpl_attrs_post_init(self): except AwsCryptographicMaterialProvidersException as mpl_exception: # Wrap MPL error into the ESDK error type # so customers only have to catch ESDK error types. - raise AWSEncryptionSDKClientError(mpl_exception) + raise AWSEncryptionSDKClientError(mpl_exception) def _no_mpl_attrs_post_init(self): """If the MPL is NOT present in the runtime, perform post-init logic diff --git a/test/unit/test_streaming_client_configs.py b/test/unit/test_streaming_client_configs.py index 26ef86be8..18886f65b 100644 --- a/test/unit/test_streaming_client_configs.py +++ b/test/unit/test_streaming_client_configs.py @@ -259,7 +259,8 @@ def test_keyring_client_config_with_mpl( ): kwargs = { "source": b"", - "keyring": FakeKeyring() + "keyring": FakeKeyring(), + "commitment_policy": CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT } test = _ClientConfig(**kwargs) From e6826eb3fdc5773dc00f5bfb1113a7d8f0c67fd3 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 28 Feb 2024 12:20:56 -0800 Subject: [PATCH 140/376] poc impl --- src/aws_encryption_sdk/__init__.py | 1 + .../internal/formatting/serialize.py | 35 +++++-- .../materials_managers/mpl/cmm.py | 1 + .../materials_managers/mpl/materials.py | 11 +++ src/aws_encryption_sdk/streaming_client.py | 93 +++++++++++++++---- 5 files changed, 115 insertions(+), 26 deletions(-) diff --git a/src/aws_encryption_sdk/__init__.py b/src/aws_encryption_sdk/__init__.py index 661d41ee6..4b35e6744 100644 --- a/src/aws_encryption_sdk/__init__.py +++ b/src/aws_encryption_sdk/__init__.py @@ -185,6 +185,7 @@ def decrypt(self, **kwargs): If source_length is not provided and read() is called, will attempt to seek() to the end of the stream and tell() to find the length of source data. + :param dict encryption_context: Dictionary defining encryption context :param int max_body_length: Maximum frame size (or content length for non-framed messages) in bytes to read from ciphertext message. :returns: Tuple containing the decrypted plaintext and the message header object diff --git a/src/aws_encryption_sdk/internal/formatting/serialize.py b/src/aws_encryption_sdk/internal/formatting/serialize.py index b4d866099..718d4ad7d 100644 --- a/src/aws_encryption_sdk/internal/formatting/serialize.py +++ b/src/aws_encryption_sdk/internal/formatting/serialize.py @@ -218,7 +218,13 @@ def _serialize_header_auth_v1(algorithm, header, data_encryption_key, signer=Non return output -def _serialize_header_auth_v2(algorithm, header, data_encryption_key, signer=None): +def _serialize_header_auth_v2( + algorithm, + header, + data_encryption_key, + signer=None, + required_encryption_context_bytes=None + ): """Creates serialized header authentication data for messages in serialization version V2. :param algorithm: Algorithm to use for encryption @@ -230,13 +236,22 @@ def _serialize_header_auth_v2(algorithm, header, data_encryption_key, signer=Non :returns: Serialized header authentication data :rtype: bytes """ - header_auth = encrypt( - algorithm=algorithm, - key=data_encryption_key, - plaintext=b"", - associated_data=header, - iv=header_auth_iv(algorithm), - ) + if required_encryption_context_bytes is None: + header_auth = encrypt( + algorithm=algorithm, + key=data_encryption_key, + plaintext=b"", + associated_data=header, + iv=header_auth_iv(algorithm), + ) + else: + header_auth = encrypt( + algorithm=algorithm, + key=data_encryption_key, + plaintext=b"", + associated_data=header + required_encryption_context_bytes, + iv=header_auth_iv(algorithm), + ) output = struct.pack( ">{tag_len}s".format(tag_len=algorithm.tag_len), header_auth.tag, @@ -246,7 +261,7 @@ def _serialize_header_auth_v2(algorithm, header, data_encryption_key, signer=Non return output -def serialize_header_auth(version, algorithm, header, data_encryption_key, signer=None): +def serialize_header_auth(version, algorithm, header, data_encryption_key, signer=None, required_encryption_context_bytes=None): """Creates serialized header authentication data. :param version: The serialization version of the message @@ -263,7 +278,7 @@ def serialize_header_auth(version, algorithm, header, data_encryption_key, signe if version == SerializationVersion.V1: return _serialize_header_auth_v1(algorithm, header, data_encryption_key, signer) elif version == SerializationVersion.V2: - return _serialize_header_auth_v2(algorithm, header, data_encryption_key, signer) + return _serialize_header_auth_v2(algorithm, header, data_encryption_key, signer, required_encryption_context_bytes) else: raise SerializationError("Unrecognized message format version: {}".format(version)) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index 53a4b3505..8df42bf48 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -143,5 +143,6 @@ def _create_mpl_decrypt_materials_input_from_request( ), encrypted_data_keys=list_edks, encryption_context=request.encryption_context, + reproduced_encryption_context=request.reproduced_encryption_context, ) return output diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index dfd1bd6fc..d2abf182c 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -95,6 +95,12 @@ def data_encryption_key(self) -> DataKey: def signing_key(self) -> bytes: """Materials' signing key.""" return self.mpl_materials.signing_key + + + @property + def required_encryption_context_keys(self) -> bytes: + """Materials' required encryption context keys.""" + return self.mpl_materials.required_encryption_context_keys class DecryptionMaterialsFromMPL(Native_DecryptionMaterials): @@ -136,3 +142,8 @@ def data_key(self) -> DataKey: def verification_key(self) -> bytes: """Materials' verification key.""" return self.mpl_materials.verification_key + + @property + def required_encryption_context_keys(self) -> bytes: + """Materials' required encryption context keys.""" + return self.mpl_materials.required_encryption_context_keys diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 5bf953244..6d779c79e 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -593,11 +593,23 @@ def generate_header(self, message_id): if self._encryption_materials.algorithm.message_format_version == 0x02: version = SerializationVersion.V2 + if hasattr(self._encryption_materials, "required_encryption_context_keys"): + self._required_encryption_context = {} + self._stored_encryption_context = {} + for (k, v) in self._encryption_materials.encryption_context: + if k in self._encryption_materials.required_encryption_context_keys: + self._required_encryption_context[k] = v + else: + self._stored_encryption_context[k] = v + else: + self._stored_encryption_context = self._encryption_materials.encryption_context, + self._required_encryption_context = None + kwargs = dict( version=version, algorithm=self._encryption_materials.algorithm, message_id=message_id, - encryption_context=self._encryption_materials.encryption_context, + encryption_context=self._stored_encryption_context, encrypted_data_keys=self._encryption_materials.encrypted_data_keys, content_type=self.content_type, frame_length=self.config.frame_length, @@ -621,13 +633,27 @@ def generate_header(self, message_id): def _write_header(self): """Builds the message header and writes it to the output stream.""" self.output_buffer += serialize_header(header=self._header, signer=self.signer) - self.output_buffer += serialize_header_auth( - version=self._header.version, - algorithm=self._encryption_materials.algorithm, - header=self.output_buffer, - data_encryption_key=self._derived_data_key, - signer=self.signer, - ) + + if self._required_encryption_context is not None: + required_ec_serialized = aws_encryption_sdk.internal.formatting.encryption_context.serialize_encryption_context( + self._required_encryption_context + ) + self.output_buffer += serialize_header_auth( + version=self._header.version, + algorithm=self._encryption_materials.algorithm, + header=self.output_buffer, + data_encryption_key=self._derived_data_key, + signer=self.signer, + required_encryption_context_bytes=required_ec_serialized, + ) + else: + self.output_buffer += serialize_header_auth( + version=self._header.version, + algorithm=self._encryption_materials.algorithm, + header=self.output_buffer, + data_encryption_key=self._derived_data_key, + signer=self.signer, + ) def _prep_non_framed(self): """Prepare the opening data for a non-framed message.""" @@ -907,14 +933,32 @@ def _read_header(self): found=header.frame_length, custom=self.config.max_body_length ) ) - - decrypt_materials_request = DecryptionMaterialsRequest( - encrypted_data_keys=header.encrypted_data_keys, - algorithm=header.algorithm, - encryption_context=header.encryption_context, - commitment_policy=self.config.commitment_policy, - ) + + if hasattr(self, "encryption_context"): + decrypt_materials_request = DecryptionMaterialsRequest( + encrypted_data_keys=header.encrypted_data_keys, + algorithm=header.algorithm, + encryption_context=header.encryption_context, + commitment_policy=self.config.commitment_policy, + reproduced_encryption_context=self.encryption_context + ) + else: + decrypt_materials_request = DecryptionMaterialsRequest( + encrypted_data_keys=header.encrypted_data_keys, + algorithm=header.algorithm, + encryption_context=header.encryption_context, + commitment_policy=self.config.commitment_policy, + ) decryption_materials = self.config.materials_manager.decrypt_materials(request=decrypt_materials_request) + + if hasattr(decryption_materials, "required_encryption_context_keys"): + self._required_encryption_context = {} + for (k, v) in self._encryption_materials.encryption_context: + if k in self._encryption_materials.required_encryption_context_keys: + self._required_encryption_context[k] = v + else: + self._required_encryption_context = None + if decryption_materials.verification_key is None: self.verifier = None else: @@ -953,7 +997,24 @@ def _read_header(self): "message. Halting processing of this message." ) - validate_header(header=header, header_auth=header_auth, raw_header=raw_header, data_key=self._derived_data_key) + if required_ec_serialized is not None: + required_ec_serialized = aws_encryption_sdk.internal.formatting.encryption_context.serialize_encryption_context( + self._required_encryption_context + ) + + validate_header( + header=header, + header_auth=header_auth, + raw_header=raw_header + required_ec_serialized, + data_key=self._derived_data_key + ) + else: + validate_header( + header=header, + header_auth=header_auth, + raw_header=raw_header, + data_key=self._derived_data_key + ) return header, header_auth From a9fa1a5579dde63c6b5556991452075738329608 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 28 Feb 2024 13:34:13 -0800 Subject: [PATCH 141/376] passing --- .../materials_managers/__init__.py | 5 ++ .../materials_managers/mpl/materials.py | 5 ++ src/aws_encryption_sdk/streaming_client.py | 63 +++++++++++++++---- 3 files changed, 62 insertions(+), 11 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/__init__.py b/src/aws_encryption_sdk/materials_managers/__init__.py index 9db1dafae..f1eb30023 100644 --- a/src/aws_encryption_sdk/materials_managers/__init__.py +++ b/src/aws_encryption_sdk/materials_managers/__init__.py @@ -89,11 +89,16 @@ class DecryptionMaterialsRequest(object): :param encrypted_data_keys: Set of encrypted data keys :type encrypted_data_keys: set of `aws_encryption_sdk.structures.EncryptedDataKey` :param dict encryption_context: Encryption context to provide to master keys for underlying decrypt requests + :param dict reproduced_encryption_context: TODO """ algorithm = attr.ib(validator=attr.validators.instance_of(Algorithm)) encrypted_data_keys = attr.ib(validator=attr.validators.instance_of(set)) encryption_context = attr.ib(validator=attr.validators.instance_of(dict)) + reproduced_encryption_context = attr.ib( + default=None, + validator=attr.validators.optional(attr.validators.instance_of(dict)) + ) commitment_policy = attr.ib( default=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, validator=attr.validators.optional(attr.validators.instance_of(CommitmentPolicy)), diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index d2abf182c..5b066c7c7 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -143,6 +143,11 @@ def verification_key(self) -> bytes: """Materials' verification key.""" return self.mpl_materials.verification_key + @property + def encryption_context(self) -> Dict[str, str]: + """Materials' encryption context.""" + return self.mpl_materials.encryption_context + @property def required_encryption_context_keys(self) -> bytes: """Materials' required encryption context keys.""" diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 6d779c79e..f678c4b77 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -76,8 +76,13 @@ from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException - from aws_cryptographic_materialproviders.mpl.models import CreateDefaultCryptographicMaterialsManagerInput - from aws_cryptographic_materialproviders.mpl.references import IKeyring + from aws_cryptographic_materialproviders.mpl.models import ( + CreateDefaultCryptographicMaterialsManagerInput, + ) + from aws_cryptographic_materialproviders.mpl.references import ( + ICryptographicMaterialsManager, + IKeyring, + ) _HAS_MPL = True # Import internal ESDK modules that depend on the MPL @@ -126,9 +131,30 @@ class _ClientConfig(object): # pylint: disable=too-many-instance-attributes max_encrypted_data_keys = attr.ib( hash=True, default=None, validator=attr.validators.optional(attr.validators.instance_of(int)) ) - materials_manager = attr.ib( - hash=True, default=None, validator=attr.validators.optional(attr.validators.instance_of(CryptoMaterialsManager)) - ) + if _HAS_MPL: + # With the MPL, the provided materials_manager can be an instance of + # either the native interface or an MPL interface. + # If it implements the MPL interface, this constructor will + # internally wrap it in a native interface. + materials_manager = attr.ib( + hash=True, + default=None, + validator=attr.validators.optional( + attr.validators.instance_of( + (CryptoMaterialsManager, ICryptographicMaterialsManager) + ) + ) + ) + else: + materials_manager = attr.ib( + hash=True, + default=None, + validator=attr.validators.optional( + attr.validators.instance_of( + CryptoMaterialsManager + ) + ) + ) key_provider = attr.ib( hash=True, default=None, validator=attr.validators.optional(attr.validators.instance_of(MasterKeyProvider)) ) @@ -172,6 +198,12 @@ def _has_mpl_attrs_post_init(self): # Wrap MPL error into the ESDK error type # so customers only have to catch ESDK error types. raise AWSEncryptionSDKClientError(mpl_exception) + # TODO-MPL: MUST wrap MPL with native + elif (self.materials_manager is not None + and isinstance(self.materials_manager, ICryptographicMaterialsManager)): + # If the provided materials manager implements an MPL interface, + # wrap it in a native interface. + self.materials_manager = CryptoMaterialsManagerFromMPL(self.materials_manager) def _no_mpl_attrs_post_init(self): """If the MPL is NOT present in the runtime, perform post-init logic @@ -596,7 +628,8 @@ def generate_header(self, message_id): if hasattr(self._encryption_materials, "required_encryption_context_keys"): self._required_encryption_context = {} self._stored_encryption_context = {} - for (k, v) in self._encryption_materials.encryption_context: + print(f"{self._encryption_materials.encryption_context=}") + for (k, v) in self._encryption_materials.encryption_context.items(): if k in self._encryption_materials.required_encryption_context_keys: self._required_encryption_context[k] = v else: @@ -856,6 +889,11 @@ class DecryptorConfig(_ClientConfig): max_body_length = attr.ib( hash=True, default=None, validator=attr.validators.optional(attr.validators.instance_of(six.integer_types)) ) + encryption_context = attr.ib( + hash=False, # dictionaries are not hashable + default=attr.Factory(dict), + validator=attr.validators.instance_of(dict), + ) class StreamDecryptor(_EncryptionStream): # pylint: disable=too-many-instance-attributes @@ -934,13 +972,15 @@ def _read_header(self): ) ) - if hasattr(self, "encryption_context"): + print(f"{self.config.encryption_context=}") + + if hasattr(self.config, "encryption_context"): decrypt_materials_request = DecryptionMaterialsRequest( encrypted_data_keys=header.encrypted_data_keys, algorithm=header.algorithm, encryption_context=header.encryption_context, commitment_policy=self.config.commitment_policy, - reproduced_encryption_context=self.encryption_context + reproduced_encryption_context=self.config.encryption_context ) else: decrypt_materials_request = DecryptionMaterialsRequest( @@ -949,12 +989,13 @@ def _read_header(self): encryption_context=header.encryption_context, commitment_policy=self.config.commitment_policy, ) + print(f"{decrypt_materials_request=}") decryption_materials = self.config.materials_manager.decrypt_materials(request=decrypt_materials_request) if hasattr(decryption_materials, "required_encryption_context_keys"): self._required_encryption_context = {} - for (k, v) in self._encryption_materials.encryption_context: - if k in self._encryption_materials.required_encryption_context_keys: + for (k, v) in decryption_materials.encryption_context.items(): + if k in decryption_materials.required_encryption_context_keys: self._required_encryption_context[k] = v else: self._required_encryption_context = None @@ -997,7 +1038,7 @@ def _read_header(self): "message. Halting processing of this message." ) - if required_ec_serialized is not None: + if self._required_encryption_context is not None: required_ec_serialized = aws_encryption_sdk.internal.formatting.encryption_context.serialize_encryption_context( self._required_encryption_context ) From 4eeb85889b498004eb865762e6503a6991bbee84 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 28 Feb 2024 14:03:10 -0800 Subject: [PATCH 142/376] cleanup --- src/aws_encryption_sdk/__init__.py | 4 +- .../internal/formatting/serialize.py | 37 +++++++++++++++---- .../materials_managers/__init__.py | 4 +- src/aws_encryption_sdk/streaming_client.py | 36 ++++++++++++++---- 4 files changed, 64 insertions(+), 17 deletions(-) diff --git a/src/aws_encryption_sdk/__init__.py b/src/aws_encryption_sdk/__init__.py index 4b35e6744..96898d446 100644 --- a/src/aws_encryption_sdk/__init__.py +++ b/src/aws_encryption_sdk/__init__.py @@ -185,7 +185,9 @@ def decrypt(self, **kwargs): If source_length is not provided and read() is called, will attempt to seek() to the end of the stream and tell() to find the length of source data. - :param dict encryption_context: Dictionary defining encryption context + :param dict encryption_context: Dictionary defining encryption context to validate + on decrypt. This is ONLY validated on decrypt if using the required encryption + context CMM from the aws-cryptographic-materialproviders library. :param int max_body_length: Maximum frame size (or content length for non-framed messages) in bytes to read from ciphertext message. :returns: Tuple containing the decrypted plaintext and the message header object diff --git a/src/aws_encryption_sdk/internal/formatting/serialize.py b/src/aws_encryption_sdk/internal/formatting/serialize.py index 718d4ad7d..344c94703 100644 --- a/src/aws_encryption_sdk/internal/formatting/serialize.py +++ b/src/aws_encryption_sdk/internal/formatting/serialize.py @@ -219,12 +219,12 @@ def _serialize_header_auth_v1(algorithm, header, data_encryption_key, signer=Non def _serialize_header_auth_v2( - algorithm, - header, - data_encryption_key, - signer=None, - required_encryption_context_bytes=None - ): + algorithm, + header, + data_encryption_key, + signer=None, + required_encryption_context_bytes=None +): """Creates serialized header authentication data for messages in serialization version V2. :param algorithm: Algorithm to use for encryption @@ -233,6 +233,11 @@ def _serialize_header_auth_v2( :param bytes data_encryption_key: Data key with which to encrypt message :param signer: Cryptographic signer object (optional) :type signer: aws_encryption_sdk.Signer + :param required_encryption_context_bytes: Serialized encryption context items + for all items whose keys are in the required_encryption_context list. + This is ONLY processed if using the aws-cryptographic-materialproviders library + AND its required encryption context CMM. (optional) + :type required_encryption_context_bytes: bytes :returns: Serialized header authentication data :rtype: bytes """ @@ -249,6 +254,11 @@ def _serialize_header_auth_v2( algorithm=algorithm, key=data_encryption_key, plaintext=b"", + # The AAD MUST be the concatenation of the serialized message header body and the serialization + # of encryption context to only authenticate. The encryption context to only authenticate MUST + # be the encryption context in the encryption materials filtered to only contain key value + # pairs listed in the encryption material's required encryption context keys serialized + # according to the encryption context serialization specification. associated_data=header + required_encryption_context_bytes, iv=header_auth_iv(algorithm), ) @@ -261,7 +271,14 @@ def _serialize_header_auth_v2( return output -def serialize_header_auth(version, algorithm, header, data_encryption_key, signer=None, required_encryption_context_bytes=None): +def serialize_header_auth( + version, + algorithm, + header, + data_encryption_key, + signer=None, + required_encryption_context_bytes=None +): """Creates serialized header authentication data. :param version: The serialization version of the message @@ -272,6 +289,12 @@ def serialize_header_auth(version, algorithm, header, data_encryption_key, signe :param bytes data_encryption_key: Data key with which to encrypt message :param signer: Cryptographic signer object (optional) :type signer: aws_encryption_sdk.Signer + :param required_encryption_context_bytes: Serialized encryption context items + for all items whose keys are in the required_encryption_context list. + This is ONLY processed if using the aws-cryptographic-materialproviders library + AND its required encryption context CMM + AND if using the v2 message format. (optional) + :type required_encryption_context_bytes: bytes :returns: Serialized header authentication data :rtype: bytes """ diff --git a/src/aws_encryption_sdk/materials_managers/__init__.py b/src/aws_encryption_sdk/materials_managers/__init__.py index f1eb30023..cc8cdcf6f 100644 --- a/src/aws_encryption_sdk/materials_managers/__init__.py +++ b/src/aws_encryption_sdk/materials_managers/__init__.py @@ -89,7 +89,9 @@ class DecryptionMaterialsRequest(object): :param encrypted_data_keys: Set of encrypted data keys :type encrypted_data_keys: set of `aws_encryption_sdk.structures.EncryptedDataKey` :param dict encryption_context: Encryption context to provide to master keys for underlying decrypt requests - :param dict reproduced_encryption_context: TODO + :param dict reproduced_encryption_context: Encryption context to provide on decrypt. + This is ONLY processed if using the required encryption context CMM from the + aws-cryptographic-materialproviders library. """ algorithm = attr.ib(validator=attr.validators.instance_of(Algorithm)) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index f678c4b77..568543863 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -198,11 +198,11 @@ def _has_mpl_attrs_post_init(self): # Wrap MPL error into the ESDK error type # so customers only have to catch ESDK error types. raise AWSEncryptionSDKClientError(mpl_exception) - # TODO-MPL: MUST wrap MPL with native + + # If the provided materials_manager is directly from the MPL, wrap it in a native interface + # for internal use. elif (self.materials_manager is not None and isinstance(self.materials_manager, ICryptographicMaterialsManager)): - # If the provided materials manager implements an MPL interface, - # wrap it in a native interface. self.materials_manager = CryptoMaterialsManagerFromMPL(self.materials_manager) def _no_mpl_attrs_post_init(self): @@ -625,15 +625,18 @@ def generate_header(self, message_id): if self._encryption_materials.algorithm.message_format_version == 0x02: version = SerializationVersion.V2 + # If the underlying materials_provider provided required_encryption_context_keys + # (ex. if the materials_provider is a required encryption context CMM), + # then partition the encryption context based on those keys. if hasattr(self._encryption_materials, "required_encryption_context_keys"): self._required_encryption_context = {} self._stored_encryption_context = {} - print(f"{self._encryption_materials.encryption_context=}") for (k, v) in self._encryption_materials.encryption_context.items(): if k in self._encryption_materials.required_encryption_context_keys: self._required_encryption_context[k] = v else: self._stored_encryption_context[k] = v + # Otherwise, store all encryption context with the message. else: self._stored_encryption_context = self._encryption_materials.encryption_context, self._required_encryption_context = None @@ -667,6 +670,8 @@ def _write_header(self): """Builds the message header and writes it to the output stream.""" self.output_buffer += serialize_header(header=self._header, signer=self.signer) + # If there is _required_encryption_context, + # serialize it, then authenticate it if self._required_encryption_context is not None: required_ec_serialized = aws_encryption_sdk.internal.formatting.encryption_context.serialize_encryption_context( self._required_encryption_context @@ -679,6 +684,7 @@ def _write_header(self): signer=self.signer, required_encryption_context_bytes=required_ec_serialized, ) + # Otherwise, do not pass in any required encryption context else: self.output_buffer += serialize_header_auth( version=self._header.version, @@ -884,6 +890,9 @@ class DecryptorConfig(_ClientConfig): :param int max_body_length: Maximum frame size (or content length for non-framed messages) in bytes to read from ciphertext message. + :param dict encryption_context: Dictionary defining encryption context to validate + on decrypt. This is ONLY validated on decrypt if using the required encryption + context CMM from the aws-cryptographic-materialproviders library. """ max_body_length = attr.ib( @@ -971,9 +980,9 @@ def _read_header(self): found=header.frame_length, custom=self.config.max_body_length ) ) - - print(f"{self.config.encryption_context=}") - + + # If encryption_context is provided on decrypt, + # pass it to the DecryptionMaterialsRequest if hasattr(self.config, "encryption_context"): decrypt_materials_request = DecryptionMaterialsRequest( encrypted_data_keys=header.encrypted_data_keys, @@ -989,9 +998,12 @@ def _read_header(self): encryption_context=header.encryption_context, commitment_policy=self.config.commitment_policy, ) - print(f"{decrypt_materials_request=}") + decryption_materials = self.config.materials_manager.decrypt_materials(request=decrypt_materials_request) + # If the materials_manager passed required_encryption_context_keys, + # get the items out of the encryption_context with the keys. + # The items are used in header validation. if hasattr(decryption_materials, "required_encryption_context_keys"): self._required_encryption_context = {} for (k, v) in decryption_materials.encryption_context.items(): @@ -1038,7 +1050,12 @@ def _read_header(self): "message. Halting processing of this message." ) + # If _required_encryption_context is present, + # serialize it and pass it to validate_header. if self._required_encryption_context is not None: + # The authenticated only encryption context is all encryption context key-value pairs where the + # key exists in Required Encryption Context Keys. It is then serialized according to the + # message header Key Value Pairs. required_ec_serialized = aws_encryption_sdk.internal.formatting.encryption_context.serialize_encryption_context( self._required_encryption_context ) @@ -1046,6 +1063,9 @@ def _read_header(self): validate_header( header=header, header_auth=header_auth, + # When verifying the header, the AAD input to the authenticated encryption algorithm + # specified by the algorithm suite is the message header body and the serialized + # authenticated only encryption context. raw_header=raw_header + required_ec_serialized, data_key=self._derived_data_key ) From 21a8c938eb44f352a76fd358e1c865f453e3f75a Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 28 Feb 2024 16:08:02 -0800 Subject: [PATCH 143/376] protect --- examples/src/basic_encryption.py | 2 +- examples/test/test_i_basic_encryption.py | 2 +- src/aws_encryption_sdk/streaming_client.py | 15 ++++++++++++++- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/examples/src/basic_encryption.py b/examples/src/basic_encryption.py index cfe8ac791..68be5c594 100644 --- a/examples/src/basic_encryption.py +++ b/examples/src/basic_encryption.py @@ -38,7 +38,7 @@ def cycle_string(key_arn, source_plaintext, botocore_session=None): ciphertext, encryptor_header = client.encrypt(source=source_plaintext, key_provider=master_key_provider) # Decrypt the ciphertext - cycled_plaintext, decrypted_header = client.decrypt(source=ciphertext, key_provider=master_key_provider) + cycled_plaintext, decrypted_header = client.decrypt(source=ciphertext, key_provider=master_key_provider, encryption_context={"a": "v"}) # Verify that the "cycled" (encrypted, then decrypted) plaintext is identical to the source plaintext assert cycled_plaintext == source_plaintext diff --git a/examples/test/test_i_basic_encryption.py b/examples/test/test_i_basic_encryption.py index f2a4fab51..5f509800e 100644 --- a/examples/test/test_i_basic_encryption.py +++ b/examples/test/test_i_basic_encryption.py @@ -23,5 +23,5 @@ def test_cycle_string(): plaintext = static_plaintext - cmk_arn = get_cmk_arn() + cmk_arn = "arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f" cycle_string(key_arn=cmk_arn, source_plaintext=plaintext, botocore_session=botocore.session.Session()) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 568543863..e3513de97 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -638,7 +638,7 @@ def generate_header(self, message_id): self._stored_encryption_context[k] = v # Otherwise, store all encryption context with the message. else: - self._stored_encryption_context = self._encryption_materials.encryption_context, + self._stored_encryption_context = self._encryption_materials.encryption_context self._required_encryption_context = None kwargs = dict( @@ -1001,6 +1001,19 @@ def _read_header(self): decryption_materials = self.config.materials_manager.decrypt_materials(request=decrypt_materials_request) + # Guard against possible misunderstanding of "encryption context on decrypt". + # The `encryption_context` parameter on the client's `decrypt` method + # is ONLY meant to be used in conjunction with a `materials_manager` + # that validates the encryption context provided to the decrypt method + if hasattr(self.config, "encryption_context"): + try: + assert hasattr(decryption_materials, "required_encryption_context_keys") + except AssertionError as e: + raise ValueError("encryption_context on decrypt is not supported with the configured CMM: " + f"{self.config.materials_manager}. " + "You MUST pass a CMM that supports required encryption context keys to " + "validate encryption context on decrypt.") + # If the materials_manager passed required_encryption_context_keys, # get the items out of the encryption_context with the keys. # The items are used in header validation. From de870b8495dcb34bb795edfa2bd72e24bf8f3c61 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 28 Feb 2024 16:08:23 -0800 Subject: [PATCH 144/376] ex --- .../required_encryption_context_cmm.py | 130 ++++++++++++++++++ .../test_i_required_encryption_context_cmm.py | 13 ++ src/aws_encryption_sdk/streaming_client.py | 6 +- 3 files changed, 147 insertions(+), 2 deletions(-) create mode 100644 examples/src/keyrings/required_encryption_context_cmm.py create mode 100644 examples/test/keyrings/test_i_required_encryption_context_cmm.py diff --git a/examples/src/keyrings/required_encryption_context_cmm.py b/examples/src/keyrings/required_encryption_context_cmm.py new file mode 100644 index 000000000..c36a4b2bd --- /dev/null +++ b/examples/src/keyrings/required_encryption_context_cmm.py @@ -0,0 +1,130 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Example showing basic encryption and decryption of a value already in memory.""" +import sys + +import boto3 +# Ignore missing MPL for pylint, but the MPL is required for this example +# noqa pylint: disable=import-error +from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders +from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig +from aws_cryptographic_materialproviders.mpl.models import ( + CacheTypeDefault, + CreateAwsKmsKeyringInput, + CreateDefaultCryptographicMaterialsManagerInput, + CreateRequiredEncryptionContextCMMInput, + DefaultCache, +) +from aws_cryptographic_materialproviders.mpl.references import ( + IKeyring, + ICryptographicMaterialsManager, +) +from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL +from typing import Dict + +import aws_encryption_sdk +from aws_encryption_sdk import CommitmentPolicy +from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError + +from .example_branch_key_id_supplier import ExampleBranchKeyIdSupplier + +module_root_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(module_root_dir) + +EXAMPLE_DATA: bytes = b"Hello World" + + +def encrypt_and_decrypt_with_keyring( + kms_key_id: str +): + """Creates a hierarchical keyring using the provided resources, then encrypts and decrypts a string with it.""" + # 1. Instantiate the encryption SDK client. + # This builds the client with the REQUIRE_ENCRYPT_REQUIRE_DECRYPT commitment policy, + # which enforces that this client only encrypts using committing algorithm suites and enforces + # that this client will only decrypt encrypted messages that were created with a committing + # algorithm suite. + # This is the default commitment policy if you were to build the client as + # `client = aws_encryption_sdk.EncryptionSDKClient()`. + + client = aws_encryption_sdk.EncryptionSDKClient( + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT + ) + + # 7. Create an encryption context. + #// Most encrypted data should have an associated encryption context + #// to protect integrity. This sample uses placeholder values. + #// For more information see: + #// blogs.aws.amazon.com/security/post/Tx2LZ6WBJJANTNW/How-to-Protect-the-Integrity-of-Your-Encrypted-Data-by-Using-AWS-Key-Management + encryption_context: Dict[str, str] = { + "key1": "value1", + "key2": "value2", + "requiredKey1": "requiredValue1", + "requiredKey2": "requiredValue2", + } + + #// 3. Create list of required encryption context keys. + #// This is a list of keys that must be present in the encryption context. + required_encryption_context_keys: List[str] = ["requiredKey1", "requiredKey2"] + + #// 4. Create the AWS KMS keyring. + mpl: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + config=MaterialProvidersConfig() + ) + keyring_input: CreateAwsKmsKeyringInput = CreateAwsKmsKeyringInput( + kms_key_id=kms_key_id, + kms_client=boto3.client('kms', region_name="us-west-2") + ) + kms_keyring: IKeyring = mpl.create_aws_kms_keyring(keyring_input) + + #// 5. Create the required encryption context CMM. + underlying_cmm: ICryptographicMaterialsManager = \ + mpl.create_default_cryptographic_materials_manager( + CreateDefaultCryptographicMaterialsManagerInput( + keyring=kms_keyring + ) + ) + + required_ec_cmm: ICryptographicMaterialsManager = \ + mpl.create_required_encryption_context_cmm( + CreateRequiredEncryptionContextCMMInput( + required_encryption_context_keys=required_encryption_context_keys, + underlying_cmm=underlying_cmm, + ) + ) + + # 6. Encrypt the data + ciphertext, _ = client.encrypt( + source=EXAMPLE_DATA, + materials_manager=required_ec_cmm, + encryption_context=encryption_context + ) + + # // 7. Reproduce the encryption context. + # // The reproduced encryption context MUST contain a value for + # // every key in the configured required encryption context keys during encryption with + # // Required Encryption Context CMM. + reproduced_encryption_context: Dict[str, str] = { + "requiredKey1": "requiredValue1", + "requiredKey2": "requiredValue2", + } + + # 8. Decrypt the data + plaintext_bytes_A, _ = client.decrypt( + source=ciphertext, + materials_manager=required_ec_cmm, + encryption_context=reproduced_encryption_context + ) + assert plaintext_bytes_A == EXAMPLE_DATA + + + # 9. If we don't provide the required encryption context, this should fail + try: + plaintext_bytes_A, _ = client.decrypt( + source=ciphertext, + materials_manager=required_ec_cmm, + # no encryption context while using required encryption context CMM makes decryption fail + ) + assert plaintext_bytes_A == EXAMPLE_DATA + except AWSEncryptionSDKClientError: + pass \ No newline at end of file diff --git a/examples/test/keyrings/test_i_required_encryption_context_cmm.py b/examples/test/keyrings/test_i_required_encryption_context_cmm.py new file mode 100644 index 000000000..9512a06ee --- /dev/null +++ b/examples/test/keyrings/test_i_required_encryption_context_cmm.py @@ -0,0 +1,13 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Unit test suite for the hierarchical keyring example.""" +import pytest + +from ...src.keyrings.required_encryption_context_cmm import encrypt_and_decrypt_with_keyring + +pytestmark = [pytest.mark.examples] + + +def test_encrypt_and_decrypt_with_keyring(): + key_arn = "arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f" + encrypt_and_decrypt_with_keyring(key_arn) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index e3513de97..58cd52051 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -1003,8 +1003,10 @@ def _read_header(self): # Guard against possible misunderstanding of "encryption context on decrypt". # The `encryption_context` parameter on the client's `decrypt` method - # is ONLY meant to be used in conjunction with a `materials_manager` - # that validates the encryption context provided to the decrypt method + # is ONLY meant to be used in conjunction with a `materials_manager` + # that validates the encryption context provided to the decrypt method. + # This guards against accidentially passing encryption context on decrypt + # and not realizing nothing is being validated. if hasattr(self.config, "encryption_context"): try: assert hasattr(decryption_materials, "required_encryption_context_keys") From eedf1a3c269473117dc284c52b0d34f63943915b Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 28 Feb 2024 16:24:03 -0800 Subject: [PATCH 145/376] changes --- examples/src/basic_encryption.py | 2 +- .../keyrings/required_encryption_context_cmm.py | 10 +++++----- src/aws_encryption_sdk/streaming_client.py | 17 +---------------- 3 files changed, 7 insertions(+), 22 deletions(-) diff --git a/examples/src/basic_encryption.py b/examples/src/basic_encryption.py index 68be5c594..cfe8ac791 100644 --- a/examples/src/basic_encryption.py +++ b/examples/src/basic_encryption.py @@ -38,7 +38,7 @@ def cycle_string(key_arn, source_plaintext, botocore_session=None): ciphertext, encryptor_header = client.encrypt(source=source_plaintext, key_provider=master_key_provider) # Decrypt the ciphertext - cycled_plaintext, decrypted_header = client.decrypt(source=ciphertext, key_provider=master_key_provider, encryption_context={"a": "v"}) + cycled_plaintext, decrypted_header = client.decrypt(source=ciphertext, key_provider=master_key_provider) # Verify that the "cycled" (encrypted, then decrypted) plaintext is identical to the source plaintext assert cycled_plaintext == source_plaintext diff --git a/examples/src/keyrings/required_encryption_context_cmm.py b/examples/src/keyrings/required_encryption_context_cmm.py index c36a4b2bd..51455fd4f 100644 --- a/examples/src/keyrings/required_encryption_context_cmm.py +++ b/examples/src/keyrings/required_encryption_context_cmm.py @@ -109,7 +109,7 @@ def encrypt_and_decrypt_with_keyring( "requiredKey2": "requiredValue2", } - # 8. Decrypt the data + # # 8. Decrypt the data plaintext_bytes_A, _ = client.decrypt( source=ciphertext, materials_manager=required_ec_cmm, @@ -117,14 +117,14 @@ def encrypt_and_decrypt_with_keyring( ) assert plaintext_bytes_A == EXAMPLE_DATA - - # 9. If we don't provide the required encryption context, this should fail + # 9. If we don't provide the required encryption context, + # decryption will fail. try: plaintext_bytes_A, _ = client.decrypt( source=ciphertext, materials_manager=required_ec_cmm, # no encryption context while using required encryption context CMM makes decryption fail ) - assert plaintext_bytes_A == EXAMPLE_DATA + raise Exception("If this exception is raised, decryption somehow succeeded!") except AWSEncryptionSDKClientError: - pass \ No newline at end of file + pass diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 58cd52051..7e6ede8cd 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -1000,22 +1000,7 @@ def _read_header(self): ) decryption_materials = self.config.materials_manager.decrypt_materials(request=decrypt_materials_request) - - # Guard against possible misunderstanding of "encryption context on decrypt". - # The `encryption_context` parameter on the client's `decrypt` method - # is ONLY meant to be used in conjunction with a `materials_manager` - # that validates the encryption context provided to the decrypt method. - # This guards against accidentially passing encryption context on decrypt - # and not realizing nothing is being validated. - if hasattr(self.config, "encryption_context"): - try: - assert hasattr(decryption_materials, "required_encryption_context_keys") - except AssertionError as e: - raise ValueError("encryption_context on decrypt is not supported with the configured CMM: " - f"{self.config.materials_manager}. " - "You MUST pass a CMM that supports required encryption context keys to " - "validate encryption context on decrypt.") - + # If the materials_manager passed required_encryption_context_keys, # get the items out of the encryption_context with the keys. # The items are used in header validation. From 1db73ebee0d6df0e2ca76b4f16998f018a042d0d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 28 Feb 2024 16:58:26 -0800 Subject: [PATCH 146/376] changes --- examples/src/keyrings/hierarchical_keyring.py | 33 ++++++++++++++- .../required_encryption_context_cmm.py | 40 +++++++++++-------- .../keyrings/test_i_hierarchical_keyring.py | 2 +- .../test_i_required_encryption_context_cmm.py | 2 +- examples/test/test_i_basic_encryption.py | 2 +- src/aws_encryption_sdk/streaming_client.py | 28 ++++++------- 6 files changed, 72 insertions(+), 35 deletions(-) diff --git a/examples/src/keyrings/hierarchical_keyring.py b/examples/src/keyrings/hierarchical_keyring.py index aa87485f9..b75421359 100644 --- a/examples/src/keyrings/hierarchical_keyring.py +++ b/examples/src/keyrings/hierarchical_keyring.py @@ -1,6 +1,36 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""Example showing basic encryption and decryption of a value already in memory.""" +""" +This example sets up the Hierarchical Keyring, which establishes a key hierarchy where "branch" +keys are persisted in DynamoDb. These branch keys are used to protect your data keys, and these +branch keys are themselves protected by a KMS Key. + +Establishing a key hierarchy like this has two benefits: +First, by caching the branch key material, and only calling KMS to re-establish authentication +regularly according to your configured TTL, you limit how often you need to call KMS to protect +your data. This is a performance security tradeoff, where your authentication, audit, and logging +from KMS is no longer one-to-one with every encrypt or decrypt call. Additionally, KMS Cloudtrail +cannot be used to distinguish Encrypt and Decrypt calls, and you cannot restrict who has +Encryption rights from who has Decryption rights since they both ONLY need KMS:Decrypt. However, +the benefit is that you no longer have to make a network call to KMS for every encrypt or +decrypt. + +Second, this key hierarchy facilitates cryptographic isolation of a tenant's data in a +multi-tenant data store. Each tenant can have a unique Branch Key, that is only used to protect +the tenant's data. You can either statically configure a single branch key to ensure you are +restricting access to a single tenant, or you can implement an interface that selects the Branch +Key based on the Encryption Context. + +This example demonstrates configuring a Hierarchical Keyring with a Branch Key ID Supplier to +encrypt and decrypt data for two separate tenants. + +This example requires access to the DDB Table where you are storing the Branch Keys. This +table must be configured with the following primary key configuration: - Partition key is named +"partition_key" with type (S) - Sort key is named "sort_key" with type (S) + +This example also requires using a KMS Key. You need the following access on this key: - +GenerateDataKeyWithoutPlaintext - Decrypt +""" import sys import boto3 @@ -25,6 +55,7 @@ from .example_branch_key_id_supplier import ExampleBranchKeyIdSupplier +# TODO-MPL: Remove this as part of removing PYTHONPATH hacks module_root_dir = '/'.join(__file__.split("/")[:-1]) sys.path.append(module_root_dir) diff --git a/examples/src/keyrings/required_encryption_context_cmm.py b/examples/src/keyrings/required_encryption_context_cmm.py index 51455fd4f..f3d58c922 100644 --- a/examples/src/keyrings/required_encryption_context_cmm.py +++ b/examples/src/keyrings/required_encryption_context_cmm.py @@ -1,6 +1,11 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""Example showing basic encryption and decryption of a value already in memory.""" +""" +Demonstrate an encrypt/decrypt cycle using a Required Encryption Context CMM. +A required encryption context CMM asks for required keys in the encryption context field +on encrypt such that they will not be stored on the message, but WILL be included in the header signature. +On decrypt, the client MUST supply the key/value pair(s) that were not stored to successfully decrypt the message. +""" import sys import boto3 @@ -28,6 +33,7 @@ from .example_branch_key_id_supplier import ExampleBranchKeyIdSupplier +# TODO-MPL: Remove this as part of removing PYTHONPATH hacks module_root_dir = '/'.join(__file__.split("/")[:-1]) sys.path.append(module_root_dir) @@ -51,11 +57,11 @@ def encrypt_and_decrypt_with_keyring( commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT ) - # 7. Create an encryption context. - #// Most encrypted data should have an associated encryption context - #// to protect integrity. This sample uses placeholder values. - #// For more information see: - #// blogs.aws.amazon.com/security/post/Tx2LZ6WBJJANTNW/How-to-Protect-the-Integrity-of-Your-Encrypted-Data-by-Using-AWS-Key-Management + # 2. Create an encryption context. + # Most encrypted data should have an associated encryption context + # to protect integrity. This sample uses placeholder values. + # For more information see: + # blogs.aws.amazon.com/security/post/Tx2LZ6WBJJANTNW/How-to-Protect-the-Integrity-of-Your-Encrypted-Data-by-Using-AWS-Key-Management # noqa: E501 encryption_context: Dict[str, str] = { "key1": "value1", "key2": "value2", @@ -63,11 +69,11 @@ def encrypt_and_decrypt_with_keyring( "requiredKey2": "requiredValue2", } - #// 3. Create list of required encryption context keys. - #// This is a list of keys that must be present in the encryption context. + # 3. Create list of required encryption context keys. + # This is a list of keys that must be present in the encryption context. required_encryption_context_keys: List[str] = ["requiredKey1", "requiredKey2"] - #// 4. Create the AWS KMS keyring. + # 4. Create the AWS KMS keyring. mpl: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( config=MaterialProvidersConfig() ) @@ -77,7 +83,7 @@ def encrypt_and_decrypt_with_keyring( ) kms_keyring: IKeyring = mpl.create_aws_kms_keyring(keyring_input) - #// 5. Create the required encryption context CMM. + # 5. Create the required encryption context CMM. underlying_cmm: ICryptographicMaterialsManager = \ mpl.create_default_cryptographic_materials_manager( CreateDefaultCryptographicMaterialsManagerInput( @@ -100,16 +106,16 @@ def encrypt_and_decrypt_with_keyring( encryption_context=encryption_context ) - # // 7. Reproduce the encryption context. - # // The reproduced encryption context MUST contain a value for - # // every key in the configured required encryption context keys during encryption with - # // Required Encryption Context CMM. + # 7. Reproduce the encryption context. + # The reproduced encryption context MUST contain a value for + # every key in the configured required encryption context keys during encryption with + # Required Encryption Context CMM. reproduced_encryption_context: Dict[str, str] = { "requiredKey1": "requiredValue1", "requiredKey2": "requiredValue2", } - # # 8. Decrypt the data + # 8. Decrypt the data plaintext_bytes_A, _ = client.decrypt( source=ciphertext, materials_manager=required_ec_cmm, @@ -117,13 +123,13 @@ def encrypt_and_decrypt_with_keyring( ) assert plaintext_bytes_A == EXAMPLE_DATA - # 9. If we don't provide the required encryption context, + # 9. Extra: Demonstrate that if we don't provide the required encryption context, # decryption will fail. try: plaintext_bytes_A, _ = client.decrypt( source=ciphertext, materials_manager=required_ec_cmm, - # no encryption context while using required encryption context CMM makes decryption fail + # No encryption context while using required encryption context CMM makes decryption fail. ) raise Exception("If this exception is raised, decryption somehow succeeded!") except AWSEncryptionSDKClientError: diff --git a/examples/test/keyrings/test_i_hierarchical_keyring.py b/examples/test/keyrings/test_i_hierarchical_keyring.py index 4cae478d7..c4583534a 100644 --- a/examples/test/keyrings/test_i_hierarchical_keyring.py +++ b/examples/test/keyrings/test_i_hierarchical_keyring.py @@ -1,6 +1,6 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""Unit test suite for the hierarchical keyring example.""" +"""Test suite for the hierarchical keyring example.""" import pytest from ...src.keyrings.hierarchical_keyring import encrypt_and_decrypt_with_keyring diff --git a/examples/test/keyrings/test_i_required_encryption_context_cmm.py b/examples/test/keyrings/test_i_required_encryption_context_cmm.py index 9512a06ee..724705faa 100644 --- a/examples/test/keyrings/test_i_required_encryption_context_cmm.py +++ b/examples/test/keyrings/test_i_required_encryption_context_cmm.py @@ -1,6 +1,6 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""Unit test suite for the hierarchical keyring example.""" +"""Test suite for the required encryption context CMM example.""" import pytest from ...src.keyrings.required_encryption_context_cmm import encrypt_and_decrypt_with_keyring diff --git a/examples/test/test_i_basic_encryption.py b/examples/test/test_i_basic_encryption.py index 5f509800e..f2a4fab51 100644 --- a/examples/test/test_i_basic_encryption.py +++ b/examples/test/test_i_basic_encryption.py @@ -23,5 +23,5 @@ def test_cycle_string(): plaintext = static_plaintext - cmk_arn = "arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f" + cmk_arn = get_cmk_arn() cycle_string(key_arn=cmk_arn, source_plaintext=plaintext, botocore_session=botocore.session.Session()) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 7e6ede8cd..34ba01c59 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -80,8 +80,8 @@ CreateDefaultCryptographicMaterialsManagerInput, ) from aws_cryptographic_materialproviders.mpl.references import ( - ICryptographicMaterialsManager, - IKeyring, + ICryptographicMaterialsManager as MPL_ICryptographicMaterialsManager, + IKeyring as MPL_IKeyring, ) _HAS_MPL = True @@ -141,7 +141,7 @@ class _ClientConfig(object): # pylint: disable=too-many-instance-attributes default=None, validator=attr.validators.optional( attr.validators.instance_of( - (CryptoMaterialsManager, ICryptographicMaterialsManager) + (CryptoMaterialsManager, MPL_ICryptographicMaterialsManager) ) ) ) @@ -161,7 +161,7 @@ class _ClientConfig(object): # pylint: disable=too-many-instance-attributes if _HAS_MPL: # Keyrings are only available if the MPL is installed in the runtime keyring = attr.ib( - hash=True, default=None, validator=attr.validators.optional(attr.validators.instance_of(IKeyring)) + hash=True, default=None, validator=attr.validators.optional(attr.validators.instance_of(MPL_IKeyring)) ) source_length = attr.ib( hash=True, default=None, validator=attr.validators.optional(attr.validators.instance_of(six.integer_types)) @@ -202,7 +202,7 @@ def _has_mpl_attrs_post_init(self): # If the provided materials_manager is directly from the MPL, wrap it in a native interface # for internal use. elif (self.materials_manager is not None - and isinstance(self.materials_manager, ICryptographicMaterialsManager)): + and isinstance(self.materials_manager, MPL_ICryptographicMaterialsManager)): self.materials_manager = CryptoMaterialsManagerFromMPL(self.materials_manager) def _no_mpl_attrs_post_init(self): @@ -437,10 +437,10 @@ class EncryptorConfig(_ClientConfig): :param key_provider: `MasterKeyProvider` from which to obtain data keys for encryption (either `materials_manager` or `key_provider` required) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider - :param keyring: `IKeyring` from the aws_cryptographic_materialproviders library + :param keyring: `MPL_IKeyring` from the aws_cryptographic_materialproviders library which handles encryption and decryption :type keyring: - aws_cryptographic_materialproviders.mpl.references.IKeyring + aws_cryptographic_materialproviders.mpl.references.MPL_IKeyring :param int source_length: Length of source data (optional) .. note:: @@ -492,10 +492,10 @@ class StreamEncryptor(_EncryptionStream): # pylint: disable=too-many-instance-a :param key_provider: `MasterKeyProvider` from which to obtain data keys for encryption (either `materials_manager` or `key_provider` required) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider - :param keyring: `IKeyring` from the aws_cryptographic_materialproviders library + :param keyring: `MPL_IKeyring` from the aws_cryptographic_materialproviders library which handles encryption and decryption :type keyring: - aws_cryptographic_materialproviders.mpl.references.IKeyring + aws_cryptographic_materialproviders.mpl.references.MPL_IKeyring :param int source_length: Length of source data (optional) .. note:: @@ -878,10 +878,10 @@ class DecryptorConfig(_ClientConfig): :param key_provider: `MasterKeyProvider` from which to obtain data keys for decryption (either `keyring`, `materials_manager` or `key_provider` required) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider - :param keyring: `IKeyring` from the aws_cryptographic_materialproviders library + :param keyring: `MPL_IKeyring` from the aws_cryptographic_materialproviders library which handles encryption and decryption :type keyring: - aws_cryptographic_materialproviders.mpl.references.IKeyring + aws_cryptographic_materialproviders.mpl.references.MPL_IKeyring :param int source_length: Length of source data (optional) .. note:: @@ -926,10 +926,10 @@ class StreamDecryptor(_EncryptionStream): # pylint: disable=too-many-instance-a :param key_provider: `MasterKeyProvider` from which to obtain data keys for decryption (either `materials_manager` or `key_provider` required) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider - :param keyring: `IKeyring` from the aws_cryptographic_materialproviders library + :param keyring: `MPL_IKeyring` from the aws_cryptographic_materialproviders library which handles encryption and decryption :type keyring: - aws_cryptographic_materialproviders.mpl.references.IKeyring + aws_cryptographic_materialproviders.mpl.references.MPL_IKeyring :param int source_length: Length of source data (optional) .. note:: @@ -1000,7 +1000,7 @@ def _read_header(self): ) decryption_materials = self.config.materials_manager.decrypt_materials(request=decrypt_materials_request) - + # If the materials_manager passed required_encryption_context_keys, # get the items out of the encryption_context with the keys. # The items are used in header validation. From 8415c2cbb2eb138e330816374dccc0f15cce6a38 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 28 Feb 2024 17:00:17 -0800 Subject: [PATCH 147/376] cleanup --- src/aws_encryption_sdk/streaming_client.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 34ba01c59..953a82a66 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -437,10 +437,10 @@ class EncryptorConfig(_ClientConfig): :param key_provider: `MasterKeyProvider` from which to obtain data keys for encryption (either `materials_manager` or `key_provider` required) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider - :param keyring: `MPL_IKeyring` from the aws_cryptographic_materialproviders library + :param keyring: `IKeyring` from the aws_cryptographic_materialproviders library which handles encryption and decryption :type keyring: - aws_cryptographic_materialproviders.mpl.references.MPL_IKeyring + aws_cryptographic_materialproviders.mpl.references.IKeyring :param int source_length: Length of source data (optional) .. note:: @@ -492,10 +492,10 @@ class StreamEncryptor(_EncryptionStream): # pylint: disable=too-many-instance-a :param key_provider: `MasterKeyProvider` from which to obtain data keys for encryption (either `materials_manager` or `key_provider` required) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider - :param keyring: `MPL_IKeyring` from the aws_cryptographic_materialproviders library + :param keyring: `IKeyring` from the aws_cryptographic_materialproviders library which handles encryption and decryption :type keyring: - aws_cryptographic_materialproviders.mpl.references.MPL_IKeyring + aws_cryptographic_materialproviders.mpl.references.IKeyring :param int source_length: Length of source data (optional) .. note:: @@ -878,10 +878,10 @@ class DecryptorConfig(_ClientConfig): :param key_provider: `MasterKeyProvider` from which to obtain data keys for decryption (either `keyring`, `materials_manager` or `key_provider` required) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider - :param keyring: `MPL_IKeyring` from the aws_cryptographic_materialproviders library + :param keyring: `IKeyring` from the aws_cryptographic_materialproviders library which handles encryption and decryption :type keyring: - aws_cryptographic_materialproviders.mpl.references.MPL_IKeyring + aws_cryptographic_materialproviders.mpl.references.IKeyring :param int source_length: Length of source data (optional) .. note:: @@ -926,10 +926,10 @@ class StreamDecryptor(_EncryptionStream): # pylint: disable=too-many-instance-a :param key_provider: `MasterKeyProvider` from which to obtain data keys for decryption (either `materials_manager` or `key_provider` required) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider - :param keyring: `MPL_IKeyring` from the aws_cryptographic_materialproviders library + :param keyring: `IKeyring` from the aws_cryptographic_materialproviders library which handles encryption and decryption :type keyring: - aws_cryptographic_materialproviders.mpl.references.MPL_IKeyring + aws_cryptographic_materialproviders.mpl.references.IKeyring :param int source_length: Length of source data (optional) .. note:: From 20bdaffca7b83444348fdfb0e6abab7c3af5c8c9 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 28 Feb 2024 18:21:42 -0800 Subject: [PATCH 148/376] cleanup --- src/aws_encryption_sdk/streaming_client.py | 2 +- .../test_streaming_client_stream_decryptor.py | 42 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 953a82a66..4a742b91f 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -672,7 +672,7 @@ def _write_header(self): # If there is _required_encryption_context, # serialize it, then authenticate it - if self._required_encryption_context is not None: + if hasattr(self, "_required_encryption_context"): required_ec_serialized = aws_encryption_sdk.internal.formatting.encryption_context.serialize_encryption_context( self._required_encryption_context ) diff --git a/test/unit/test_streaming_client_stream_decryptor.py b/test/unit/test_streaming_client_stream_decryptor.py index e06cad308..4929646b5 100644 --- a/test/unit/test_streaming_client_stream_decryptor.py +++ b/test/unit/test_streaming_client_stream_decryptor.py @@ -193,6 +193,9 @@ def test_read_header(self, mock_derive_datakey, mock_decrypt_materials_request, test_decryptor.source_stream = ct_stream test_decryptor._stream_length = len(VALUES["data_128"]) + # Mock: hasattr(self.config, "encryption_context") returns False + del test_decryptor.config.encryption_context + test_header, test_header_auth = test_decryptor._read_header() self.mock_deserialize_header.assert_called_once_with(ct_stream, None) @@ -230,6 +233,45 @@ def test_read_header(self, mock_derive_datakey, mock_decrypt_materials_request, assert test_header is self.mock_header assert test_header_auth is sentinel.header_auth + @patch("aws_encryption_sdk.streaming_client.derive_data_encryption_key") + @patch("aws_encryption_sdk.streaming_client.DecryptionMaterialsRequest") + @patch("aws_encryption_sdk.streaming_client.Verifier") + # Given: no MPL + @pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") + def test_GIVEN_verification_key_AND_no_mpl_WHEN_read_header_THEN_calls_from_key_bytes( + self, + mock_verifier, + mock_decrypt_materials_request, + *_, + ): + + mock_verifier_instance = MagicMock() + mock_verifier.from_key_bytes.return_value = mock_verifier_instance + ct_stream = io.BytesIO(VALUES["data_128"]) + mock_commitment_policy = MagicMock(__class__=CommitmentPolicy) + test_decryptor = StreamDecryptor( + materials_manager=self.mock_materials_manager, + source=ct_stream, + commitment_policy=mock_commitment_policy, + ) + test_decryptor.source_stream = ct_stream + test_decryptor._stream_length = len(VALUES["data_128"]) + # Given: self.config has "encryption_context" + any_reproduced_ec = {"some": "ec"} + test_decryptor.config.encryption_context = any_reproduced_ec + + # When: read header + test_decryptor._read_header() + + # Then: calls decrypt_materials with reproduced_encryption_context + mock_decrypt_materials_request.assert_called_once_with( + encrypted_data_keys=self.mock_header.encrypted_data_keys, + algorithm=self.mock_header.algorithm, + encryption_context=sentinel.encryption_context, + commitment_policy=mock_commitment_policy, + reproduced_encryption_context=any_reproduced_ec, + ) + @patch("aws_encryption_sdk.streaming_client.DecryptionMaterialsRequest") @patch("aws_encryption_sdk.streaming_client.derive_data_encryption_key") @patch("aws_encryption_sdk.streaming_client.Verifier") From 6bf6094eff239f36c7dffb010e2b59cb7e948ab4 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 28 Feb 2024 18:29:50 -0800 Subject: [PATCH 149/376] cleanup --- .../internal/formatting/serialize.py | 8 +++++--- .../materials_managers/mpl/materials.py | 1 - src/aws_encryption_sdk/streaming_client.py | 18 ++++++++++-------- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/aws_encryption_sdk/internal/formatting/serialize.py b/src/aws_encryption_sdk/internal/formatting/serialize.py index 344c94703..5a054989c 100644 --- a/src/aws_encryption_sdk/internal/formatting/serialize.py +++ b/src/aws_encryption_sdk/internal/formatting/serialize.py @@ -237,7 +237,7 @@ def _serialize_header_auth_v2( for all items whose keys are in the required_encryption_context list. This is ONLY processed if using the aws-cryptographic-materialproviders library AND its required encryption context CMM. (optional) - :type required_encryption_context_bytes: bytes + :type required_encryption_context_bytes: bytes :returns: Serialized header authentication data :rtype: bytes """ @@ -294,14 +294,16 @@ def serialize_header_auth( This is ONLY processed if using the aws-cryptographic-materialproviders library AND its required encryption context CMM AND if using the v2 message format. (optional) - :type required_encryption_context_bytes: bytes + :type required_encryption_context_bytes: bytes :returns: Serialized header authentication data :rtype: bytes """ if version == SerializationVersion.V1: return _serialize_header_auth_v1(algorithm, header, data_encryption_key, signer) elif version == SerializationVersion.V2: - return _serialize_header_auth_v2(algorithm, header, data_encryption_key, signer, required_encryption_context_bytes) + return _serialize_header_auth_v2( + algorithm, header, data_encryption_key, signer, required_encryption_context_bytes + ) else: raise SerializationError("Unrecognized message format version: {}".format(version)) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index 5b066c7c7..5e4a66318 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -95,7 +95,6 @@ def data_encryption_key(self) -> DataKey: def signing_key(self) -> bytes: """Materials' signing key.""" return self.mpl_materials.signing_key - @property def required_encryption_context_keys(self) -> bytes: diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 4a742b91f..048a6caa3 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -198,7 +198,7 @@ def _has_mpl_attrs_post_init(self): # Wrap MPL error into the ESDK error type # so customers only have to catch ESDK error types. raise AWSEncryptionSDKClientError(mpl_exception) - + # If the provided materials_manager is directly from the MPL, wrap it in a native interface # for internal use. elif (self.materials_manager is not None @@ -673,9 +673,10 @@ def _write_header(self): # If there is _required_encryption_context, # serialize it, then authenticate it if hasattr(self, "_required_encryption_context"): - required_ec_serialized = aws_encryption_sdk.internal.formatting.encryption_context.serialize_encryption_context( - self._required_encryption_context - ) + required_ec_serialized = \ + aws_encryption_sdk.internal.formatting.encryption_context.serialize_encryption_context( + self._required_encryption_context + ) self.output_buffer += serialize_header_auth( version=self._header.version, algorithm=self._encryption_materials.algorithm, @@ -955,7 +956,7 @@ def _prep_message(self): self._prep_non_framed() self._message_prepped = True - def _read_header(self): + def _read_header(self): # noqa: C901 """Reads the message header from the input stream. :returns: tuple containing deserialized header and header_auth objects @@ -1056,9 +1057,10 @@ def _read_header(self): # The authenticated only encryption context is all encryption context key-value pairs where the # key exists in Required Encryption Context Keys. It is then serialized according to the # message header Key Value Pairs. - required_ec_serialized = aws_encryption_sdk.internal.formatting.encryption_context.serialize_encryption_context( - self._required_encryption_context - ) + required_ec_serialized = \ + aws_encryption_sdk.internal.formatting.encryption_context.serialize_encryption_context( + self._required_encryption_context + ) validate_header( header=header, From febe6dba05160087e6ac905f2f1582c79c56a1e8 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 28 Feb 2024 18:38:46 -0800 Subject: [PATCH 150/376] cleanup --- .../keyrings/required_encryption_context_cmm.py | 14 +++----------- .../internal/formatting/serialize.py | 10 +++++----- .../materials_managers/mpl/materials.py | 4 ++++ src/aws_encryption_sdk/streaming_client.py | 17 +++++++++-------- .../test_streaming_client_stream_decryptor.py | 2 +- 5 files changed, 22 insertions(+), 25 deletions(-) diff --git a/examples/src/keyrings/required_encryption_context_cmm.py b/examples/src/keyrings/required_encryption_context_cmm.py index f3d58c922..9f8de9976 100644 --- a/examples/src/keyrings/required_encryption_context_cmm.py +++ b/examples/src/keyrings/required_encryption_context_cmm.py @@ -14,25 +14,17 @@ from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig from aws_cryptographic_materialproviders.mpl.models import ( - CacheTypeDefault, CreateAwsKmsKeyringInput, CreateDefaultCryptographicMaterialsManagerInput, CreateRequiredEncryptionContextCMMInput, - DefaultCache, ) -from aws_cryptographic_materialproviders.mpl.references import ( - IKeyring, - ICryptographicMaterialsManager, -) -from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL -from typing import Dict +from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager, IKeyring +from typing import Dict, List import aws_encryption_sdk from aws_encryption_sdk import CommitmentPolicy from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError -from .example_branch_key_id_supplier import ExampleBranchKeyIdSupplier - # TODO-MPL: Remove this as part of removing PYTHONPATH hacks module_root_dir = '/'.join(__file__.split("/")[:-1]) @@ -98,7 +90,7 @@ def encrypt_and_decrypt_with_keyring( underlying_cmm=underlying_cmm, ) ) - + # 6. Encrypt the data ciphertext, _ = client.encrypt( source=EXAMPLE_DATA, diff --git a/src/aws_encryption_sdk/internal/formatting/serialize.py b/src/aws_encryption_sdk/internal/formatting/serialize.py index 5a054989c..66f4800de 100644 --- a/src/aws_encryption_sdk/internal/formatting/serialize.py +++ b/src/aws_encryption_sdk/internal/formatting/serialize.py @@ -223,7 +223,7 @@ def _serialize_header_auth_v2( header, data_encryption_key, signer=None, - required_encryption_context_bytes=None + required_ec_bytes=None ): """Creates serialized header authentication data for messages in serialization version V2. @@ -241,7 +241,7 @@ def _serialize_header_auth_v2( :returns: Serialized header authentication data :rtype: bytes """ - if required_encryption_context_bytes is None: + if required_ec_bytes is None: header_auth = encrypt( algorithm=algorithm, key=data_encryption_key, @@ -259,7 +259,7 @@ def _serialize_header_auth_v2( # be the encryption context in the encryption materials filtered to only contain key value # pairs listed in the encryption material's required encryption context keys serialized # according to the encryption context serialization specification. - associated_data=header + required_encryption_context_bytes, + associated_data=header + required_ec_bytes, iv=header_auth_iv(algorithm), ) output = struct.pack( @@ -277,7 +277,7 @@ def serialize_header_auth( header, data_encryption_key, signer=None, - required_encryption_context_bytes=None + required_ec_bytes=None ): """Creates serialized header authentication data. @@ -302,7 +302,7 @@ def serialize_header_auth( return _serialize_header_auth_v1(algorithm, header, data_encryption_key, signer) elif version == SerializationVersion.V2: return _serialize_header_auth_v2( - algorithm, header, data_encryption_key, signer, required_encryption_context_bytes + algorithm, header, data_encryption_key, signer, required_ec_bytes ) else: raise SerializationError("Unrecognized message format version: {}".format(version)) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index 5e4a66318..54ea21b39 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -97,6 +97,8 @@ def signing_key(self) -> bytes: return self.mpl_materials.signing_key @property + # Pylint thinks this name is too long, but it's the best descriptor for this... + # pylint: disable=invalid-name def required_encryption_context_keys(self) -> bytes: """Materials' required encryption context keys.""" return self.mpl_materials.required_encryption_context_keys @@ -148,6 +150,8 @@ def encryption_context(self) -> Dict[str, str]: return self.mpl_materials.encryption_context @property + # Pylint thinks this name is too long, but it's the best descriptor for this... + # pylint: disable=invalid-name def required_encryption_context_keys(self) -> bytes: """Materials' required encryption context keys.""" return self.mpl_materials.required_encryption_context_keys diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 048a6caa3..bbae73bef 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -76,9 +76,7 @@ from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException - from aws_cryptographic_materialproviders.mpl.models import ( - CreateDefaultCryptographicMaterialsManagerInput, - ) + from aws_cryptographic_materialproviders.mpl.models import CreateDefaultCryptographicMaterialsManagerInput from aws_cryptographic_materialproviders.mpl.references import ( ICryptographicMaterialsManager as MPL_ICryptographicMaterialsManager, IKeyring as MPL_IKeyring, @@ -631,11 +629,11 @@ def generate_header(self, message_id): if hasattr(self._encryption_materials, "required_encryption_context_keys"): self._required_encryption_context = {} self._stored_encryption_context = {} - for (k, v) in self._encryption_materials.encryption_context.items(): - if k in self._encryption_materials.required_encryption_context_keys: - self._required_encryption_context[k] = v + for (key, value) in self._encryption_materials.encryption_context.items(): + if key in self._encryption_materials.required_encryption_context_keys: + self._required_encryption_context[key] = value else: - self._stored_encryption_context[k] = v + self._stored_encryption_context[key] = value # Otherwise, store all encryption context with the message. else: self._stored_encryption_context = self._encryption_materials.encryption_context @@ -956,7 +954,10 @@ def _prep_message(self): self._prep_non_framed() self._message_prepped = True - def _read_header(self): # noqa: C901 + # TODO-MPL: Refactor this function, remove these linter disablers + # noqa: C901 + # pylint: disable=too-many-branches + def _read_header(self): """Reads the message header from the input stream. :returns: tuple containing deserialized header and header_auth objects diff --git a/test/unit/test_streaming_client_stream_decryptor.py b/test/unit/test_streaming_client_stream_decryptor.py index 4929646b5..2066dcbdb 100644 --- a/test/unit/test_streaming_client_stream_decryptor.py +++ b/test/unit/test_streaming_client_stream_decryptor.py @@ -238,7 +238,7 @@ def test_read_header(self, mock_derive_datakey, mock_decrypt_materials_request, @patch("aws_encryption_sdk.streaming_client.Verifier") # Given: no MPL @pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") - def test_GIVEN_verification_key_AND_no_mpl_WHEN_read_header_THEN_calls_from_key_bytes( + def test_GIVEN_decrypt_config_has_ec_WHEN_read_header_THEN_calls_decrypt_materials_with_reproduced_ec( self, mock_verifier, mock_decrypt_materials_request, From dc8abca6925ce2cd139ef53c7f3b2c8f9b3e09ce Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 28 Feb 2024 18:40:26 -0800 Subject: [PATCH 151/376] cleanup --- src/aws_encryption_sdk/streaming_client.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index bbae73bef..9488c9f08 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -954,10 +954,8 @@ def _prep_message(self): self._prep_non_framed() self._message_prepped = True - # TODO-MPL: Refactor this function, remove these linter disablers - # noqa: C901 - # pylint: disable=too-many-branches - def _read_header(self): + # TODO-MPL: Refactor this function, remove linter disablers + def _read_header(self): # noqa pylint: disable=too-many-branches """Reads the message header from the input stream. :returns: tuple containing deserialized header and header_auth objects From 8ff46f4520e510767d89365634c0e2da6a139f58 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 28 Feb 2024 18:41:15 -0800 Subject: [PATCH 152/376] cleanup --- src/aws_encryption_sdk/streaming_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 9488c9f08..ffef9cd3a 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -1006,9 +1006,9 @@ def _read_header(self): # noqa pylint: disable=too-many-branches # The items are used in header validation. if hasattr(decryption_materials, "required_encryption_context_keys"): self._required_encryption_context = {} - for (k, v) in decryption_materials.encryption_context.items(): - if k in decryption_materials.required_encryption_context_keys: - self._required_encryption_context[k] = v + for (key, value) in decryption_materials.encryption_context.items(): + if key in decryption_materials.required_encryption_context_keys: + self._required_encryption_context[key] = value else: self._required_encryption_context = None From aba7cccad05ba9ba60b49fb14a9ee3153354b66b Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 28 Feb 2024 18:42:53 -0800 Subject: [PATCH 153/376] cleanup --- src/aws_encryption_sdk/streaming_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index ffef9cd3a..fb0935ff2 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -681,7 +681,7 @@ def _write_header(self): header=self.output_buffer, data_encryption_key=self._derived_data_key, signer=self.signer, - required_encryption_context_bytes=required_ec_serialized, + required_ec_bytes=required_ec_serialized, ) # Otherwise, do not pass in any required encryption context else: From 40fecc05de1e42eca0d860ac2b04bee5670d504c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 29 Feb 2024 10:54:22 -0800 Subject: [PATCH 154/376] all message format versions --- .../internal/formatting/serialize.py | 44 ++++++++++++++----- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/src/aws_encryption_sdk/internal/formatting/serialize.py b/src/aws_encryption_sdk/internal/formatting/serialize.py index 66f4800de..310cf1436 100644 --- a/src/aws_encryption_sdk/internal/formatting/serialize.py +++ b/src/aws_encryption_sdk/internal/formatting/serialize.py @@ -189,7 +189,13 @@ def serialize_header(header, signer=None): raise SerializationError("Unrecognized message format version: {}".format(header.version)) -def _serialize_header_auth_v1(algorithm, header, data_encryption_key, signer=None): +def _serialize_header_auth_v1( + algorithm, + header, + data_encryption_key, + signer=None, + required_ec_bytes=None +): """Creates serialized header authentication data for messages in serialization version V1. :param algorithm: Algorithm to use for encryption @@ -198,16 +204,35 @@ def _serialize_header_auth_v1(algorithm, header, data_encryption_key, signer=Non :param bytes data_encryption_key: Data key with which to encrypt message :param signer: Cryptographic signer object (optional) :type signer: aws_encryption_sdk.Signer + :param required_encryption_context_bytes: Serialized encryption context items + for all items whose keys are in the required_encryption_context list. + This is ONLY processed if using the aws-cryptographic-materialproviders library + AND its required encryption context CMM. (optional) + :type required_encryption_context_bytes: bytes :returns: Serialized header authentication data :rtype: bytes """ - header_auth = encrypt( - algorithm=algorithm, - key=data_encryption_key, - plaintext=b"", - associated_data=header, - iv=header_auth_iv(algorithm), - ) + if required_ec_bytes is None: + header_auth = encrypt( + algorithm=algorithm, + key=data_encryption_key, + plaintext=b"", + associated_data=header, + iv=header_auth_iv(algorithm), + ) + else: + header_auth = encrypt( + algorithm=algorithm, + key=data_encryption_key, + plaintext=b"", + # The AAD MUST be the concatenation of the serialized message header body and the serialization + # of encryption context to only authenticate. The encryption context to only authenticate MUST + # be the encryption context in the encryption materials filtered to only contain key value + # pairs listed in the encryption material's required encryption context keys serialized + # according to the encryption context serialization specification. + associated_data=header + required_ec_bytes, + iv=header_auth_iv(algorithm), + ) output = struct.pack( ">{iv_len}s{tag_len}s".format(iv_len=algorithm.iv_len, tag_len=algorithm.tag_len), header_auth.iv, @@ -292,8 +317,7 @@ def serialize_header_auth( :param required_encryption_context_bytes: Serialized encryption context items for all items whose keys are in the required_encryption_context list. This is ONLY processed if using the aws-cryptographic-materialproviders library - AND its required encryption context CMM - AND if using the v2 message format. (optional) + AND its required encryption context CMM. (optional) :type required_encryption_context_bytes: bytes :returns: Serialized header authentication data :rtype: bytes From 52043b9fd3c5cb3ce40b222f7cfe2e3e0427a771 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 29 Feb 2024 10:55:07 -0800 Subject: [PATCH 155/376] sync upstream --- src/aws_encryption_sdk/internal/formatting/serialize.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/internal/formatting/serialize.py b/src/aws_encryption_sdk/internal/formatting/serialize.py index 310cf1436..9f1325f98 100644 --- a/src/aws_encryption_sdk/internal/formatting/serialize.py +++ b/src/aws_encryption_sdk/internal/formatting/serialize.py @@ -323,7 +323,9 @@ def serialize_header_auth( :rtype: bytes """ if version == SerializationVersion.V1: - return _serialize_header_auth_v1(algorithm, header, data_encryption_key, signer) + return _serialize_header_auth_v1( + algorithm, header, data_encryption_key, signer, required_ec_bytes + ) elif version == SerializationVersion.V2: return _serialize_header_auth_v2( algorithm, header, data_encryption_key, signer, required_ec_bytes From d256bf53fd6be9e9b8ddba61b638602f3cd60b2b Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 6 Mar 2024 11:11:36 -0800 Subject: [PATCH 156/376] working --- .../materials_managers/mpl/cmm.py | 5 + .../commands/full_message_decrypt.py | 9 +- .../commands/full_message_decrypt_generate.py | 9 +- .../manifests/full_message/decrypt.py | 72 ++++++++-- .../full_message/decrypt_generation.py | 79 +++++++++-- .../manifests/full_message/encrypt.py | 133 ++++++++++++++++++ 6 files changed, 283 insertions(+), 24 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index 8df42bf48..56c936012 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -85,6 +85,9 @@ def _native_to_mpl_get_encryption_materials( encryption_context=request.encryption_context, commitment_policy=commitment_policy, max_plaintext_length=request.plaintext_length, + algorithm_suite_id=CryptoMaterialsManagerFromMPL._native_algorithm_id_to_mpl_algorithm_id( + request.algorithm.algorithm_id + ) ) return output @@ -112,6 +115,8 @@ def decrypt_materials( try: mpl_input: 'MPL_DecryptMaterialsInput' = \ CryptoMaterialsManagerFromMPL._create_mpl_decrypt_materials_input_from_request(request) + print(f"{mpl_input=}") + print(f"{self.mpl_cmm._impl.__dict__=}") mpl_output: 'MPL_DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(mpl_input) return DecryptionMaterialsFromMPL(mpl_output.decryption_materials) except AwsCryptographicMaterialProvidersException as mpl_exception: diff --git a/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt.py b/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt.py index baf1d1f03..ad457fbd3 100644 --- a/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt.py @@ -29,9 +29,16 @@ def cli(args=None): parser.add_argument( "--input", required=True, type=argparse.FileType("r"), help="Existing full message decrypt manifest" ) + parser.add_argument( + "--keyrings", + action="store_true", + required=False, + default=False, + help="Use keyring interfaces to encrypt", + ) parsed = parser.parse_args(args) - decrypt_manifest = MessageDecryptionManifest.from_file(parsed.input) + decrypt_manifest = MessageDecryptionManifest.from_file(parsed.input, parsed.keyrings) decrypt_manifest.run() diff --git a/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt_generate.py b/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt_generate.py index 5d8b94893..9d5c9e1fa 100644 --- a/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt_generate.py +++ b/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt_generate.py @@ -39,9 +39,16 @@ def cli(args=None): dest="json_indent", help="Output human-readable JSON", ) + parser.add_argument( + "--keyrings", + action="store_true", + required=False, + default=False, + help="Use keyring interfaces to encrypt", + ) parsed = parser.parse_args(args) - encrypt_manifest = MessageDecryptionGenerationManifest.from_file(parsed.input) + encrypt_manifest = MessageDecryptionGenerationManifest.from_file(parsed.input, parsed.keyrings) encrypt_manifest.run_and_write_to_dir(target_directory=parsed.output, json_indent=parsed.json_indent) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index c94fd1452..dd09dc440 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -35,6 +35,9 @@ from awses_test_vectors.manifests.keys import KeysManifest from awses_test_vectors.manifests.master_key import MasterKeySpec, master_key_provider_from_master_key_specs +from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_provider_from_master_key_specs + + try: # Python 3.5.0 and 3.5.1 have incompatible typing modules from typing import IO, Callable, Dict, Iterable, Optional # noqa pylint: disable=unused-import @@ -202,6 +205,7 @@ class MessageDecryptionTestScenario(object): master_key_specs = attr.ib(validator=iterable_validator(list, MasterKeySpec)) master_key_provider_fn = attr.ib(validator=attr.validators.is_callable()) result = attr.ib(validator=attr.validators.instance_of(MessageDecryptionTestResult)) + keyrings = attr.ib(validator=attr.validators.instance_of(bool)) decryption_method = attr.ib( default=None, validator=attr.validators.optional(attr.validators.instance_of(DecryptionMethod)) ) @@ -216,6 +220,7 @@ def __init__( result, # type: MessageDecryptionTestResult master_key_specs, # type: Iterable[MasterKeySpec] master_key_provider_fn, # type: Callable + keyrings, # type: bool decryption_method=None, # type: Optional[DecryptionMethod] description=None, # type: Optional[str] ): # noqa=D107 @@ -231,6 +236,7 @@ def __init__( self.master_key_provider_fn = master_key_provider_fn self.decryption_method = decryption_method self.description = description + self.keyrings = keyrings attr.validate(self) @classmethod @@ -240,6 +246,8 @@ def from_scenario( plaintext_reader, # type: Callable[[str], bytes] ciphertext_reader, # type: Callable[[str], bytes] keys, # type: KeysManifest + keyrings, # type: bool + keys_uri, # type: str ): # type: (...) -> MessageDecryptionTestScenario """Load from a scenario specification. @@ -252,10 +260,18 @@ def from_scenario( :rtype: MessageDecryptionTestScenario """ raw_master_key_specs = scenario["master-keys"] # type: Iterable[MASTER_KEY_SPEC] - master_key_specs = [MasterKeySpec.from_scenario(spec) for spec in raw_master_key_specs] + if keyrings: + master_key_specs = [KeyringSpec.from_scenario(spec) for spec in raw_master_key_specs] + else: + master_key_specs = [MasterKeySpec.from_scenario(spec) for spec in raw_master_key_specs] + + print(f"{master_key_specs=}") def master_key_provider_fn(): - return master_key_provider_from_master_key_specs(keys, master_key_specs) + if keyrings: + return keyring_provider_from_master_key_specs(keys_uri, master_key_specs) + else: + return master_key_provider_from_master_key_specs(keys, master_key_specs) decryption_method_spec = scenario.get("decryption-method") decryption_method = DecryptionMethod(decryption_method_spec) if decryption_method_spec else None @@ -268,6 +284,7 @@ def master_key_provider_fn(): master_key_specs=master_key_specs, master_key_provider_fn=master_key_provider_fn, result=result, + keyrings=keyrings, decryption_method=decryption_method, description=scenario.get("description"), ) @@ -292,16 +309,27 @@ def scenario_spec(self): return spec def _one_shot_decrypt(self): + keyring = self.master_key_provider_fn() + print(f"{keyring=}") client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - return client.decrypt(source=self.ciphertext, key_provider=self.master_key_provider_fn()) + if self.keyrings: + return client.decrypt(source=self.ciphertext, keyring=keyring) + else: + return client.decrypt(source=self.ciphertext, key_provider=self.master_key_provider_fn()) def _streaming_decrypt(self): result = bytearray() client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - with client.stream(source=self.ciphertext, mode="d", key_provider=self.master_key_provider_fn()) as decryptor: - for chunk in decryptor: - result.extend(chunk) - return result, decryptor.header + if self.keyrings: + with client.stream(source=self.ciphertext, mode="d", keyring=self.master_key_provider_fn()) as decryptor: + for chunk in decryptor: + result.extend(chunk) + return result, decryptor.header + else: + with client.stream(source=self.ciphertext, mode="d", key_provider=self.master_key_provider_fn()) as decryptor: + for chunk in decryptor: + result.extend(chunk) + return result, decryptor.header def _streaming_decrypt_unsigned(self): result = bytearray() @@ -388,11 +416,12 @@ def manifest_spec(self): return {"manifest": manifest_spec, "client": client_spec, "keys": self.keys_uri, "tests": test_specs} @classmethod - def from_file(cls, input_file): + def from_file(cls, input_file, keyrings): # type: (IO) -> MessageDecryptionManifest """Load from a file containing a full message decrypt manifest. :param file input_file: File object for file containing JSON manifest + :param bool keyrings: True if should encrypt with keyring interfaces; False otherwise :return: Loaded manifest :rtype: MessageDecryptionManifest """ @@ -407,6 +436,10 @@ def from_file(cls, input_file): version = raw_manifest["manifest"]["version"] # type: int keys_uri = raw_manifest["keys"] # type: str + keys_uri = raw_manifest["keys"] + keys_filename = keys_uri.replace("file://", "") + joined = os.path.join(parent_dir, keys_filename) + raw_keys_manifest = json.loads(root_reader(keys_uri).decode(ENCODING)) keys = KeysManifest.from_manifest_spec(raw_keys_manifest) @@ -415,10 +448,31 @@ def from_file(cls, input_file): raw_scenarios = raw_manifest["tests"] # type: Dict[str, DECRYPT_SCENARIO_SPEC] test_scenarios = { name: MessageDecryptionTestScenario.from_scenario( - scenario=scenario, plaintext_reader=root_reader, ciphertext_reader=root_reader, keys=keys + scenario=scenario, + plaintext_reader=root_reader, + ciphertext_reader=root_reader, + keys=keys, + keyrings=False, + keys_uri=joined, ) for name, scenario in raw_scenarios.items() } + # If optional keyrings argument is true, + # also add scenarios to decrypt with keyrings. + if keyrings: + keyrings_test_scenarios = { + name + "-keyring": MessageDecryptionTestScenario.from_scenario( + scenario=scenario, + plaintext_reader=root_reader, + ciphertext_reader=root_reader, + keys=keys, + keyrings=True, + keys_uri=joined, + ) + for name, scenario in raw_scenarios.items() + } + # Merge into test_scenarios + test_scenarios = {**keyrings_test_scenarios, **test_scenarios} return cls( keys_uri=keys_uri, diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index e407a1b65..040ec07d8 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -23,10 +23,25 @@ import attr import six from aws_encryption_sdk.caches.local import LocalCryptoMaterialsCache +from aws_encryption_sdk.key_providers.base import MasterKeyProvider from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager from aws_encryption_sdk.materials_managers.caching import CachingCryptoMaterialsManager from aws_encryption_sdk.materials_managers.default import DefaultCryptoMaterialsManager +from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders +from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig +from aws_cryptographic_materialproviders.mpl.references import ( + IKeyring, + CryptographicMaterialsManager, +) +from aws_cryptographic_materialproviders.mpl.models import ( + CreateDefaultCryptographicMaterialsManagerInput, +) +from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL + +from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_provider_from_master_key_specs + + from awses_test_vectors.internal.defaults import ENCODING from awses_test_vectors.internal.util import ( dictionary_validator, @@ -92,9 +107,23 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, pl return: a list of (ciphertext, result) pairs """ - materials_manager = DefaultCryptoMaterialsManager( - generation_scenario.encryption_scenario.master_key_provider_fn() - ) + tmp = generation_scenario.encryption_scenario.master_key_provider_fn() + if isinstance(tmp, MasterKeyProvider): + materials_manager = DefaultCryptoMaterialsManager( + tmp + ) + elif isinstance(tmp, IKeyring): + mpl = AwsCryptographicMaterialProviders(MaterialProvidersConfig()) + mpl_cmm = mpl.create_default_cryptographic_materials_manager( + CreateDefaultCryptographicMaterialsManagerInput( + keyring=tmp + ) + ) + materials_manager = CryptoMaterialsManagerFromMPL( + mpl_cmm=mpl_cmm + ) + else: + raise ValueError(f"Unrecognized master_key_provider_fn return type: {str(tmp)}") ciphertext_to_decrypt = generation_scenario.encryption_scenario.run(materials_manager) if generation_scenario.result: expected_result = generation_scenario.result @@ -334,30 +363,40 @@ class MessageDecryptionTestScenarioGenerator(object): decryption_master_key_specs = attr.ib(validator=iterable_validator(list, MasterKeySpec)) decryption_master_key_provider_fn = attr.ib(validator=attr.validators.is_callable()) result = attr.ib(validator=attr.validators.optional(attr.validators.instance_of(MessageDecryptionTestResult))) + keyrings = attr.ib(validator=attr.validators.instance_of(bool)) @classmethod - def from_scenario(cls, scenario, keys, plaintexts): + def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): """Load from a scenario specification. :param dict scenario: Scenario specification JSON :param KeysManifest keys: Loaded keys :param dict plaintexts: Mapping of plaintext names to plaintext values + :param bool keyrings: True if should encrypt with keyring interfaces; False otherwise :return: Loaded test scenario :rtype: MessageDecryptionTestScenarioGenerator """ encryption_scenario_spec = scenario["encryption-scenario"] - encryption_scenario = MessageEncryptionTestScenario.from_scenario(encryption_scenario_spec, keys, plaintexts) + encryption_scenario = MessageEncryptionTestScenario.from_scenario(encryption_scenario_spec, keys, plaintexts, keyrings, keys_uri) tampering = scenario.get("tampering") tampering_method = TamperingMethod.from_tampering_spec(tampering) decryption_method_spec = scenario.get("decryption-method") decryption_method = DecryptionMethod(decryption_method_spec) if decryption_method_spec else None if "decryption-master-keys" in scenario: - decryption_master_key_specs = [ - MasterKeySpec.from_scenario(spec) for spec in scenario["decryption-master-keys"] - ] + if keyrings: + decryption_master_key_specs = [ + KeyringSpec.from_scenario(spec) for spec in scenario["decryption-master-keys"] + ] + else: + decryption_master_key_specs = [ + MasterKeySpec.from_scenario(spec) for spec in scenario["decryption-master-keys"] + ] def decryption_master_key_provider_fn(): - return master_key_provider_from_master_key_specs(keys, decryption_master_key_specs) + if keyrings: + return keyring_provider_from_master_key_specs(keys_uri, decryption_master_key_specs) + else: + return master_key_provider_from_master_key_specs(keys, decryption_master_key_specs) else: decryption_master_key_specs = encryption_scenario.master_key_specs @@ -372,6 +411,7 @@ def decryption_master_key_provider_fn(): decryption_master_key_specs=decryption_master_key_specs, decryption_master_key_provider_fn=decryption_master_key_provider_fn, result=result, + keyrings=keyrings, ) def run(self, ciphertext_writer, plaintext_uri): @@ -400,6 +440,7 @@ def decryption_test_scenario_pair(self, ciphertext_writer, ciphertext_to_decrypt master_key_provider_fn=self.decryption_master_key_provider_fn, decryption_method=self.decryption_method, result=expected_result, + keyrings=self.keyrings, ), ) @@ -414,12 +455,14 @@ class MessageDecryptionGenerationManifest(object): :param KeysManifest keys: Loaded keys :param dict plaintexts: Mapping of plaintext names to plaintext values :param dict tests: Mapping of test scenario names to :class:`MessageDecryptionGenerationManifest`s + :param bool keyrings: True if should encrypt with keyring interfaces; False otherwise """ version = attr.ib(validator=membership_validator(SUPPORTED_VERSIONS)) keys = attr.ib(validator=attr.validators.instance_of(KeysManifest)) plaintexts = attr.ib(validator=dictionary_validator(six.string_types, six.binary_type)) tests = attr.ib(validator=dictionary_validator(six.string_types, MessageDecryptionTestScenarioGenerator)) + keyrings = attr.ib(validator=attr.validators.instance_of(bool)) type_name = "awses-decrypt-generate" @staticmethod @@ -434,11 +477,12 @@ def _generate_plaintexts(plaintexts_specs): return {name: os.urandom(size) for name, size in plaintexts_specs.items()} @classmethod - def from_file(cls, input_file): + def from_file(cls, input_file, keyrings): # type: (IO) -> MessageDecryptionGenerationManifest """Load from a file containing a full message encrypt manifest. :param file input_file: File object for file containing JSON manifest + :param bool keyrings: True if should encrypt with keyring interfaces; False otherwise :return: Loaded manifest :rtype: MessageEncryptionManifest """ @@ -449,18 +493,27 @@ def from_file(cls, input_file): parent_dir = os.path.abspath(os.path.dirname(input_file.name)) reader = file_reader(parent_dir) - raw_keys_manifest = json.loads(reader(raw_manifest["keys"]).decode(ENCODING)) + keys_uri = raw_manifest["keys"] + keys_filename = keys_uri.replace("file://", "") + print(f"{parent_dir=}") + print(f"{input_file=}") + print(f"{keys_uri=}") + print(f"{parent_dir+keys_uri=}") + print(f"{os.path.join(parent_dir, keys_uri)=}") + joined = os.path.join(parent_dir, keys_filename) + raw_keys_manifest = json.loads(reader(keys_uri).decode(ENCODING)) keys = KeysManifest.from_manifest_spec(raw_keys_manifest) plaintexts = cls._generate_plaintexts(raw_manifest["plaintexts"]) tests = {} + # For some bizarre reason, the for name, scenario in raw_manifest["tests"].items(): try: tests[name] = MessageDecryptionTestScenarioGenerator.from_scenario( - scenario=scenario, keys=keys, plaintexts=plaintexts + scenario=scenario, keys=keys, plaintexts=plaintexts, keyrings=keyrings, keys_uri=joined, ) except NotImplementedError: continue - return cls(version=raw_manifest["manifest"]["version"], keys=keys, plaintexts=plaintexts, tests=tests) + return cls(version=raw_manifest["manifest"]["version"], keys=keys, plaintexts=plaintexts, tests=tests, keyrings=keyrings) def run_and_write_to_dir(self, target_directory, json_indent=None): # type: (str, Optional[int]) -> None diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index c77fed1ce..b22071b95 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -39,6 +39,13 @@ except ImportError: from aws_encryption_sdk.identifiers import Algorithm as AlgorithmSuite +try: + import aws_cryptographic_materialproviders +except ImportError as e: + print("IMPORT OOPS") + print(e) + +from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_provider_from_master_key_specs try: # Python 3.5.0 and 3.5.1 have incompatible typing modules from typing import IO, Callable, Dict, Iterable, Optional # noqa pylint: disable=unused-import @@ -76,6 +83,54 @@ class MessageEncryptionTestScenario(object): algorithm = attr.ib(validator=attr.validators.instance_of(AlgorithmSuite)) frame_size = attr.ib(validator=attr.validators.instance_of(int)) encryption_context = attr.ib(validator=dictionary_validator(six.string_types, six.string_types)) + master_key = attr.ib(validator=attr.validators.instance_of(bool)) + + @classmethod + def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): + # type: (ENCRYPT_SCENARIO_SPEC, KeysManifest, Dict[str, bytes], bool) -> MessageEncryptionTestScenario + """Load from a scenario specification. + + :param dict scenario: Scenario specification JSON + :param KeysManifest keys: Loaded keys + :param dict plaintexts: Mapping of plaintext names to plaintext values + :param bool keyrings: True if should encrypt with master key interfaces; False otherwise + :return: Loaded test scenario + :rtype: MessageEncryptionTestScenario + """ + + if keyrings: + print("KEYRINGS") + return MessageEncryptionWithKeyringsTestScenario.from_scenario( + scenario, keys_uri, plaintexts + ) + else: + return MessageEncryptionWithMasterKeysTestScenario.from_scenario( + scenario, keys, plaintexts + ) + + def run(self, materials_manager=None): + """Run this scenario, writing the resulting ciphertext with ``ciphertext_writer`` and returning + a :class:`MessageDecryptionTestScenario` that describes the matching decrypt scenario. + + :param callable ciphertext_writer: Callable that will write the requested named ciphertext and + return a URI locating the written data + :param str plaintext_uri: URI locating the written plaintext data for this scenario + :return: Decrypt test scenario that describes the generated scenario + :rtype: MessageDecryptionTestScenario + """ + raise NotImplementedError("MUST specify keyrings bool") + + +@attr.s +class MessageEncryptionWithMasterKeysTestScenario(MessageEncryptionTestScenario): + # pylint: disable=too-many-instance-attributes + """Data class for a single full message decrypt test scenario that uses master keys. + + :param master_key_specs: Iterable of loaded master key specifications + :type master_key_specs: iterable of :class:`MasterKeySpec` + :param Callable master_key_provider_fn: + """ + master_key_specs = attr.ib(validator=iterable_validator(list, MasterKeySpec)) master_key_provider_fn = attr.ib(validator=attr.validators.is_callable()) @@ -102,6 +157,7 @@ def master_key_provider_fn(): algorithm=algorithm, frame_size=scenario["frame-size"], encryption_context=scenario["encryption-context"], + master_key=True, master_key_specs=master_key_specs, master_key_provider_fn=master_key_provider_fn, ) @@ -134,6 +190,83 @@ def run(self, materials_manager=None): ciphertext, _header = client.encrypt(**encrypt_kwargs) return ciphertext +@attr.s +class MessageEncryptionWithKeyringsTestScenario(MessageEncryptionTestScenario): + # pylint: disable=too-many-instance-attributes + """Data class for a single full message decrypt test scenario that uses keyrings. + + :param master_key_specs: Iterable of loaded master key specifications + :type master_key_specs: iterable of :class:`MasterKeySpec` + :param Callable master_key_provider_fn: + """ + + master_key_specs = attr.ib(validator=iterable_validator(list, MasterKeySpec)) + master_key_provider_fn = attr.ib(validator=attr.validators.is_callable()) + + @classmethod + def from_scenario(cls, scenario, keys_uri, plaintexts): + print("FROM_SCENARIO") + print(f"{len(scenario['master-keys'])=}") + # type: (ENCRYPT_SCENARIO_SPEC, KeysManifest, Dict[str, bytes]) -> MessageEncryptionTestScenario + """Load from a scenario specification. + + :param dict scenario: Scenario specification JSON + :param KeysManifest keys: Loaded keys + :param dict plaintexts: Mapping of plaintext names to plaintext values + :return: Loaded test scenario + :rtype: MessageEncryptionTestScenario + """ + print("1") + algorithm = algorithm_suite_from_string_id(scenario["algorithm"]) + print("2") + # manifest still keys these as `master-keys` even though these are keyrings + try: + master_key_specs = [KeyringSpec.from_scenario(spec) for spec in scenario["master-keys"]] + except Exception as e: + print(e) + + def keyring_provider_fn(): + return keyring_provider_from_master_key_specs(keys_uri, master_key_specs) + + return cls( + plaintext_name=scenario["plaintext"], + plaintext=plaintexts[scenario["plaintext"]], + algorithm=algorithm, + frame_size=scenario["frame-size"], + encryption_context=scenario["encryption-context"], + master_key=True, + master_key_specs=master_key_specs, + master_key_provider_fn=keyring_provider_fn, + ) + + def run(self, materials_manager=None): + """Run this scenario, writing the resulting ciphertext with ``ciphertext_writer`` and returning + a :class:`MessageDecryptionTestScenario` that describes the matching decrypt scenario. + + :param callable ciphertext_writer: Callable that will write the requested named ciphertext and + return a URI locating the written data + :param str plaintext_uri: URI locating the written plaintext data for this scenario + :return: Decrypt test scenario that describes the generated scenario + :rtype: MessageDecryptionTestScenario + """ + commitment_policy = CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT + if self.algorithm.is_committing(): + commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT + + client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=commitment_policy) + print(f"{self.algorithm=}") + encrypt_kwargs = dict( + source=self.plaintext, + algorithm=self.algorithm, + frame_length=self.frame_size, + encryption_context=self.encryption_context, + ) + if materials_manager: + encrypt_kwargs["materials_manager"] = materials_manager + else: + encrypt_kwargs["keyring"] = self.keyring_provider_fn() + ciphertext, _header = client.encrypt(**encrypt_kwargs) + return ciphertext @attr.s class MessageEncryptionManifest(object): From 488bdda1180c25cafa674fe5888be1bda32a7d2b Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 6 Mar 2024 15:57:26 -0800 Subject: [PATCH 157/376] cleanup --- .../materials_managers/mpl/cmm.py | 2 - .../commands/full_message_decrypt.py | 10 +++ .../commands/full_message_decrypt_generate.py | 9 ++ .../commands/full_message_encrypt.py | 10 +++ .../manifests/full_message/decrypt.py | 57 +++++++------ .../full_message/decrypt_generation.py | 84 +++++++++++-------- .../manifests/full_message/encrypt.py | 11 +-- 7 files changed, 117 insertions(+), 66 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index 56c936012..3749dde97 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -115,8 +115,6 @@ def decrypt_materials( try: mpl_input: 'MPL_DecryptMaterialsInput' = \ CryptoMaterialsManagerFromMPL._create_mpl_decrypt_materials_input_from_request(request) - print(f"{mpl_input=}") - print(f"{self.mpl_cmm._impl.__dict__=}") mpl_output: 'MPL_DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(mpl_input) return DecryptionMaterialsFromMPL(mpl_output.decryption_materials) except AwsCryptographicMaterialProvidersException as mpl_exception: diff --git a/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt.py b/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt.py index ad457fbd3..f28354f31 100644 --- a/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt.py @@ -15,6 +15,13 @@ from awses_test_vectors.manifests.full_message.decrypt import MessageDecryptionManifest +try: + import aws_cryptographic_materialproviders # noqa pylint: disable=unused-import + _HAS_MPL = True +except Exception as e: + _HAS_MPL = False + + try: # Python 3.5.0 and 3.5.1 have incompatible typing modules from typing import Iterable, Optional # noqa pylint: disable=unused-import except ImportError: # pragma: no cover @@ -39,6 +46,9 @@ def cli(args=None): parsed = parser.parse_args(args) + if parsed.keyrings and not _HAS_MPL: + raise ImportError("The --keyrings flag requires the aws-cryptographic-material-providers library.") + decrypt_manifest = MessageDecryptionManifest.from_file(parsed.input, parsed.keyrings) decrypt_manifest.run() diff --git a/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt_generate.py b/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt_generate.py index 9d5c9e1fa..ae6afa538 100644 --- a/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt_generate.py +++ b/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt_generate.py @@ -15,6 +15,12 @@ from awses_test_vectors.manifests.full_message.decrypt_generation import MessageDecryptionGenerationManifest +try: + import aws_cryptographic_materialproviders # noqa pylint: disable=unused-import + _HAS_MPL = True +except Exception as e: + _HAS_MPL = False + try: # Python 3.5.0 and 3.5.1 have incompatible typing modules from typing import Iterable, Optional # noqa pylint: disable=unused-import except ImportError: # pragma: no cover @@ -49,6 +55,9 @@ def cli(args=None): parsed = parser.parse_args(args) + if parsed.keyrings and not _HAS_MPL: + raise ImportError("The --keyrings flag requires the aws-cryptographic-material-providers library.") + encrypt_manifest = MessageDecryptionGenerationManifest.from_file(parsed.input, parsed.keyrings) encrypt_manifest.run_and_write_to_dir(target_directory=parsed.output, json_indent=parsed.json_indent) diff --git a/test_vector_handlers/src/awses_test_vectors/commands/full_message_encrypt.py b/test_vector_handlers/src/awses_test_vectors/commands/full_message_encrypt.py index 2b8b92f3c..6bea002dc 100644 --- a/test_vector_handlers/src/awses_test_vectors/commands/full_message_encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/commands/full_message_encrypt.py @@ -15,6 +15,13 @@ from awses_test_vectors.manifests.full_message.encrypt import MessageEncryptionManifest +try: + import aws_cryptographic_materialproviders # noqa pylint: disable=unused-import + _HAS_MPL = True +except Exception as e: + _HAS_MPL = False + + try: # Python 3.5.0 and 3.5.1 have incompatible typing modules from typing import Iterable, Optional # noqa pylint: disable=unused-import except ImportError: # pragma: no cover @@ -32,6 +39,9 @@ def cli(args=None): parsed = parser.parse_args(args) + if parsed.keyrings and not _HAS_MPL: + raise ImportError("The --keyrings flag requires the aws-cryptographic-material-providers library.") + encrypt_manifest = MessageEncryptionManifest.from_file(parsed.input) encrypt_manifest.run() diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index dd09dc440..797eadf67 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -35,7 +35,13 @@ from awses_test_vectors.manifests.keys import KeysManifest from awses_test_vectors.manifests.master_key import MasterKeySpec, master_key_provider_from_master_key_specs -from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_provider_from_master_key_specs +try: + from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_from_master_key_specs + + _HAS_MPL = True + +except ImportError as e: + _HAS_MPL = False try: # Python 3.5.0 and 3.5.1 have incompatible typing modules @@ -195,6 +201,7 @@ class MessageDecryptionTestScenario(object): :param master_key_specs: Iterable of master key specifications :type master_key_specs: iterable of :class:`MasterKeySpec` :param Callable master_key_provider_fn: + :param bool keyrings: True if should decrypt with keyring interfaces; False otherwise :param str description: Description of test scenario (optional) """ @@ -260,16 +267,15 @@ def from_scenario( :rtype: MessageDecryptionTestScenario """ raw_master_key_specs = scenario["master-keys"] # type: Iterable[MASTER_KEY_SPEC] - if keyrings: - master_key_specs = [KeyringSpec.from_scenario(spec) for spec in raw_master_key_specs] - else: - master_key_specs = [MasterKeySpec.from_scenario(spec) for spec in raw_master_key_specs] - - print(f"{master_key_specs=}") + master_key_specs = [MasterKeySpec.from_scenario(spec) for spec in raw_master_key_specs] + # if keyrings: + # master_key_specs = [KeyringSpec.from_scenario(spec) for spec in raw_master_key_specs] + # else: + # master_key_specs = [MasterKeySpec.from_scenario(spec) for spec in raw_master_key_specs] def master_key_provider_fn(): if keyrings: - return keyring_provider_from_master_key_specs(keys_uri, master_key_specs) + return keyring_from_master_key_specs(keys_uri, master_key_specs) else: return master_key_provider_from_master_key_specs(keys, master_key_specs) @@ -309,28 +315,30 @@ def scenario_spec(self): return spec def _one_shot_decrypt(self): - keyring = self.master_key_provider_fn() - print(f"{keyring=}") client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) if self.keyrings: - return client.decrypt(source=self.ciphertext, keyring=keyring) + return client.decrypt(source=self.ciphertext, keyring=self.master_key_provider_fn()) else: return client.decrypt(source=self.ciphertext, key_provider=self.master_key_provider_fn()) def _streaming_decrypt(self): result = bytearray() client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + + kwargs = { + "source": self.ciphertext, + "mode": "d" + } if self.keyrings: - with client.stream(source=self.ciphertext, mode="d", keyring=self.master_key_provider_fn()) as decryptor: - for chunk in decryptor: - result.extend(chunk) - return result, decryptor.header + kwargs["keyring"] = self.master_key_provider_fn() else: - with client.stream(source=self.ciphertext, mode="d", key_provider=self.master_key_provider_fn()) as decryptor: - for chunk in decryptor: - result.extend(chunk) - return result, decryptor.header + kwargs["key_provider"] = self.master_key_provider_fn() + with client.stream(**kwargs) as decryptor: + for chunk in decryptor: + result.extend(chunk) + return result, decryptor.header + def _streaming_decrypt_unsigned(self): result = bytearray() client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) @@ -421,7 +429,7 @@ def from_file(cls, input_file, keyrings): """Load from a file containing a full message decrypt manifest. :param file input_file: File object for file containing JSON manifest - :param bool keyrings: True if should encrypt with keyring interfaces; False otherwise + :param bool keyrings: True if should decrypt with keyring interfaces; False otherwise :return: Loaded manifest :rtype: MessageDecryptionManifest """ @@ -436,9 +444,10 @@ def from_file(cls, input_file, keyrings): version = raw_manifest["manifest"]["version"] # type: int keys_uri = raw_manifest["keys"] # type: str + # MPL TestVector keyring needs to know the path to the keys file keys_uri = raw_manifest["keys"] keys_filename = keys_uri.replace("file://", "") - joined = os.path.join(parent_dir, keys_filename) + keys_abs_path = os.path.join(parent_dir, keys_filename) raw_keys_manifest = json.loads(root_reader(keys_uri).decode(ENCODING)) keys = KeysManifest.from_manifest_spec(raw_keys_manifest) @@ -453,7 +462,7 @@ def from_file(cls, input_file, keyrings): ciphertext_reader=root_reader, keys=keys, keyrings=False, - keys_uri=joined, + keys_uri=keys_abs_path, ) for name, scenario in raw_scenarios.items() } @@ -467,11 +476,11 @@ def from_file(cls, input_file, keyrings): ciphertext_reader=root_reader, keys=keys, keyrings=True, - keys_uri=joined, + keys_uri=keys_abs_path, ) for name, scenario in raw_scenarios.items() } - # Merge into test_scenarios + # Merge keyring scenarios into test_scenarios test_scenarios = {**keyrings_test_scenarios, **test_scenarios} return cls( diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 040ec07d8..61a62dd22 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -28,18 +28,23 @@ from aws_encryption_sdk.materials_managers.caching import CachingCryptoMaterialsManager from aws_encryption_sdk.materials_managers.default import DefaultCryptoMaterialsManager -from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders -from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig -from aws_cryptographic_materialproviders.mpl.references import ( - IKeyring, - CryptographicMaterialsManager, -) -from aws_cryptographic_materialproviders.mpl.models import ( - CreateDefaultCryptographicMaterialsManagerInput, -) -from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL +try: + from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders + from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig + from aws_cryptographic_materialproviders.mpl.references import ( + IKeyring, + CryptographicMaterialsManager, + ) + from aws_cryptographic_materialproviders.mpl.models import ( + CreateDefaultCryptographicMaterialsManagerInput, + ) + from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL -from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_provider_from_master_key_specs + from awses_test_vectors.manifests.mpl_keyring import keyring_from_master_key_specs + + _HAS_MPL = True +except ImportError as e: + _HAS_MPL = False from awses_test_vectors.internal.defaults import ENCODING @@ -107,23 +112,23 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, pl return: a list of (ciphertext, result) pairs """ - tmp = generation_scenario.encryption_scenario.master_key_provider_fn() - if isinstance(tmp, MasterKeyProvider): + key_provider = generation_scenario.encryption_scenario.master_key_provider_fn() + if isinstance(key_provider, MasterKeyProvider): materials_manager = DefaultCryptoMaterialsManager( - tmp + key_provider ) - elif isinstance(tmp, IKeyring): + elif isinstance(key_provider, IKeyring): mpl = AwsCryptographicMaterialProviders(MaterialProvidersConfig()) mpl_cmm = mpl.create_default_cryptographic_materials_manager( CreateDefaultCryptographicMaterialsManagerInput( - keyring=tmp + keyring=key_provider ) ) materials_manager = CryptoMaterialsManagerFromMPL( mpl_cmm=mpl_cmm ) else: - raise ValueError(f"Unrecognized master_key_provider_fn return type: {str(tmp)}") + raise ValueError(f"Unrecognized master_key_provider_fn return type: {str(key_provider)}") ciphertext_to_decrypt = generation_scenario.encryption_scenario.run(materials_manager) if generation_scenario.result: expected_result = generation_scenario.result @@ -355,6 +360,7 @@ class MessageDecryptionTestScenarioGenerator(object): :type decryption_master_key_specs: iterable of :class:`MasterKeySpec` :param Callable decryption_master_key_provider_fn: :param result: + :param bool keyrings: True if should encrypt with keyring interfaces; False otherwise """ encryption_scenario = attr.ib(validator=attr.validators.instance_of(MessageEncryptionTestScenario)) @@ -373,28 +379,39 @@ def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): :param KeysManifest keys: Loaded keys :param dict plaintexts: Mapping of plaintext names to plaintext values :param bool keyrings: True if should encrypt with keyring interfaces; False otherwise + :param string keys_uri: Filepath to keys manifest. Used by MPL TestVector keyring constructor. :return: Loaded test scenario :rtype: MessageDecryptionTestScenarioGenerator """ encryption_scenario_spec = scenario["encryption-scenario"] - encryption_scenario = MessageEncryptionTestScenario.from_scenario(encryption_scenario_spec, keys, plaintexts, keyrings, keys_uri) + encryption_scenario = MessageEncryptionTestScenario.from_scenario( + encryption_scenario_spec, + keys, + plaintexts, + keyrings, + keys_uri, + ) tampering = scenario.get("tampering") tampering_method = TamperingMethod.from_tampering_spec(tampering) decryption_method_spec = scenario.get("decryption-method") decryption_method = DecryptionMethod(decryption_method_spec) if decryption_method_spec else None if "decryption-master-keys" in scenario: - if keyrings: - decryption_master_key_specs = [ - KeyringSpec.from_scenario(spec) for spec in scenario["decryption-master-keys"] - ] - else: - decryption_master_key_specs = [ - MasterKeySpec.from_scenario(spec) for spec in scenario["decryption-master-keys"] - ] + decryption_master_key_specs = [ + MasterKeySpec.from_scenario(spec) for spec in scenario["decryption-master-keys"] + ] + + # if keyrings: + # decryption_master_key_specs = [ + # KeyringSpec.from_scenario(spec) for spec in scenario["decryption-master-keys"] + # ] + # else: + # decryption_master_key_specs = [ + # MasterKeySpec.from_scenario(spec) for spec in scenario["decryption-master-keys"] + # ] def decryption_master_key_provider_fn(): if keyrings: - return keyring_provider_from_master_key_specs(keys_uri, decryption_master_key_specs) + return keyring_from_master_key_specs(keys_uri, decryption_master_key_specs) else: return master_key_provider_from_master_key_specs(keys, decryption_master_key_specs) @@ -493,23 +510,20 @@ def from_file(cls, input_file, keyrings): parent_dir = os.path.abspath(os.path.dirname(input_file.name)) reader = file_reader(parent_dir) + + # MPL TestVector keyring needs to know the path to the keys file keys_uri = raw_manifest["keys"] keys_filename = keys_uri.replace("file://", "") - print(f"{parent_dir=}") - print(f"{input_file=}") - print(f"{keys_uri=}") - print(f"{parent_dir+keys_uri=}") - print(f"{os.path.join(parent_dir, keys_uri)=}") - joined = os.path.join(parent_dir, keys_filename) + keys_abs_path = os.path.join(parent_dir, keys_filename) + raw_keys_manifest = json.loads(reader(keys_uri).decode(ENCODING)) keys = KeysManifest.from_manifest_spec(raw_keys_manifest) plaintexts = cls._generate_plaintexts(raw_manifest["plaintexts"]) tests = {} - # For some bizarre reason, the for name, scenario in raw_manifest["tests"].items(): try: tests[name] = MessageDecryptionTestScenarioGenerator.from_scenario( - scenario=scenario, keys=keys, plaintexts=plaintexts, keyrings=keyrings, keys_uri=joined, + scenario=scenario, keys=keys, plaintexts=plaintexts, keyrings=keyrings, keys_uri=keys_abs_path, ) except NotImplementedError: continue diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index b22071b95..4e0edf0ca 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -40,12 +40,13 @@ from aws_encryption_sdk.identifiers import Algorithm as AlgorithmSuite try: - import aws_cryptographic_materialproviders + from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_from_master_key_specs + + _HAS_MPL = True + except ImportError as e: - print("IMPORT OOPS") - print(e) + _HAS_MPL = False -from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_provider_from_master_key_specs try: # Python 3.5.0 and 3.5.1 have incompatible typing modules from typing import IO, Callable, Dict, Iterable, Optional # noqa pylint: disable=unused-import @@ -226,7 +227,7 @@ def from_scenario(cls, scenario, keys_uri, plaintexts): print(e) def keyring_provider_fn(): - return keyring_provider_from_master_key_specs(keys_uri, master_key_specs) + return keyring_from_master_key_specs(keys_uri, master_key_specs) return cls( plaintext_name=scenario["plaintext"], From cea9dab5cd32875aa21e8cd0e4a5bf0f34e7de34 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 6 Mar 2024 16:16:07 -0800 Subject: [PATCH 158/376] cleanup: --- .../manifests/full_message/decrypt.py | 4 ++-- .../manifests/full_message/encrypt.py | 11 +---------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index 797eadf67..6336400ce 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -36,7 +36,7 @@ from awses_test_vectors.manifests.master_key import MasterKeySpec, master_key_provider_from_master_key_specs try: - from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_from_master_key_specs + from awses_test_vectors.manifests.mpl_keyring import keyring_from_master_key_specs _HAS_MPL = True @@ -338,7 +338,7 @@ def _streaming_decrypt(self): for chunk in decryptor: result.extend(chunk) return result, decryptor.header - + def _streaming_decrypt_unsigned(self): result = bytearray() client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 4e0edf0ca..66c729056 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -100,7 +100,6 @@ def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): """ if keyrings: - print("KEYRINGS") return MessageEncryptionWithKeyringsTestScenario.from_scenario( scenario, keys_uri, plaintexts ) @@ -206,8 +205,6 @@ class MessageEncryptionWithKeyringsTestScenario(MessageEncryptionTestScenario): @classmethod def from_scenario(cls, scenario, keys_uri, plaintexts): - print("FROM_SCENARIO") - print(f"{len(scenario['master-keys'])=}") # type: (ENCRYPT_SCENARIO_SPEC, KeysManifest, Dict[str, bytes]) -> MessageEncryptionTestScenario """Load from a scenario specification. @@ -217,14 +214,9 @@ def from_scenario(cls, scenario, keys_uri, plaintexts): :return: Loaded test scenario :rtype: MessageEncryptionTestScenario """ - print("1") algorithm = algorithm_suite_from_string_id(scenario["algorithm"]) - print("2") # manifest still keys these as `master-keys` even though these are keyrings - try: - master_key_specs = [KeyringSpec.from_scenario(spec) for spec in scenario["master-keys"]] - except Exception as e: - print(e) + master_key_specs = [KeyringSpec.from_scenario(spec) for spec in scenario["master-keys"]] def keyring_provider_fn(): return keyring_from_master_key_specs(keys_uri, master_key_specs) @@ -255,7 +247,6 @@ def run(self, materials_manager=None): commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=commitment_policy) - print(f"{self.algorithm=}") encrypt_kwargs = dict( source=self.plaintext, algorithm=self.algorithm, From 98fd0af3479678bde4457ea6a4d713c52cc4097d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 09:06:06 -0800 Subject: [PATCH 159/376] fix testvector run --- .../awses_test_vectors/commands/full_message_encrypt.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test_vector_handlers/src/awses_test_vectors/commands/full_message_encrypt.py b/test_vector_handlers/src/awses_test_vectors/commands/full_message_encrypt.py index 6bea002dc..1e0484276 100644 --- a/test_vector_handlers/src/awses_test_vectors/commands/full_message_encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/commands/full_message_encrypt.py @@ -36,6 +36,13 @@ def cli(args=None): parser.add_argument( "--input", required=True, type=argparse.FileType("r"), help="Existing full message encrypt manifest" ) + parser.add_argument( + "--keyrings", + action="store_true", + required=False, + default=False, + help="Use keyring interfaces to encrypt", + ) parsed = parser.parse_args(args) From b4ba23c5b7912a7da74d40da1887ba668f4168a0 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 09:12:14 -0800 Subject: [PATCH 160/376] rm module_ --- examples/src/keyrings/module_.py | 1 - examples/src/module_.py | 1 - 2 files changed, 2 deletions(-) delete mode 100644 examples/src/keyrings/module_.py delete mode 100644 examples/src/module_.py diff --git a/examples/src/keyrings/module_.py b/examples/src/keyrings/module_.py deleted file mode 100644 index 3e8d3062a..000000000 --- a/examples/src/keyrings/module_.py +++ /dev/null @@ -1 +0,0 @@ -"""Should remove this once PYTHONPATH issues are resolved by adding doo files.""" diff --git a/examples/src/module_.py b/examples/src/module_.py deleted file mode 100644 index 3e8d3062a..000000000 --- a/examples/src/module_.py +++ /dev/null @@ -1 +0,0 @@ -"""Should remove this once PYTHONPATH issues are resolved by adding doo files.""" From 3e6dfa1b8f578b9b4fb2e73e73d2b1880dab2a0c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 09:28:47 -0800 Subject: [PATCH 161/376] kwargify input --- .../materials_managers/mpl/cmm.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index 3749dde97..c454a114e 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -81,13 +81,19 @@ def _native_to_mpl_get_encryption_materials( commitment_policy = CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy( request.commitment_policy ) + mpl_input_kwargs = { + "encryption_context": request.encryption_context, + "commitment_policy": commitment_policy, + "max_plaintext_length": request.plaintext_length, + } + if request.algorithm is not None: + mpl_input_kwargs["algorithm_suite_id"] = \ + CryptoMaterialsManagerFromMPL._native_algorithm_id_to_mpl_algorithm_id( + request.algorithm.algorithm_id + ) + output: MPL_GetEncryptionMaterialsInput = MPL_GetEncryptionMaterialsInput( - encryption_context=request.encryption_context, - commitment_policy=commitment_policy, - max_plaintext_length=request.plaintext_length, - algorithm_suite_id=CryptoMaterialsManagerFromMPL._native_algorithm_id_to_mpl_algorithm_id( - request.algorithm.algorithm_id - ) + **mpl_input_kwargs ) return output From bcc689ce8723158e41de963d71c9e4eb1772b425 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 09:40:44 -0800 Subject: [PATCH 162/376] fix testvector --- .../commands/full_message_encrypt.py | 2 +- .../manifests/full_message/encrypt.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/commands/full_message_encrypt.py b/test_vector_handlers/src/awses_test_vectors/commands/full_message_encrypt.py index 1e0484276..5294e1791 100644 --- a/test_vector_handlers/src/awses_test_vectors/commands/full_message_encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/commands/full_message_encrypt.py @@ -49,6 +49,6 @@ def cli(args=None): if parsed.keyrings and not _HAS_MPL: raise ImportError("The --keyrings flag requires the aws-cryptographic-material-providers library.") - encrypt_manifest = MessageEncryptionManifest.from_file(parsed.input) + encrypt_manifest = MessageEncryptionManifest.from_file(parsed.input, parsed.keyrings) encrypt_manifest.run() diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 66c729056..eabbe7343 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -290,11 +290,12 @@ def _generate_plaintexts(plaintexts_specs): return {name: os.urandom(size) for name, size in plaintexts_specs.items()} @classmethod - def from_file(cls, input_file): + def from_file(cls, input_file, keyrings): # type: (IO) -> MessageEncryptionManifest """Load frome a file containing a full message encrypt manifest. :param file input_file: File object for file containing JSON manifest + :param bool keyrings: True if should encrypt with keyring interfaces; False otherwise :return: Loaded manifest :rtype: MessageEncryptionManifest """ @@ -305,14 +306,20 @@ def from_file(cls, input_file): parent_dir = os.path.abspath(os.path.dirname(input_file.name)) reader = file_reader(parent_dir) - raw_keys_manifest = json.loads(reader(raw_manifest["keys"]).decode(ENCODING)) + + # MPL TestVector keyring needs to know the path to the keys file + keys_uri = raw_manifest["keys"] + keys_filename = keys_uri.replace("file://", "") + keys_abs_path = os.path.join(parent_dir, keys_filename) + + raw_keys_manifest = json.loads(reader(keys_uri).decode(ENCODING)) keys = KeysManifest.from_manifest_spec(raw_keys_manifest) plaintexts = cls._generate_plaintexts(raw_manifest["plaintexts"]) tests = {} for name, scenario in raw_manifest["tests"].items(): try: tests[name] = MessageEncryptionTestScenario.from_scenario( - scenario=scenario, keys=keys, plaintexts=plaintexts + scenario=scenario, keys=keys, plaintexts=plaintexts, keyrings=keyrings, keys_uri=keys_abs_path ) except NotImplementedError: continue From ca3e1654cdc0f926c4715a030a827b341ade072c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 10:06:04 -0800 Subject: [PATCH 163/376] cleanup --- .../materials_managers/mpl/cmm.py | 2 +- .../commands/full_message_decrypt.py | 4 +- .../commands/full_message_decrypt_generate.py | 4 +- .../commands/full_message_encrypt.py | 4 +- .../manifests/full_message/encrypt.py | 264 ++++++++++-------- 5 files changed, 154 insertions(+), 124 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index c454a114e..ebef5f7ac 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -90,7 +90,7 @@ def _native_to_mpl_get_encryption_materials( mpl_input_kwargs["algorithm_suite_id"] = \ CryptoMaterialsManagerFromMPL._native_algorithm_id_to_mpl_algorithm_id( request.algorithm.algorithm_id - ) + ) output: MPL_GetEncryptionMaterialsInput = MPL_GetEncryptionMaterialsInput( **mpl_input_kwargs diff --git a/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt.py b/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt.py index f28354f31..2a44cd597 100644 --- a/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt.py @@ -16,9 +16,9 @@ from awses_test_vectors.manifests.full_message.decrypt import MessageDecryptionManifest try: - import aws_cryptographic_materialproviders # noqa pylint: disable=unused-import + import aws_cryptographic_materialproviders # noqa pylint: disable=unused-import,import-error _HAS_MPL = True -except Exception as e: +except ImportError: _HAS_MPL = False diff --git a/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt_generate.py b/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt_generate.py index ae6afa538..69fe44d78 100644 --- a/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt_generate.py +++ b/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt_generate.py @@ -16,9 +16,9 @@ from awses_test_vectors.manifests.full_message.decrypt_generation import MessageDecryptionGenerationManifest try: - import aws_cryptographic_materialproviders # noqa pylint: disable=unused-import + import aws_cryptographic_materialproviders # noqa pylint: disable=unused-import,import-error _HAS_MPL = True -except Exception as e: +except ImportError: _HAS_MPL = False try: # Python 3.5.0 and 3.5.1 have incompatible typing modules diff --git a/test_vector_handlers/src/awses_test_vectors/commands/full_message_encrypt.py b/test_vector_handlers/src/awses_test_vectors/commands/full_message_encrypt.py index 5294e1791..268d7ca99 100644 --- a/test_vector_handlers/src/awses_test_vectors/commands/full_message_encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/commands/full_message_encrypt.py @@ -16,9 +16,9 @@ from awses_test_vectors.manifests.full_message.encrypt import MessageEncryptionManifest try: - import aws_cryptographic_materialproviders # noqa pylint: disable=unused-import + import aws_cryptographic_materialproviders # noqa pylint: disable=unused-import,import-error _HAS_MPL = True -except Exception as e: +except ImportError: _HAS_MPL = False diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index eabbe7343..1323fce88 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -88,60 +88,14 @@ class MessageEncryptionTestScenario(object): @classmethod def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): - # type: (ENCRYPT_SCENARIO_SPEC, KeysManifest, Dict[str, bytes], bool) -> MessageEncryptionTestScenario - """Load from a scenario specification. - - :param dict scenario: Scenario specification JSON - :param KeysManifest keys: Loaded keys - :param dict plaintexts: Mapping of plaintext names to plaintext values - :param bool keyrings: True if should encrypt with master key interfaces; False otherwise - :return: Loaded test scenario - :rtype: MessageEncryptionTestScenario - """ - - if keyrings: - return MessageEncryptionWithKeyringsTestScenario.from_scenario( - scenario, keys_uri, plaintexts - ) - else: - return MessageEncryptionWithMasterKeysTestScenario.from_scenario( - scenario, keys, plaintexts - ) - - def run(self, materials_manager=None): - """Run this scenario, writing the resulting ciphertext with ``ciphertext_writer`` and returning - a :class:`MessageDecryptionTestScenario` that describes the matching decrypt scenario. - - :param callable ciphertext_writer: Callable that will write the requested named ciphertext and - return a URI locating the written data - :param str plaintext_uri: URI locating the written plaintext data for this scenario - :return: Decrypt test scenario that describes the generated scenario - :rtype: MessageDecryptionTestScenario - """ - raise NotImplementedError("MUST specify keyrings bool") - - -@attr.s -class MessageEncryptionWithMasterKeysTestScenario(MessageEncryptionTestScenario): - # pylint: disable=too-many-instance-attributes - """Data class for a single full message decrypt test scenario that uses master keys. - - :param master_key_specs: Iterable of loaded master key specifications - :type master_key_specs: iterable of :class:`MasterKeySpec` - :param Callable master_key_provider_fn: - """ - - master_key_specs = attr.ib(validator=iterable_validator(list, MasterKeySpec)) - master_key_provider_fn = attr.ib(validator=attr.validators.is_callable()) - - @classmethod - def from_scenario(cls, scenario, keys, plaintexts): - # type: (ENCRYPT_SCENARIO_SPEC, KeysManifest, Dict[str, bytes]) -> MessageEncryptionTestScenario + # type: (ENCRYPT_SCENARIO_SPEC, KeysManifest, Dict[str, bytes], bool, str) -> MessageEncryptionTestScenario """Load from a scenario specification. :param dict scenario: Scenario specification JSON :param KeysManifest keys: Loaded keys :param dict plaintexts: Mapping of plaintext names to plaintext values + :param bool keyrings: True if should encrypt with keyring interfaces; False otherwise + :param str keys_uri: Path to the keys manifest :return: Loaded test scenario :rtype: MessageEncryptionTestScenario """ @@ -149,6 +103,8 @@ def from_scenario(cls, scenario, keys, plaintexts): master_key_specs = [MasterKeySpec.from_scenario(spec) for spec in scenario["master-keys"]] def master_key_provider_fn(): + if keyrings: + return keyring_from_master_key_specs(keys_uri, master_key_specs) return master_key_provider_from_master_key_specs(keys, master_key_specs) return cls( @@ -160,6 +116,8 @@ def master_key_provider_fn(): master_key=True, master_key_specs=master_key_specs, master_key_provider_fn=master_key_provider_fn, + keyrings=keyrings, + keys_uri=keys_uri, ) def run(self, materials_manager=None): @@ -185,80 +143,152 @@ def run(self, materials_manager=None): ) if materials_manager: encrypt_kwargs["materials_manager"] = materials_manager + elif self.keyrings: + encrypt_kwargs["keyring"] = self.master_key_provider_fn() else: encrypt_kwargs["key_provider"] = self.master_key_provider_fn() ciphertext, _header = client.encrypt(**encrypt_kwargs) return ciphertext -@attr.s -class MessageEncryptionWithKeyringsTestScenario(MessageEncryptionTestScenario): - # pylint: disable=too-many-instance-attributes - """Data class for a single full message decrypt test scenario that uses keyrings. - - :param master_key_specs: Iterable of loaded master key specifications - :type master_key_specs: iterable of :class:`MasterKeySpec` - :param Callable master_key_provider_fn: - """ - - master_key_specs = attr.ib(validator=iterable_validator(list, MasterKeySpec)) - master_key_provider_fn = attr.ib(validator=attr.validators.is_callable()) - - @classmethod - def from_scenario(cls, scenario, keys_uri, plaintexts): - # type: (ENCRYPT_SCENARIO_SPEC, KeysManifest, Dict[str, bytes]) -> MessageEncryptionTestScenario - """Load from a scenario specification. - - :param dict scenario: Scenario specification JSON - :param KeysManifest keys: Loaded keys - :param dict plaintexts: Mapping of plaintext names to plaintext values - :return: Loaded test scenario - :rtype: MessageEncryptionTestScenario - """ - algorithm = algorithm_suite_from_string_id(scenario["algorithm"]) - # manifest still keys these as `master-keys` even though these are keyrings - master_key_specs = [KeyringSpec.from_scenario(spec) for spec in scenario["master-keys"]] - - def keyring_provider_fn(): - return keyring_from_master_key_specs(keys_uri, master_key_specs) - - return cls( - plaintext_name=scenario["plaintext"], - plaintext=plaintexts[scenario["plaintext"]], - algorithm=algorithm, - frame_size=scenario["frame-size"], - encryption_context=scenario["encryption-context"], - master_key=True, - master_key_specs=master_key_specs, - master_key_provider_fn=keyring_provider_fn, - ) - - def run(self, materials_manager=None): - """Run this scenario, writing the resulting ciphertext with ``ciphertext_writer`` and returning - a :class:`MessageDecryptionTestScenario` that describes the matching decrypt scenario. - :param callable ciphertext_writer: Callable that will write the requested named ciphertext and - return a URI locating the written data - :param str plaintext_uri: URI locating the written plaintext data for this scenario - :return: Decrypt test scenario that describes the generated scenario - :rtype: MessageDecryptionTestScenario - """ - commitment_policy = CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT - if self.algorithm.is_committing(): - commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT - - client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=commitment_policy) - encrypt_kwargs = dict( - source=self.plaintext, - algorithm=self.algorithm, - frame_length=self.frame_size, - encryption_context=self.encryption_context, - ) - if materials_manager: - encrypt_kwargs["materials_manager"] = materials_manager - else: - encrypt_kwargs["keyring"] = self.keyring_provider_fn() - ciphertext, _header = client.encrypt(**encrypt_kwargs) - return ciphertext +# @attr.s +# class MessageEncryptionWithMasterKeysTestScenario(MessageEncryptionTestScenario): +# # pylint: disable=too-many-instance-attributes +# """Data class for a single full message decrypt test scenario that uses master keys. + +# :param master_key_specs: Iterable of loaded master key specifications +# :type master_key_specs: iterable of :class:`MasterKeySpec` +# :param Callable master_key_provider_fn: +# """ + +# master_key_specs = attr.ib(validator=iterable_validator(list, MasterKeySpec)) +# master_key_provider_fn = attr.ib(validator=attr.validators.is_callable()) + +# @classmethod +# def from_scenario(cls, scenario, keys, plaintexts): +# # type: (ENCRYPT_SCENARIO_SPEC, KeysManifest, Dict[str, bytes]) -> MessageEncryptionTestScenario +# """Load from a scenario specification. + +# :param dict scenario: Scenario specification JSON +# :param KeysManifest keys: Loaded keys +# :param dict plaintexts: Mapping of plaintext names to plaintext values +# :return: Loaded test scenario +# :rtype: MessageEncryptionTestScenario +# """ +# algorithm = algorithm_suite_from_string_id(scenario["algorithm"]) +# master_key_specs = [MasterKeySpec.from_scenario(spec) for spec in scenario["master-keys"]] + +# def master_key_provider_fn(): +# return master_key_provider_from_master_key_specs(keys, master_key_specs) + +# return cls( +# plaintext_name=scenario["plaintext"], +# plaintext=plaintexts[scenario["plaintext"]], +# algorithm=algorithm, +# frame_size=scenario["frame-size"], +# encryption_context=scenario["encryption-context"], +# master_key=True, +# master_key_specs=master_key_specs, +# master_key_provider_fn=master_key_provider_fn, +# ) + +# def run(self, materials_manager=None): +# """Run this scenario, writing the resulting ciphertext with ``ciphertext_writer`` and returning +# a :class:`MessageDecryptionTestScenario` that describes the matching decrypt scenario. + +# :param callable ciphertext_writer: Callable that will write the requested named ciphertext and +# return a URI locating the written data +# :param str plaintext_uri: URI locating the written plaintext data for this scenario +# :return: Decrypt test scenario that describes the generated scenario +# :rtype: MessageDecryptionTestScenario +# """ +# commitment_policy = CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT +# if self.algorithm.is_committing(): +# commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT + +# client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=commitment_policy) +# encrypt_kwargs = dict( +# source=self.plaintext, +# algorithm=self.algorithm, +# frame_length=self.frame_size, +# encryption_context=self.encryption_context, +# ) +# if materials_manager: +# encrypt_kwargs["materials_manager"] = materials_manager +# else: +# encrypt_kwargs["key_provider"] = self.master_key_provider_fn() +# ciphertext, _header = client.encrypt(**encrypt_kwargs) +# return ciphertext + +# @attr.s +# class MessageEncryptionWithKeyringsTestScenario(MessageEncryptionTestScenario): +# # pylint: disable=too-many-instance-attributes +# """Data class for a single full message decrypt test scenario that uses keyrings. + +# :param master_key_specs: Iterable of loaded master key specifications +# :type master_key_specs: iterable of :class:`MasterKeySpec` +# :param Callable master_key_provider_fn: +# """ + +# master_key_specs = attr.ib(validator=iterable_validator(list, MasterKeySpec)) +# master_key_provider_fn = attr.ib(validator=attr.validators.is_callable()) + +# @classmethod +# def from_scenario(cls, scenario, keys_uri, plaintexts): +# # type: (ENCRYPT_SCENARIO_SPEC, KeysManifest, Dict[str, bytes]) -> MessageEncryptionTestScenario +# """Load from a scenario specification. + +# :param dict scenario: Scenario specification JSON +# :param KeysManifest keys: Loaded keys +# :param dict plaintexts: Mapping of plaintext names to plaintext values +# :return: Loaded test scenario +# :rtype: MessageEncryptionTestScenario +# """ +# algorithm = algorithm_suite_from_string_id(scenario["algorithm"]) +# # manifest still keys these as `master-keys` even though these are keyrings +# master_key_specs = [KeyringSpec.from_scenario(spec) for spec in scenario["master-keys"]] + +# def keyring_provider_fn(): +# return keyring_from_master_key_specs(keys_uri, master_key_specs) + +# return cls( +# plaintext_name=scenario["plaintext"], +# plaintext=plaintexts[scenario["plaintext"]], +# algorithm=algorithm, +# frame_size=scenario["frame-size"], +# encryption_context=scenario["encryption-context"], +# master_key=True, +# master_key_specs=master_key_specs, +# master_key_provider_fn=keyring_provider_fn, +# ) + +# def run(self, materials_manager=None): +# """Run this scenario, writing the resulting ciphertext with ``ciphertext_writer`` and returning +# a :class:`MessageDecryptionTestScenario` that describes the matching decrypt scenario. + +# :param callable ciphertext_writer: Callable that will write the requested named ciphertext and +# return a URI locating the written data +# :param str plaintext_uri: URI locating the written plaintext data for this scenario +# :return: Decrypt test scenario that describes the generated scenario +# :rtype: MessageDecryptionTestScenario +# """ +# commitment_policy = CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT +# if self.algorithm.is_committing(): +# commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT + +# client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=commitment_policy) +# encrypt_kwargs = dict( +# source=self.plaintext, +# algorithm=self.algorithm, +# frame_length=self.frame_size, +# encryption_context=self.encryption_context, +# ) +# if materials_manager: +# encrypt_kwargs["materials_manager"] = materials_manager +# else: +# encrypt_kwargs["keyring"] = self.keyring_provider_fn() +# ciphertext, _header = client.encrypt(**encrypt_kwargs) +# return ciphertext @attr.s class MessageEncryptionManifest(object): From d6d1493e552bde19f39d86488e23712799c5c7b9 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 10:15:35 -0800 Subject: [PATCH 164/376] cleanup --- .../awses_test_vectors/manifests/full_message/encrypt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 1323fce88..09c2acf6a 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -84,7 +84,9 @@ class MessageEncryptionTestScenario(object): algorithm = attr.ib(validator=attr.validators.instance_of(AlgorithmSuite)) frame_size = attr.ib(validator=attr.validators.instance_of(int)) encryption_context = attr.ib(validator=dictionary_validator(six.string_types, six.string_types)) - master_key = attr.ib(validator=attr.validators.instance_of(bool)) + master_key_specs = attr.ib(validator=iterable_validator(list, MasterKeySpec)) + master_key_provider_fn = attr.ib(validator=attr.validators.is_callable()) + keyrings = attr.ib(validator=attr.validators.instance_of(bool)) @classmethod def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): @@ -113,11 +115,9 @@ def master_key_provider_fn(): algorithm=algorithm, frame_size=scenario["frame-size"], encryption_context=scenario["encryption-context"], - master_key=True, master_key_specs=master_key_specs, master_key_provider_fn=master_key_provider_fn, keyrings=keyrings, - keys_uri=keys_uri, ) def run(self, materials_manager=None): From 68ce94a1feb16a518e00799be465f9c6bd94d048 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 10:23:20 -0800 Subject: [PATCH 165/376] flake8 --- .../manifests/full_message/decrypt.py | 4 +--- .../manifests/full_message/decrypt_generation.py | 15 +++++++++------ .../manifests/full_message/encrypt.py | 9 +++------ 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index 6336400ce..73c5edd65 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -37,10 +37,8 @@ try: from awses_test_vectors.manifests.mpl_keyring import keyring_from_master_key_specs - _HAS_MPL = True - -except ImportError as e: +except ImportError: _HAS_MPL = False diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 61a62dd22..cf5e26ff4 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -33,7 +33,6 @@ from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig from aws_cryptographic_materialproviders.mpl.references import ( IKeyring, - CryptographicMaterialsManager, ) from aws_cryptographic_materialproviders.mpl.models import ( CreateDefaultCryptographicMaterialsManagerInput, @@ -41,10 +40,8 @@ from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL from awses_test_vectors.manifests.mpl_keyring import keyring_from_master_key_specs - - _HAS_MPL = True -except ImportError as e: - _HAS_MPL = False +except ImportError: + pass from awses_test_vectors.internal.defaults import ENCODING @@ -527,7 +524,13 @@ def from_file(cls, input_file, keyrings): ) except NotImplementedError: continue - return cls(version=raw_manifest["manifest"]["version"], keys=keys, plaintexts=plaintexts, tests=tests, keyrings=keyrings) + return cls( + version=raw_manifest["manifest"]["version"], + keys=keys, + plaintexts=plaintexts, + ests=tests, + keyrings=keyrings, + ) def run_and_write_to_dir(self, target_directory, json_indent=None): # type: (str, Optional[int]) -> None diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 09c2acf6a..42c9e14d2 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -40,12 +40,9 @@ from aws_encryption_sdk.identifiers import Algorithm as AlgorithmSuite try: - from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_from_master_key_specs - - _HAS_MPL = True - -except ImportError as e: - _HAS_MPL = False + from awses_test_vectors.manifests.mpl_keyring import keyring_from_master_key_specs +except ImportError: + pass try: # Python 3.5.0 and 3.5.1 have incompatible typing modules From 9269fc4666eca876f0d1e7280906cba8af5e0c58 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 10:30:02 -0800 Subject: [PATCH 166/376] pylint --- .../manifests/full_message/decrypt.py | 13 ++++--------- .../manifests/full_message/decrypt_generation.py | 13 ++----------- .../manifests/full_message/encrypt.py | 1 + 3 files changed, 7 insertions(+), 20 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index 73c5edd65..cb7df63c9 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -188,7 +188,7 @@ class DecryptionMethod(Enum): @attr.s(init=False) class MessageDecryptionTestScenario(object): - # pylint: disable=too-many-arguments + # pylint: disable=too-many-arguments,too-many-instance-attributes """Data class for a single full message decrypt test scenario. Handles serialization and deserialization to and from manifest specs. @@ -266,16 +266,11 @@ def from_scenario( """ raw_master_key_specs = scenario["master-keys"] # type: Iterable[MASTER_KEY_SPEC] master_key_specs = [MasterKeySpec.from_scenario(spec) for spec in raw_master_key_specs] - # if keyrings: - # master_key_specs = [KeyringSpec.from_scenario(spec) for spec in raw_master_key_specs] - # else: - # master_key_specs = [MasterKeySpec.from_scenario(spec) for spec in raw_master_key_specs] def master_key_provider_fn(): if keyrings: return keyring_from_master_key_specs(keys_uri, master_key_specs) - else: - return master_key_provider_from_master_key_specs(keys, master_key_specs) + return master_key_provider_from_master_key_specs(keys, master_key_specs) decryption_method_spec = scenario.get("decryption-method") decryption_method = DecryptionMethod(decryption_method_spec) if decryption_method_spec else None @@ -316,8 +311,7 @@ def _one_shot_decrypt(self): client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) if self.keyrings: return client.decrypt(source=self.ciphertext, keyring=self.master_key_provider_fn()) - else: - return client.decrypt(source=self.ciphertext, key_provider=self.master_key_provider_fn()) + return client.decrypt(source=self.ciphertext, key_provider=self.master_key_provider_fn()) def _streaming_decrypt(self): result = bytearray() @@ -423,6 +417,7 @@ def manifest_spec(self): @classmethod def from_file(cls, input_file, keyrings): + # noqa pylint disable=too-many-locals # type: (IO) -> MessageDecryptionManifest """Load from a file containing a full message decrypt manifest. diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index cf5e26ff4..41453fdb6 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -370,6 +370,7 @@ class MessageDecryptionTestScenarioGenerator(object): @classmethod def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): + # noqa pylint disable=too-many-arguments,too-many-locals """Load from a scenario specification. :param dict scenario: Scenario specification JSON @@ -397,20 +398,10 @@ def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): MasterKeySpec.from_scenario(spec) for spec in scenario["decryption-master-keys"] ] - # if keyrings: - # decryption_master_key_specs = [ - # KeyringSpec.from_scenario(spec) for spec in scenario["decryption-master-keys"] - # ] - # else: - # decryption_master_key_specs = [ - # MasterKeySpec.from_scenario(spec) for spec in scenario["decryption-master-keys"] - # ] - def decryption_master_key_provider_fn(): if keyrings: return keyring_from_master_key_specs(keys_uri, decryption_master_key_specs) - else: - return master_key_provider_from_master_key_specs(keys, decryption_master_key_specs) + return master_key_provider_from_master_key_specs(keys, decryption_master_key_specs) else: decryption_master_key_specs = encryption_scenario.master_key_specs diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 42c9e14d2..2d3c06249 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -87,6 +87,7 @@ class MessageEncryptionTestScenario(object): @classmethod def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): + # noqa pylint disable=too-many-arguments # type: (ENCRYPT_SCENARIO_SPEC, KeysManifest, Dict[str, bytes], bool, str) -> MessageEncryptionTestScenario """Load from a scenario specification. From 0a972e6ce811fcc1ce018980f434060d61739322 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 10:33:43 -0800 Subject: [PATCH 167/376] pylint --- .../src/awses_test_vectors/manifests/full_message/decrypt.py | 2 +- .../manifests/full_message/decrypt_generation.py | 2 +- .../src/awses_test_vectors/manifests/full_message/encrypt.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index cb7df63c9..f5c93058d 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -417,7 +417,7 @@ def manifest_spec(self): @classmethod def from_file(cls, input_file, keyrings): - # noqa pylint disable=too-many-locals + # pylint: disable=too-many-locals # type: (IO) -> MessageDecryptionManifest """Load from a file containing a full message decrypt manifest. diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 41453fdb6..2e434e780 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -370,7 +370,7 @@ class MessageDecryptionTestScenarioGenerator(object): @classmethod def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): - # noqa pylint disable=too-many-arguments,too-many-locals + # pylint: disable=too-many-arguments,too-many-locals """Load from a scenario specification. :param dict scenario: Scenario specification JSON diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 2d3c06249..084981eaf 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -87,7 +87,7 @@ class MessageEncryptionTestScenario(object): @classmethod def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): - # noqa pylint disable=too-many-arguments + # pylint: disable=too-many-arguments # type: (ENCRYPT_SCENARIO_SPEC, KeysManifest, Dict[str, bytes], bool, str) -> MessageEncryptionTestScenario """Load from a scenario specification. From e185c35a61243b4f3bfabaad8c60f2c75e12f1b7 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 10:54:58 -0800 Subject: [PATCH 168/376] fix --- test/mpl/unit/test_material_managers_mpl_cmm.py | 2 +- .../manifests/full_message/decrypt_generation.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/mpl/unit/test_material_managers_mpl_cmm.py b/test/mpl/unit/test_material_managers_mpl_cmm.py index 80d6f00ee..16323d496 100644 --- a/test/mpl/unit/test_material_managers_mpl_cmm.py +++ b/test/mpl/unit/test_material_managers_mpl_cmm.py @@ -96,7 +96,7 @@ def test_GIVEN_valid_request_WHEN_get_encryption_materials_THEN_return_Encryptio @patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL" - "._native_to_mpl_commmitment_policy") + "._native_to_mpl_get_encryption_materials") def test_GIVEN_mpl_cmm_raises_MPLException_WHEN_get_encryption_materials_THEN_raise_ESDKException( _ ): diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 2e434e780..3b0b50a94 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -519,7 +519,7 @@ def from_file(cls, input_file, keyrings): version=raw_manifest["manifest"]["version"], keys=keys, plaintexts=plaintexts, - ests=tests, + tests=tests, keyrings=keyrings, ) From 4f8633786f79a76d6292276882dcd94efe97c44a Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 10:59:57 -0800 Subject: [PATCH 169/376] fix --- test/mpl/unit/test_material_managers_mpl_cmm.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/mpl/unit/test_material_managers_mpl_cmm.py b/test/mpl/unit/test_material_managers_mpl_cmm.py index 16323d496..8701cde00 100644 --- a/test/mpl/unit/test_material_managers_mpl_cmm.py +++ b/test/mpl/unit/test_material_managers_mpl_cmm.py @@ -110,10 +110,13 @@ def test_GIVEN_mpl_cmm_raises_MPLException_WHEN_get_encryption_materials_THEN_ra cmm.get_encryption_materials(mock_encryption_materials_request) +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL" + "._native_algorithm_id_to_mpl_algorithm_id") @patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL" "._native_to_mpl_commmitment_policy") def test_GIVEN_valid_mpl_commitment_policy_WHEN_native_to_mpl_get_encryption_materials_THEN_returns_MPL_GetEncryptionMaterialsInput( # noqa: E501 - mock_mpl_commitment_policy + mock_mpl_commitment_policy, + mock_mpl_algorithm, ): # Given: commitment policy is some MPL ESDK commitment policy mock_commitment_policy = MagicMock(__class__=MPL_CommitmentPolicyESDK) @@ -129,6 +132,7 @@ def test_GIVEN_valid_mpl_commitment_policy_WHEN_native_to_mpl_get_encryption_mat assert output.encryption_context == mock_encryption_materials_request.encryption_context assert output.commitment_policy == mock_commitment_policy assert output.max_plaintext_length == mock_encryption_materials_request.plaintext_length + assert output.algorithm_suite_id == mock_mpl_algorithm() def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_MPL_CommitmentPolicyESDK_FORBID_ENCRYPT_ALLOW_DECRYPT(): # noqa: E501 From dd89e32af52012e64a6d44c60489e888db051fbd Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 13:14:12 -0800 Subject: [PATCH 170/376] add mpl --- buildspec.yml | 8 ++++++++ .../manifests/full_message/decrypt.py | 11 +++++++++-- .../manifests/full_message/decrypt_generation.py | 13 +++++++++---- .../manifests/full_message/encrypt.py | 12 ++++++++++-- test_vector_handlers/tox.ini | 9 ++++++++- 5 files changed, 44 insertions(+), 9 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index 5dbd3f2b8..fff7c68d1 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -78,6 +78,10 @@ batch: buildspec: codebuild/py311/awses_local_mpl.yml env: image: aws/codebuild/standard:7.0 + - identifier: py311_mplawses_latest_mpl + buildspec: codebuild/py311/mplawses_local_mpl.yml + env: + image: aws/codebuild/standard:7.0 - identifier: py312_integ buildspec: codebuild/py312/integ.yml @@ -103,6 +107,10 @@ batch: buildspec: codebuild/py312/awses_local_mpl.yml env: image: aws/codebuild/standard:7.0 + - identifier: py312_mplawses_latest_mpl + buildspec: codebuild/py312/mplawses_local_mpl.yml + env: + image: aws/codebuild/standard:7.0 - identifier: code_coverage buildspec: codebuild/coverage/coverage.yml diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index f5c93058d..6d4a4a75d 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -36,7 +36,7 @@ from awses_test_vectors.manifests.master_key import MasterKeySpec, master_key_provider_from_master_key_specs try: - from awses_test_vectors.manifests.mpl_keyring import keyring_from_master_key_specs + from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_from_master_key_specs _HAS_MPL = True except ImportError: _HAS_MPL = False @@ -265,7 +265,14 @@ def from_scenario( :rtype: MessageDecryptionTestScenario """ raw_master_key_specs = scenario["master-keys"] # type: Iterable[MASTER_KEY_SPEC] - master_key_specs = [MasterKeySpec.from_scenario(spec) for spec in raw_master_key_specs] + if keyrings: + master_key_specs = [ + KeyringSpec.from_scenario(spec) for spec in raw_master_key_specs + ] + else: + master_key_specs = [ + MasterKeySpec.from_scenario(spec) for spec in raw_master_key_specs + ] def master_key_provider_fn(): if keyrings: diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 3b0b50a94..4bbb3df5d 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -39,7 +39,7 @@ ) from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL - from awses_test_vectors.manifests.mpl_keyring import keyring_from_master_key_specs + from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_from_master_key_specs except ImportError: pass @@ -394,9 +394,14 @@ def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): decryption_method_spec = scenario.get("decryption-method") decryption_method = DecryptionMethod(decryption_method_spec) if decryption_method_spec else None if "decryption-master-keys" in scenario: - decryption_master_key_specs = [ - MasterKeySpec.from_scenario(spec) for spec in scenario["decryption-master-keys"] - ] + if keyrings: + decryption_master_key_specs = [ + KeyringSpec.from_scenario(spec) for spec in scenario["decryption-master-keys"] + ] + else: + decryption_master_key_specs = [ + MasterKeySpec.from_scenario(spec) for spec in scenario["decryption-master-keys"] + ] def decryption_master_key_provider_fn(): if keyrings: diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 084981eaf..82a5e379a 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -40,7 +40,7 @@ from aws_encryption_sdk.identifiers import Algorithm as AlgorithmSuite try: - from awses_test_vectors.manifests.mpl_keyring import keyring_from_master_key_specs + from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_from_master_key_specs except ImportError: pass @@ -100,7 +100,15 @@ def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): :rtype: MessageEncryptionTestScenario """ algorithm = algorithm_suite_from_string_id(scenario["algorithm"]) - master_key_specs = [MasterKeySpec.from_scenario(spec) for spec in scenario["master-keys"]] + + if keyrings: + master_key_specs = [ + KeyringSpec.from_scenario(spec) for spec in scenario["master-keys"] + ] + else: + master_key_specs = [ + MasterKeySpec.from_scenario(spec) for spec in scenario["master-keys"] + ] def master_key_provider_fn(): if keyrings: diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index 580b641e0..eeb672833 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -4,6 +4,7 @@ envlist = # so until release we can only effectively test the local version of the ESDK. py{37,38,39,310}-awses_local py{311,312}-awses_local{,-mpl} + py{311,312}-mplawses_local-mpl # 1.2.0 and 1.2.max are being difficult because of attrs bandit, doc8, readme, {flake8,pylint}{,-tests}, @@ -36,7 +37,7 @@ envlist = # release :: Builds dist files and uploads to pypi pypirc profile. [testenv:base-command] -commands = pytest --basetemp={envtmpdir} -l --cov awses_test_vectors test/ {posargs} +commands = pytest --basetemp={envtmpdir} -l --cov awses_test_vectors test/ --ignore test/keyrings/ {posargs} [testenv] passenv = @@ -55,6 +56,12 @@ deps = commands = {[testenv:base-command]commands} +[testenv:mplawses_local] +basepython = python3 +sitepackages = False +deps = .. +commands = pytest --basetemp={envtmpdir} -l test/ {posargs} + [testenv:full-encrypt] basepython = python3 sitepackages = False From 5c5fb4b5c8e4716a6237a11e88ae0bb149325198 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 13:20:46 -0800 Subject: [PATCH 171/376] missing --- codebuild/py311/mplawses_local_mpl.yml | 26 ++++++++++++++++++++ codebuild/py312/mplawses_local_mpl.yml | 33 ++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 codebuild/py311/mplawses_local_mpl.yml create mode 100644 codebuild/py312/mplawses_local_mpl.yml diff --git a/codebuild/py311/mplawses_local_mpl.yml b/codebuild/py311/mplawses_local_mpl.yml new file mode 100644 index 000000000..22cd1dd81 --- /dev/null +++ b/codebuild/py311/mplawses_local_mpl.yml @@ -0,0 +1,26 @@ +version: 0.2 + +env: + variables: + TOXENV: "py311-mplawses_local-mpl" + REGION: "us-west-2" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + python: 3.11 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - tox diff --git a/codebuild/py312/mplawses_local_mpl.yml b/codebuild/py312/mplawses_local_mpl.yml new file mode 100644 index 000000000..c9ab1b618 --- /dev/null +++ b/codebuild/py312/mplawses_local_mpl.yml @@ -0,0 +1,33 @@ +# Runs the same tests as awses_local in an environment with the MPL installed. +# This asserts existing tests continue to pass with the MPL installed. +version: 0.2 + +env: + variables: + TOXENV: "py312-mplawses_local-mpl" + REGION: "us-west-2" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + python: latest + build: + commands: + - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - + - pyenv install --skip-existing 3.12.0 + - pyenv local 3.12.0 + - pip install --upgrade pip + - pip install setuptools + - pip install "tox < 4.0" + - cd test_vector_handlers + - tox From 9cf2191b8400655e316dae92b4da64055a2cb2bd Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 13:29:20 -0800 Subject: [PATCH 172/376] fix --- test_vector_handlers/tox.ini | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index eeb672833..607f854d2 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -4,7 +4,7 @@ envlist = # so until release we can only effectively test the local version of the ESDK. py{37,38,39,310}-awses_local py{311,312}-awses_local{,-mpl} - py{311,312}-mplawses_local-mpl + py{311,312}-mplvectors-mpl # 1.2.0 and 1.2.max are being difficult because of attrs bandit, doc8, readme, {flake8,pylint}{,-tests}, @@ -37,7 +37,7 @@ envlist = # release :: Builds dist files and uploads to pypi pypirc profile. [testenv:base-command] -commands = pytest --basetemp={envtmpdir} -l --cov awses_test_vectors test/ --ignore test/keyrings/ {posargs} +commands = pytest --basetemp={envtmpdir} -l --cov awses_test_vectors test/integration {posargs} [testenv] passenv = @@ -56,11 +56,11 @@ deps = commands = {[testenv:base-command]commands} -[testenv:mplawses_local] +[testenv:mplvectors-mpl] basepython = python3 sitepackages = False deps = .. -commands = pytest --basetemp={envtmpdir} -l test/ {posargs} +commands = pytest --basetemp={envtmpdir} -l test/keyrings {posargs} [testenv:full-encrypt] basepython = python3 From df10d915062ceff9bb7da349e16541c1983a0ad9 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 13:32:11 -0800 Subject: [PATCH 173/376] fix --- codebuild/py311/mplawses_local_mpl.yml | 2 +- codebuild/py312/mplawses_local_mpl.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/codebuild/py311/mplawses_local_mpl.yml b/codebuild/py311/mplawses_local_mpl.yml index 22cd1dd81..92dbdb086 100644 --- a/codebuild/py311/mplawses_local_mpl.yml +++ b/codebuild/py311/mplawses_local_mpl.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - TOXENV: "py311-mplawses_local-mpl" + TOXENV: "py311-mplvectors-mpl" REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f diff --git a/codebuild/py312/mplawses_local_mpl.yml b/codebuild/py312/mplawses_local_mpl.yml index c9ab1b618..e3f06e7f6 100644 --- a/codebuild/py312/mplawses_local_mpl.yml +++ b/codebuild/py312/mplawses_local_mpl.yml @@ -4,7 +4,7 @@ version: 0.2 env: variables: - TOXENV: "py312-mplawses_local-mpl" + TOXENV: "py312-mplvectors-mpl" REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f From e332d5b80b03727c1bc7790d1dffa1170b451ad4 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 13:43:15 -0800 Subject: [PATCH 174/376] fix --- test_vector_handlers/tox.ini | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index 607f854d2..0b18bf715 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -37,7 +37,7 @@ envlist = # release :: Builds dist files and uploads to pypi pypirc profile. [testenv:base-command] -commands = pytest --basetemp={envtmpdir} -l --cov awses_test_vectors test/integration {posargs} +commands = pytest --basetemp={envtmpdir} -l --cov awses_test_vectors {posargs} [testenv] passenv = @@ -54,13 +54,8 @@ deps = mpl: -r../requirements_mpl.txt .. commands = - {[testenv:base-command]commands} - -[testenv:mplvectors-mpl] -basepython = python3 -sitepackages = False -deps = .. -commands = pytest --basetemp={envtmpdir} -l test/keyrings {posargs} + awses_local: {[testenv:base-command]commands} test/integration + mplvectors: {[testenv:base-command]commands} test/keyrings [testenv:full-encrypt] basepython = python3 From 8dbeeb34591d44592f0f2eece21ceedab88f052e Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 13:46:19 -0800 Subject: [PATCH 175/376] add missing files --- .../manifests/mpl_keyring.py | 119 ++++++++++ test_vector_handlers/test/__init__.py | 0 .../test/keyrings/__init__.py | 0 .../test/keyrings/integration/__init__.py | 0 .../keyrings/integration/commands/__init__.py | 0 .../test_i_full_message_encrypt_keyrings.py | 62 +++++ test_vector_handlers/test/keys.json | 214 ++++++++++++++++++ 7 files changed, 395 insertions(+) create mode 100644 test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py create mode 100644 test_vector_handlers/test/__init__.py create mode 100644 test_vector_handlers/test/keyrings/__init__.py create mode 100644 test_vector_handlers/test/keyrings/integration/__init__.py create mode 100644 test_vector_handlers/test/keyrings/integration/commands/__init__.py create mode 100644 test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py create mode 100644 test_vector_handlers/test/keys.json diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py new file mode 100644 index 000000000..820cd00c2 --- /dev/null +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -0,0 +1,119 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Keyring Manifest handler. + +This REQUIRES the aws-cryptographic-material-providers library. +""" +import attr + +from aws_cryptography_materialproviderstestvectorkeys.smithygenerated.\ + aws_cryptography_materialproviderstestvectorkeys.models import ( + GetKeyDescriptionInput, + GetKeyDescriptionOutput, + TestVectorKeyringInput, + ) +from aws_cryptography_materialproviderstestvectorkeys.smithygenerated.\ + aws_cryptography_materialproviderstestvectorkeys.client import ( + KeyVectors, + ) +from aws_cryptography_materialproviderstestvectorkeys.smithygenerated.\ + aws_cryptography_materialproviderstestvectorkeys.config import ( + KeyVectorsConfig + ) +from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders +from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig +from aws_cryptographic_materialproviders.mpl.references import IKeyring +from aws_cryptographic_materialproviders.mpl.models import CreateMultiKeyringInput + +from awses_test_vectors.manifests.keys import KeysManifest # noqa pylint disable=unused-import + +import json + +from .master_key import MasterKeySpec + + +@attr.s +class KeyringSpec(MasterKeySpec): # pylint: disable=too-many-instance-attributes + """AWS Encryption SDK master key specification utilities. + + Described in AWS Crypto Tools Test Vector Framework features #0003 and #0004. + + :param str type_name: Master key type name + :param str key_name: Name of key in keys spec + :param str provider_id: Master key provider ID + :param str encryption_algorithm: Wrapping key encryption algorithm (required for raw master keys) + :param str padding_algorithm: Wrapping key padding algorithm (required for raw master keys) + :param str padding_hash: Wrapping key padding hash (required for raw master keys) + """ + + def keyring(self, keys_uri): + # type: (KeysManifest) -> IKeyring + """Build a keyring using this specification. + + :param str keys_uri: Path to the keys manifest + """ + + keyvectors = KeyVectors(KeyVectorsConfig(key_manifest_path=keys_uri)) + + # Construct the input to KeyVectorsConfig + input_as_dict = { + "type": self.type_name, + "key": self.key_name, + "provider-id": self.provider_id, + "encryption-algorithm": self.encryption_algorithm, + "padding-algorithm": self.padding_algorithm, + "padding-hash": self.padding_hash + } + # stringify the dict + input_as_string = json.dumps(input_as_dict) + # convert to unicode code point (expected representation) + encoded_json = [ord(c) for c in input_as_string] + + output: GetKeyDescriptionOutput = keyvectors.get_key_description( + GetKeyDescriptionInput(json=encoded_json) + ) + + keyring: IKeyring = keyvectors.create_test_vector_keyring( + TestVectorKeyringInput( + key_description=output.key_description + ) + ) + + return keyring + + +def keyring_from_master_key_specs(keys_uri, master_key_specs): + # type: (str, list[KeyringSpec]) -> IKeyring + """Build and combine all keyrings identified by the provided specs and + using the provided keys. + + :param str keys_uri: Path to the keys manifest + :param master_key_specs: Master key specs from which to load master keys + :type master_key_specs: iterable of MasterKeySpec + :return: Master key provider combining all loaded master keys + :rtype: IKeyring + """ + keyrings = [spec.keyring(keys_uri) for spec in master_key_specs] + primary = keyrings[0] + others = keyrings[1:] + + mpl: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + MaterialProvidersConfig() + ) + multi_keyring: IKeyring = mpl.create_multi_keyring( + CreateMultiKeyringInput( + generator=primary, + child_keyrings=others + ) + ) + return multi_keyring diff --git a/test_vector_handlers/test/__init__.py b/test_vector_handlers/test/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test_vector_handlers/test/keyrings/__init__.py b/test_vector_handlers/test/keyrings/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test_vector_handlers/test/keyrings/integration/__init__.py b/test_vector_handlers/test/keyrings/integration/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test_vector_handlers/test/keyrings/integration/commands/__init__.py b/test_vector_handlers/test/keyrings/integration/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py b/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py new file mode 100644 index 000000000..56bf3112c --- /dev/null +++ b/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py @@ -0,0 +1,62 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +""" +Integration tests for `awses_test_vectors.commands` with keyrings. +""" +import pytest + +from awses_test_vectors.commands import full_message_decrypt, full_message_decrypt_generate, full_message_encrypt + +from ....integration.integration_test_utils import ( # noqa pylint: disable=unused-import + full_message_decrypt_generation_vectors, + full_message_encrypt_vectors, +) + +pytestmark = [pytest.mark.integ] + + +def test_full_message_encrypt_canonical_full(full_message_encrypt_vectors): + full_message_encrypt.cli(["--input", full_message_encrypt_vectors]) + full_message_encrypt.cli(["--input", full_message_encrypt_vectors], "--keyrings") + + +def test_full_message_cycle_canonical_full(tmpdir, full_message_decrypt_generation_vectors): + # Generate vectors using keyring interfaces + keyring_output_dir = tmpdir.join("output-keyrings") + full_message_decrypt_generate.cli([ + "--output", + str(keyring_output_dir), + "--input", + full_message_decrypt_generation_vectors, + "--keyrings" + ]) + # Generate vectors using master key interfaces + master_key_output_dir = tmpdir.join("output-master-key") + full_message_decrypt_generate.cli([ + "--output", + str(master_key_output_dir), + "--input", + full_message_decrypt_generation_vectors + ]) + + # Validate that vectors generated using keyring interfaces + # can be decrypted by BOTH keyring and master key interfaces + keyring_decrypt_manifest_file = keyring_output_dir.join("manifest.json") + full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file), "--keyrings"]) + full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file)]) + + # Validate that vectors generated using master key interfaces + # can be decrypted by BOTH keyring and master key interfaces + master_key_decrypt_manifest_file = keyring_output_dir.join("manifest.json") + full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file), "--keyrings"]) + full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file)]) diff --git a/test_vector_handlers/test/keys.json b/test_vector_handlers/test/keys.json new file mode 100644 index 000000000..304dae5f7 --- /dev/null +++ b/test_vector_handlers/test/keys.json @@ -0,0 +1,214 @@ +{ + "manifest": { + "type": "keys", + "version": 3 + }, + "keys": { + "aes-128": { + "encrypt": true, + "decrypt": true, + "algorithm": "aes", + "type": "symmetric", + "bits": 128, + "encoding": "base64", + "material": "AAECAwQFBgcICRAREhMUFQ==", + "key-id": "aes-128" + }, + "aes-192": { + "encrypt": true, + "decrypt": true, + "algorithm": "aes", + "type": "symmetric", + "bits": 192, + "encoding": "base64", + "material": "AAECAwQFBgcICRAREhMUFRYXGBkgISIj", + "key-id": "aes-192" + }, + "aes-256": { + "encrypt": true, + "decrypt": true, + "algorithm": "aes", + "type": "symmetric", + "bits": 256, + "encoding": "base64", + "material": "AAECAwQFBgcICRAREhMUFRYXGBkgISIjJCUmJygpMDE=", + "key-id": "aes-256" + }, + "rsa-4096-private": { + "encrypt": true, + "decrypt": true, + "algorithm": "rsa", + "type": "private", + "bits": 4096, + "encoding": "pem", + "material": "-----BEGIN PRIVATE KEY-----\nMIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCztGg1gQ8AjCzz\n1VX6StqtW//jBt2ZQBoApaBa7FmLmdr0YlKaeEKSrItGbvA9tBjgsKhrn8gxTGQc\nuxgM92651jRCbQZyjE6W8kodijhGMXsfKJLfgPp2/I7gZ3dqrSZkejFIYLFb/uF/\nTfAQzNyJUldYdeFojSUPqevMgSAusTgv7dXYt4BCO9mxMp35tgyp5k4vazKJVUgB\nTw87AAYZUGugmi94Wb9JSnqUKI3QzaRN7JADZrHdBO1lIBryfCsjtTnZc7NWZ0yJ\nwmzLY+C5b3y17cy44N0rbjI2QciRhqZ4/9SZ/9ImyFQlB3lr9NSndcT4eE5YC6bH\nba0gOUK9lLXVy6TZ+nRZ4dSddoLX03mpYp+8cQpK6DO3L/PeUY/si0WGsXZfWokd\n4ACwvXWSOjotzjwqwTW8q9udbhUvIHfB02JW+ZQ07b209fBpHRDkZuveOTedTN2Q\nQei4dZDjWW5s4cIIE3dXXeaH8yC02ERIeN+aY6eHngSsP2xoDV3sKNN/yDbCqaMS\nq8ZJbo2rvOFxZHa2nWiV+VLugfO6Xj8jeGeR8vopvbEBZZpAq+Dea2xjY4+XMUQ/\nS1HlRwc9+nkJ5LVfODuE3q9EgJbqbiXe7YckWV3ZqQMybW+dLPxEJs9buOntgHFS\nRYmbKky0bti/ZoZlcZtS0zyjVxlqsQIDAQABAoICAEr3m/GWIXgNAkPGX9PGnmtr\n0dgX6SIhh7d1YOwNZV3DlYAV9HfUa5Fcwc1kQny7QRWbHOepBI7sW2dQ9buTDXIh\nVjPP37yxo6d89EZWfxtpUP+yoXL0D4jL257qCvtJuJZ6E00qaVMDhXbiQKABlo8C\n9sVEiABhwXBDZsctpwtTiykTgv6hrrPy2+H8R8MAm0/VcBCAG9kG5r8FCEmIvQKa\ndgvNxrfiWNZuZ6yfLmpJH54SbhG9Kb4WbCKfvh4ihqyi0btRdSM6fMeLgG9o/zrc\ns54B0kHeLOYNVo0j7FQpZBFeSIbmHfln4RKBh7ntrTke/Ejbh3NbiPvxWSP0P067\nSYWPkQpip2q0ION81wSQZ1haP2GewFFu4IEjG3DlqqpKKGLqXrmjMufnildVFpBx\nir+MgvgQfEBoGEx0aElyO7QuRYaEiXeb/BhMZeC5O65YhJrWSuTVizh3xgJWjgfV\naYwYgxN8SBXBhXLIVvnPhadTqsW1C/aevLOk110eSFWcHf+FCK781ykIzcpXoRGX\nOwWcZzC/fmSABS0yH56ow+I0tjdLIEEMhoa4/kkamioHOJ4yyB+W1DO6/DnMyQlx\ng7y2WsAaIEBoWUARy776k70xPPMtYAxzFXI9KhqRVrPfeaRZ+ojeyLyr3GQGyyoo\ncuGRdMUblsmODv4ixmOxAoIBAQDvkznvVYNdP3Eg5vQeLm/qsP6dLejLijBLeq9i\n7DZH2gRpKcflXZxCkRjsKDDE+fgDcBYEp2zYfRIVvgrxlTQZdaSG+GoDcbjbNQn3\ndjCCtOOACioN/vg2zFlX4Bs6Q+NaV7g5qP5SUaxUBjuHLe7Nc+ZkyheMHuNYVLvk\nHL/IoWyANpZYjMUU3xMbL/J29Gz7CPGr8Si28TihAHGfcNgn8S04OQZhTX+bU805\n/+7B4XW47Mthg/u7hlqFl+YIAaSJYvWkEaVP1A9I7Ve0aMDSMWwzTg9cle2uVaL3\n+PTzWY5coBlHKjqAg9ufhYSDhAqBd/JOSlv8RwcA3PDXJ6C/AoIBAQDABmXXYQky\n7phExXBvkLtJt2TBGjjwulf4R8TC6W5F51jJuoqY/mTqYcLcOn2nYGVwoFvPsy/Q\nCTjfODwJBXzbloXtYFR3PWAeL1Y6+7Cm+koMWIPJyVbD5Fzm+gZStM0GwP8FhDt2\nWt8fWEyXmoLdAy6RAwiEmCagEh8o+13oBfwnBllbz7TxaErsUuR+XVgl/iHwztdv\ncdJKyRgaFfWSh9aiO7EMV2rBGWsoX09SRvprPFAGx8Ffm7YcqIk34QXsQyc45Dyn\nCwkvypxHoaB3ot/48FeFm9IubApb/ctv+EgkBfL4S4bdwRXS1rt+0+QihBoFyP2o\nJ91cdm4hEWCPAoIBAQC6l11hFaYZo0bWDGsHcr2B+dZkzxPoKznQH76n+jeQoLIc\nwgjJkK4afm39yJOrZtEOxGaxu0CgIFFMk9ZsL/wC9EhvQt02z4TdXiLkFK5VrtMd\nr0zv16y06VWQhqBOMf/KJlX6uq9RqADi9HO6pkC+zc0cpPXQEWKaMmygju+kMG2U\nMm/IieMZjWCRJTfgBCE5J88qTsqaKagkZXcZakdAXKwOhQN+F2EStiM6UCZB5PrO\nS8dfrO8ML+ki8Zqck8L1qhiNb5zkXtKExy4u+gNr8khGcT6vqqoSxOoH3mPRgOfL\nJnppne8wlwIf7Vq3H8ka6zPSXEHma999gZcmy9t7AoIBAGbQhiLl79j3a0wXMvZp\nVf5IVYgXFDnAbG2hb7a06bhAAIgyexcjzsC4C2+DWdgOgwHkuoPg+062QV8zauGh\nsJKaa6cHlvIpSJeg3NjD/nfJN3CYzCd0yCIm2Z9Ka6xI5iYhm+pGPNhIG4Na8deS\ngVL46yv1pc/o73VxfoGg5UzgN3xlp97Cva0sHEGguHr4W8Qr59xZw3wGQ4SLW35M\nF6qXVNKUh12GSMCPbZK2RXBWVKqqJmca+WzJoJ6DlsT2lQdFhXCus9L007xlDXxF\nC/hCmw1dEl+VaNo2Ou26W/zdwTKYhNlxBwsg4SB8nPNxXIsmlBBY54froFhriNfn\nx/0CggEAUzz+VMtjoEWw2HSHLOXrO4EmwJniNgiiwfX3DfZE4tMNZgqZwLkq67ns\nT0n3b0XfAOOkLgMZrUoOxPHkxFeyLLf7pAEJe7QNB+Qilw8e2zVqtiJrRk6uDIGJ\nSv+yM52zkImZAe2jOdU3KeUZxSMmb5vIoiPBm+tb2WupAg3YdpKn1/jWTpVmV/+G\nUtTLVE6YpAyFp1gMxhutE9vfIS94ek+vt03AoEOlltt6hqZfv3xmY8vGuAjlnj12\nzHaq+fhCRPsbsZkzJ9nIVdXYnNIEGtMGNnxax7tYRej/UXqyazbxHiJ0iPF4PeDn\ndzxtGxpeTBi+KhKlca8SlCdCqYwG6Q==\n-----END PRIVATE KEY-----", + "key-id": "rsa-4096" + }, + "rsa-4096-public": { + "encrypt": true, + "decrypt": false, + "algorithm": "rsa", + "type": "public", + "bits": 4096, + "encoding": "pem", + "material": "-----BEGIN PUBLIC KEY-----\nMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAs7RoNYEPAIws89VV+kra\nrVv/4wbdmUAaAKWgWuxZi5na9GJSmnhCkqyLRm7wPbQY4LCoa5/IMUxkHLsYDPdu\nudY0Qm0GcoxOlvJKHYo4RjF7HyiS34D6dvyO4Gd3aq0mZHoxSGCxW/7hf03wEMzc\niVJXWHXhaI0lD6nrzIEgLrE4L+3V2LeAQjvZsTKd+bYMqeZOL2syiVVIAU8POwAG\nGVBroJoveFm/SUp6lCiN0M2kTeyQA2ax3QTtZSAa8nwrI7U52XOzVmdMicJsy2Pg\nuW98te3MuODdK24yNkHIkYameP/Umf/SJshUJQd5a/TUp3XE+HhOWAumx22tIDlC\nvZS11cuk2fp0WeHUnXaC19N5qWKfvHEKSugzty/z3lGP7ItFhrF2X1qJHeAAsL11\nkjo6Lc48KsE1vKvbnW4VLyB3wdNiVvmUNO29tPXwaR0Q5Gbr3jk3nUzdkEHouHWQ\n41lubOHCCBN3V13mh/MgtNhESHjfmmOnh54ErD9saA1d7CjTf8g2wqmjEqvGSW6N\nq7zhcWR2tp1olflS7oHzul4/I3hnkfL6Kb2xAWWaQKvg3mtsY2OPlzFEP0tR5UcH\nPfp5CeS1Xzg7hN6vRICW6m4l3u2HJFld2akDMm1vnSz8RCbPW7jp7YBxUkWJmypM\ntG7Yv2aGZXGbUtM8o1cZarECAwEAAQ==\n-----END PUBLIC KEY-----", + "key-id": "rsa-4096" + }, + "us-west-2-decryptable": { + "encrypt": true, + "decrypt": true, + "type": "aws-kms", + "key-id": "arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f" + }, + "us-west-2-encrypt-only": { + "encrypt": true, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws:kms:us-west-2:658956600833:key/590fd781-ddde-4036-abec-3e1ab5a5d2ad" + }, + "us-west-2-mrk": { + "encrypt": true, + "decrypt": true, + "type": "aws-kms", + "key-id": "arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "us-east-1-mrk": { + "encrypt": true, + "decrypt": true, + "type": "aws-kms", + "key-id": "arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "aws:kms:us-west-2:658956600833:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "aws:kms:us-west-2:658956600833:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + ":aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": ":aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "arn-not:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn-not:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "arn:kms:us-west-2:658956600833:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:kms:us-west-2:658956600833:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "arn::kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn::kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "arn:aws-not:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws-not:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "arn:aws:us-west-2:658956600833:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws:us-west-2:658956600833:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "arn:aws::us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws::us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "arn:aws:kms-not:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws:kms-not:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "arn:aws:kms:658956600833:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws:kms:658956600833:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "arn:aws:kms::658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws:kms::658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "arn:aws:kms:us-west-2:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws:kms:us-west-2:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "arn:aws:kms:us-west-2::key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws:kms:us-west-2::key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "arn:aws:kms:us-west-2:658956600833-not:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws:kms:us-west-2:658956600833-not:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "arn:aws:kms:us-west-2:658956600833:mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws:kms:us-west-2:658956600833:mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "arn:aws:kms:us-west-2:658956600833:/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws:kms:us-west-2:658956600833:/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "arn:aws:kms:us-west-2:658956600833:key-not/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws:kms:us-west-2:658956600833:key-not/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "arn:aws:kms:us-west-2:658956600833:key": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws:kms:us-west-2:658956600833:key" + }, + "arn:aws:kms:us-west-2:658956600833:key/": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws:kms:us-west-2:658956600833:key/" + }, + "arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7-not": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7-not" + }, + "arn:aws:kms:us-west-2:658956600833:alias/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws:kms:us-west-2:658956600833:alias/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + } + } +} From 15a69761bb737f89824b61fc461acbc7fbf4e1f4 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 14:24:56 -0800 Subject: [PATCH 176/376] install testvectors --- test_vector_handlers/requirements_mpl.txt | 1 + test_vector_handlers/tox.ini | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 test_vector_handlers/requirements_mpl.txt diff --git a/test_vector_handlers/requirements_mpl.txt b/test_vector_handlers/requirements_mpl.txt new file mode 100644 index 000000000..c7927a851 --- /dev/null +++ b/test_vector_handlers/requirements_mpl.txt @@ -0,0 +1 @@ +amazon-cryptographic-material-providers-test-vectors @ git+https://github.com/aws/aws-cryptographic-material-providers-library.git@lucmcdon/python-mpl#subdirectory=TestVectorsAwsCryptographicMaterialProviders/runtimes/python \ No newline at end of file diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index 0b18bf715..949efc89a 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -51,7 +51,7 @@ sitepackages = False deps = -rtest/requirements.txt # Install the MPL requirements if the `-mpl` suffix is present - mpl: -r../requirements_mpl.txt + mpl: -r../requirements_mpl.txt -rtest/requirements_mpl.txt .. commands = awses_local: {[testenv:base-command]commands} test/integration From 46ac8a0dbf77b24f710a221879701e02014576ba Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 14:27:37 -0800 Subject: [PATCH 177/376] fix --- test_vector_handlers/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index 949efc89a..c107d3527 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -51,7 +51,7 @@ sitepackages = False deps = -rtest/requirements.txt # Install the MPL requirements if the `-mpl` suffix is present - mpl: -r../requirements_mpl.txt -rtest/requirements_mpl.txt + mpl: -r../requirements_mpl.txt -rrequirements_mpl.txt .. commands = awses_local: {[testenv:base-command]commands} test/integration From 104ff8a341b4a14f59585027d2d5bef1b6aa8c4f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 14:33:32 -0800 Subject: [PATCH 178/376] fix --- test_vector_handlers/tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index c107d3527..9f86d448f 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -51,7 +51,8 @@ sitepackages = False deps = -rtest/requirements.txt # Install the MPL requirements if the `-mpl` suffix is present - mpl: -r../requirements_mpl.txt -rrequirements_mpl.txt + mpl: -r../requirements_mpl.txt + mpl: -rrequirements_mpl.txt .. commands = awses_local: {[testenv:base-command]commands} test/integration From 65227c2b2380cc78077d74ec2ca1978b1802bfa2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 14:40:20 -0800 Subject: [PATCH 179/376] fix --- test_vector_handlers/tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index 9f86d448f..c2ff913c2 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -51,7 +51,6 @@ sitepackages = False deps = -rtest/requirements.txt # Install the MPL requirements if the `-mpl` suffix is present - mpl: -r../requirements_mpl.txt mpl: -rrequirements_mpl.txt .. commands = From a2484a0271ca4cf1ef0b99c345279b3c73d1da0a Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 11 Mar 2024 16:01:25 -0700 Subject: [PATCH 180/376] working --- .../materials_managers/mpl/cmm.py | 13 ++++ .../manifests/full_message/decrypt.py | 2 +- .../full_message/decrypt_generation.py | 2 +- .../manifests/full_message/encrypt.py | 2 +- .../manifests/mpl_keyring.py | 58 ++++++++++++----- .../integration/integration_test_utils.py | 2 +- .../test_i_full_message_encrypt_keyrings.py | 62 ++++++++++++------- test_vector_handlers/tox.ini | 4 +- 8 files changed, 101 insertions(+), 44 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index ebef5f7ac..9efb8c1f0 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -68,6 +68,11 @@ def get_encryption_materials( request ) mpl_output: MPL_GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) + + print(f"{mpl_output.as_dict()=}") + + mpl_output.encryption_materials.encrypted_data_keys[0].key_provider_info = b"rsa-4096-private" + return EncryptionMaterialsFromMPL(mpl_output.encryption_materials) except AwsCryptographicMaterialProvidersException as mpl_exception: # Wrap MPL error into the ESDK error type @@ -118,15 +123,23 @@ def decrypt_materials( Returns a DecryptionMaterialsFromMPL for the configured CMM. :param request: Request for decryption materials """ + from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.errors import CollectionOfErrors as COE try: mpl_input: 'MPL_DecryptMaterialsInput' = \ CryptoMaterialsManagerFromMPL._create_mpl_decrypt_materials_input_from_request(request) + print(f"{mpl_input.as_dict()=}") + # input() mpl_output: 'MPL_DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(mpl_input) + print(f"{mpl_output.as_dict()=}") + # input() return DecryptionMaterialsFromMPL(mpl_output.decryption_materials) except AwsCryptographicMaterialProvidersException as mpl_exception: # Wrap MPL error into the ESDK error type # so customers only have to catch ESDK error types. raise AWSEncryptionSDKClientError(mpl_exception) + except COE as coe: + print(f"{coe.list=}") + raise AWSEncryptionSDKClientError(coe) @staticmethod def _native_algorithm_id_to_mpl_algorithm_id(native_algorithm_id: str) -> 'MPL_AlgorithmSuiteIdESDK': diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index 6d4a4a75d..0b8dfe3b8 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -276,7 +276,7 @@ def from_scenario( def master_key_provider_fn(): if keyrings: - return keyring_from_master_key_specs(keys_uri, master_key_specs) + return keyring_from_master_key_specs(keys, keys_uri, master_key_specs, "decrypt") return master_key_provider_from_master_key_specs(keys, master_key_specs) decryption_method_spec = scenario.get("decryption-method") diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 4bbb3df5d..3fce71e36 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -405,7 +405,7 @@ def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): def decryption_master_key_provider_fn(): if keyrings: - return keyring_from_master_key_specs(keys_uri, decryption_master_key_specs) + return keyring_from_master_key_specs(keys, keys_uri, decryption_master_key_specs, "decrypt-generation") return master_key_provider_from_master_key_specs(keys, decryption_master_key_specs) else: diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 82a5e379a..a3d351317 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -112,7 +112,7 @@ def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): def master_key_provider_fn(): if keyrings: - return keyring_from_master_key_specs(keys_uri, master_key_specs) + return keyring_from_master_key_specs(keys, keys_uri, master_key_specs, "encrypt") return master_key_provider_from_master_key_specs(keys, master_key_specs) return cls( diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index 820cd00c2..d64c323d0 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -22,19 +22,12 @@ GetKeyDescriptionOutput, TestVectorKeyringInput, ) -from aws_cryptography_materialproviderstestvectorkeys.smithygenerated.\ - aws_cryptography_materialproviderstestvectorkeys.client import ( - KeyVectors, - ) -from aws_cryptography_materialproviderstestvectorkeys.smithygenerated.\ - aws_cryptography_materialproviderstestvectorkeys.config import ( - KeyVectorsConfig - ) from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig from aws_cryptographic_materialproviders.mpl.references import IKeyring from aws_cryptographic_materialproviders.mpl.models import CreateMultiKeyringInput +from awses_test_vectors.internal.keyvectors_provider import KeyVectorsProvider from awses_test_vectors.manifests.keys import KeysManifest # noqa pylint disable=unused-import import json @@ -56,26 +49,55 @@ class KeyringSpec(MasterKeySpec): # pylint: disable=too-many-instance-attribute :param str padding_hash: Wrapping key padding hash (required for raw master keys) """ - def keyring(self, keys_uri): + def keyring(self, keys, keys_uri, mode): # type: (KeysManifest) -> IKeyring """Build a keyring using this specification. :param str keys_uri: Path to the keys manifest """ - keyvectors = KeyVectors(KeyVectorsConfig(key_manifest_path=keys_uri)) + ''' + encryptmaterials keyProviderInfo = rsa-4096-public' + MUST be private. + somehow, it is writing "rsa-4096-public". + + ''' + + print(f"{keys=}") + + keyvectors = KeyVectorsProvider.get_keyvectors(keys_path=keys_uri) # Construct the input to KeyVectorsConfig - input_as_dict = { + input_kwargs = { "type": self.type_name, "key": self.key_name, "provider-id": self.provider_id, "encryption-algorithm": self.encryption_algorithm, - "padding-algorithm": self.padding_algorithm, - "padding-hash": self.padding_hash + } + if self.padding_algorithm is not None and self.padding_algorithm is not "": + input_kwargs["padding-algorithm"] = self.padding_algorithm + if self.padding_hash is not None: + input_kwargs["padding-hash"] = self.padding_hash + + # Normalize input for MPL + if input_kwargs["type"] == "raw" \ + and input_kwargs["encryption-algorithm"] == "rsa": + if input_kwargs["key"] == "rsa-4096-private" \ + and (mode == "decrypt-generate" or mode == "encrypt"): + print(f"changed private to public") + input_kwargs["key"] = "rsa-4096-public" + # if input_kwargs["key"] == "rsa-4096-private" \ + # and (mode == "decrypt"): + # input_kwargs["provider-id"] = "rsa-4096-public" + if "padding-hash" not in input_kwargs: + print("added paddinghash") + input_kwargs["padding-hash"] = "sha1" + + print(f"keyring {input_kwargs=}") + # stringify the dict - input_as_string = json.dumps(input_as_dict) + input_as_string = json.dumps(input_kwargs) # convert to unicode code point (expected representation) encoded_json = [ord(c) for c in input_as_string] @@ -83,6 +105,10 @@ def keyring(self, keys_uri): GetKeyDescriptionInput(json=encoded_json) ) + print(f"{output.key_description.value=}") + + keyvectors + keyring: IKeyring = keyvectors.create_test_vector_keyring( TestVectorKeyringInput( key_description=output.key_description @@ -92,7 +118,7 @@ def keyring(self, keys_uri): return keyring -def keyring_from_master_key_specs(keys_uri, master_key_specs): +def keyring_from_master_key_specs(keys, keys_uri, master_key_specs, mode): # type: (str, list[KeyringSpec]) -> IKeyring """Build and combine all keyrings identified by the provided specs and using the provided keys. @@ -103,7 +129,7 @@ def keyring_from_master_key_specs(keys_uri, master_key_specs): :return: Master key provider combining all loaded master keys :rtype: IKeyring """ - keyrings = [spec.keyring(keys_uri) for spec in master_key_specs] + keyrings = [spec.keyring(keys, keys_uri, mode) for spec in master_key_specs] primary = keyrings[0] others = keyrings[1:] diff --git a/test_vector_handlers/test/integration/integration_test_utils.py b/test_vector_handlers/test/integration/integration_test_utils.py index fbe6cf7b7..25efe6b79 100644 --- a/test_vector_handlers/test/integration/integration_test_utils.py +++ b/test_vector_handlers/test/integration/integration_test_utils.py @@ -33,5 +33,5 @@ def full_message_encrypt_vectors(): @pytest.fixture def full_message_decrypt_generation_vectors(): return os.path.join( - vectors_dir(), "features", "CANONICAL-GENERATED-MANIFESTS", "0006-awses-message-decryption-generation.v2.json" + vectors_dir(), "features", "CANONICAL-GENERATED-MANIFESTS", "decrypt-generate-lite.json" ) diff --git a/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py b/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py index 56bf3112c..de5f10299 100644 --- a/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py +++ b/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py @@ -22,17 +22,20 @@ full_message_encrypt_vectors, ) +import cProfile + pytestmark = [pytest.mark.integ] -def test_full_message_encrypt_canonical_full(full_message_encrypt_vectors): - full_message_encrypt.cli(["--input", full_message_encrypt_vectors]) - full_message_encrypt.cli(["--input", full_message_encrypt_vectors], "--keyrings") +# def test_full_message_encrypt_canonical_full(full_message_encrypt_vectors): +# full_message_encrypt.cli(["--input", full_message_encrypt_vectors]) +# full_message_encrypt.cli(["--input", full_message_encrypt_vectors], "--keyrings") def test_full_message_cycle_canonical_full(tmpdir, full_message_decrypt_generation_vectors): # Generate vectors using keyring interfaces keyring_output_dir = tmpdir.join("output-keyrings") + print("Generating vectors with keyrings... ", end="") full_message_decrypt_generate.cli([ "--output", str(keyring_output_dir), @@ -40,23 +43,38 @@ def test_full_message_cycle_canonical_full(tmpdir, full_message_decrypt_generati full_message_decrypt_generation_vectors, "--keyrings" ]) - # Generate vectors using master key interfaces - master_key_output_dir = tmpdir.join("output-master-key") - full_message_decrypt_generate.cli([ - "--output", - str(master_key_output_dir), - "--input", - full_message_decrypt_generation_vectors - ]) + print("done") + + # print("Generating vectors with master keys... ", end="") + # # Generate vectors using master key interfaces + # master_key_output_dir = tmpdir.join("output-master-key") + # full_message_decrypt_generate.cli([ + # "--output", + # str(master_key_output_dir), + # "--input", + # full_message_decrypt_generation_vectors + # ]) + # print("done") + + # # Validate that vectors generated using keyring interfaces + # # can be decrypted by BOTH keyring and master key interfaces + # keyring_decrypt_manifest_file = keyring_output_dir.join("manifest.json") + # print("Decrypting keyring-encrypted vectors with keyrings... ", end="") + # full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file), "--keyrings"]) + # print("done") + + # print("Decrypting keyring-encrypted vectors with master keys... ", end="") + # full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file)]) + # print("done") + + # # Validate that vectors generated using master key interfaces + # # can be decrypted by BOTH keyring and master key interfaces + # master_key_decrypt_manifest_file = keyring_output_dir.join("manifest.json") + + # print("Decrypting master key-encrypted vectors with keyrings... ", end="") + # full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file), "--keyrings"]) + # print("done") - # Validate that vectors generated using keyring interfaces - # can be decrypted by BOTH keyring and master key interfaces - keyring_decrypt_manifest_file = keyring_output_dir.join("manifest.json") - full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file), "--keyrings"]) - full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file)]) - - # Validate that vectors generated using master key interfaces - # can be decrypted by BOTH keyring and master key interfaces - master_key_decrypt_manifest_file = keyring_output_dir.join("manifest.json") - full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file), "--keyrings"]) - full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file)]) + # print("Decrypting master key-encrypted vectors with master keys... ", end="") + # full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file)]) + # print("done") diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index c2ff913c2..18b3710e5 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -37,7 +37,7 @@ envlist = # release :: Builds dist files and uploads to pypi pypirc profile. [testenv:base-command] -commands = pytest --basetemp={envtmpdir} -l --cov awses_test_vectors {posargs} +commands = python3 -m cProfile -o profile.txt -m pytest --basetemp={envtmpdir} -l --cov awses_test_vectors {posargs} [testenv] passenv = @@ -55,7 +55,7 @@ deps = .. commands = awses_local: {[testenv:base-command]commands} test/integration - mplvectors: {[testenv:base-command]commands} test/keyrings + mplvectors: {[testenv:base-command]commands} test/keyrings -s -v [testenv:full-encrypt] basepython = python3 From faa92a0e1df93a0c749bd8c2d2d1db273cccf264 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 12 Mar 2024 11:56:55 -0700 Subject: [PATCH 181/376] decrypt-lite workign --- .../materials_managers/mpl/cmm.py | 20 +++++++++++++--- .../materials_managers/mpl/materials.py | 2 ++ .../full_message/decrypt_generation.py | 2 +- .../manifests/mpl_keyring.py | 23 +++++++++++++++++++ 4 files changed, 43 insertions(+), 4 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index 9efb8c1f0..1655e0746 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -67,11 +67,25 @@ def get_encryption_materials( CryptoMaterialsManagerFromMPL._native_to_mpl_get_encryption_materials( request ) + mpl_output: MPL_GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) - print(f"{mpl_output.as_dict()=}") + print(f"get {mpl_output=}") + + # ???????????????????????????? + # kpis = set() + # for edk in mpl_output.encryption_materials.encrypted_data_keys: + # kpis.add(edk.key_provider_info) + + # print(kpis) + # input + + # if len(kpis) == 1: + # for edk in mpl_output.encryption_materials.encrypted_data_keys: + # if edk.key_provider_info == b"rsa-4096-public": + # edk.key_provider_info = b"rsa-4096-private" - mpl_output.encryption_materials.encrypted_data_keys[0].key_provider_info = b"rsa-4096-private" + # mpl_output.encryption_materials.encrypted_data_keys[0].key_provider_info = b"rsa-4096-private" return EncryptionMaterialsFromMPL(mpl_output.encryption_materials) except AwsCryptographicMaterialProvidersException as mpl_exception: @@ -139,7 +153,7 @@ def decrypt_materials( raise AWSEncryptionSDKClientError(mpl_exception) except COE as coe: print(f"{coe.list=}") - raise AWSEncryptionSDKClientError(coe) + # raise AWSEncryptionSDKClientError(coe) @staticmethod def _native_algorithm_id_to_mpl_algorithm_id(native_algorithm_id: str) -> 'MPL_AlgorithmSuiteIdESDK': diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index 54ea21b39..9f3d4f0fb 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -75,6 +75,8 @@ def encrypted_data_keys(self) -> List[Native_EncryptedDataKey]: ), encrypted_data_key=mpl_edk.ciphertext, ) for mpl_edk in mpl_edk_list} + print(f"{key_blob_list=}") + # input() return key_blob_list @property diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 3fce71e36..5536fc845 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -484,7 +484,7 @@ def _generate_plaintexts(plaintexts_specs): :return: Mapping of plaintext name to randomly generated bytes :rtype: dict """ - return {name: os.urandom(size) for name, size in plaintexts_specs.items()} + return {name: b"a" * size for name, size in plaintexts_specs.items()} @classmethod def from_file(cls, input_file, keyrings): diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index d64c323d0..c9e834439 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -67,6 +67,8 @@ def keyring(self, keys, keys_uri, mode): keyvectors = KeyVectorsProvider.get_keyvectors(keys_path=keys_uri) + changed = False + # Construct the input to KeyVectorsConfig input_kwargs = { "type": self.type_name, @@ -86,6 +88,7 @@ def keyring(self, keys, keys_uri, mode): if input_kwargs["key"] == "rsa-4096-private" \ and (mode == "decrypt-generate" or mode == "encrypt"): print(f"changed private to public") + changed = True input_kwargs["key"] = "rsa-4096-public" # if input_kwargs["key"] == "rsa-4096-private" \ # and (mode == "decrypt"): @@ -115,6 +118,20 @@ def keyring(self, keys, keys_uri, mode): ) ) + import _dafny + import UTF8 + + if hasattr(keyring, "_impl"): + if hasattr(keyring._impl, "_keyName"): + if keyring._impl._keyName == UTF8.default__.Encode(_dafny.Seq("rsa-4096-public")).value \ + and (mode == "decrypt-generate" or mode == "encrypt"): + if changed: + print("YES") + # input() + print(f"changed public to private") + keyring._impl._keyName = UTF8.default__.Encode(_dafny.Seq("rsa-4096-private")).value + + return keyring @@ -129,7 +146,13 @@ def keyring_from_master_key_specs(keys, keys_uri, master_key_specs, mode): :return: Master key provider combining all loaded master keys :rtype: IKeyring """ + # print(f"{master_key_specs=}") + # input() keyrings = [spec.keyring(keys, keys_uri, mode) for spec in master_key_specs] + # print(f"speckeyrings {keyrings=}") + # input() + # print(f"speckeys {keys=}") + # input() primary = keyrings[0] others = keyrings[1:] From bbc36f9cf9a271d958c5ff69f7b34e4773e9c10c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 12 Mar 2024 12:20:42 -0700 Subject: [PATCH 182/376] ..? --- .../manifests/mpl_keyring.py | 56 ++++++++++++------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index c9e834439..0f63ca143 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -82,23 +82,23 @@ def keyring(self, keys, keys_uri, mode): if self.padding_hash is not None: input_kwargs["padding-hash"] = self.padding_hash - # Normalize input for MPL if input_kwargs["type"] == "raw" \ and input_kwargs["encryption-algorithm"] == "rsa": + # Weird hack #1. + # If generating decrypt vectors (i.e. encrypting) + # and the manifest specified an RSA private key, + # change the input to KeyVectors to a public key. + # KeyVectors requires a public key to encrypt. + # If this is not done, then keyring.OnEncrypt fails with + # "A RawRSAKeyring without a public key cannot provide OnEncrypt" if input_kwargs["key"] == "rsa-4096-private" \ and (mode == "decrypt-generate" or mode == "encrypt"): - print(f"changed private to public") changed = True input_kwargs["key"] = "rsa-4096-public" - # if input_kwargs["key"] == "rsa-4096-private" \ - # and (mode == "decrypt"): - # input_kwargs["provider-id"] = "rsa-4096-public" + # Specify default padding-hash if "padding-hash" not in input_kwargs: - print("added paddinghash") input_kwargs["padding-hash"] = "sha1" - print(f"keyring {input_kwargs=}") - # stringify the dict input_as_string = json.dumps(input_kwargs) # convert to unicode code point (expected representation) @@ -108,16 +108,39 @@ def keyring(self, keys, keys_uri, mode): GetKeyDescriptionInput(json=encoded_json) ) - print(f"{output.key_description.value=}") - - keyvectors - keyring: IKeyring = keyvectors.create_test_vector_keyring( TestVectorKeyringInput( key_description=output.key_description ) ) + # Weird hack #2. + # Generating decrypt vectors for RSA keys. + # The MPL sets the encrypting keyring's keyName to "rsa-4096-private", + # somewhat undoing weird hack #1. + # Weird hack #1 allows the encrypting keyring to be created with a public key. + # However, it also changes the keyName of the encrypting keyring, + # which is changed back with this hack. + # If this is not done, then decryption fails + # (for BOTH native master keys and MPL keyrings) + # with error + # native master keys: "Unable to decrypt any data key" + # MPL: "Raw RSA Key was unable to decrypt any encrypted data key" + # + # digging, they key is unable to decrypt + # because the EDK keyProviderInfo differs from the keyring keyName, + # and this check fails: + # https://github.com/aws/aws-cryptographic-material-providers-library/blob/bd549c88cefc93ba8a2d204bd23134b3b12c69fb/AwsCryptographicMaterialProviders/dafny/AwsCryptographicMaterialProviders/src/Keyrings/RawRSAKeyring.dfy#L382 + # due to the two variables not being equal: + # edk.keyProviderInfo='rsa-4096-public' + # decrypting keyring.keyName='rsa-4096-private' + # + # changing the encrypting keyring's keyName back to 'rsa-4096-private' + # (somewhat undoing weird hack #1) + # sets edk.keyProviderInfo='rsa-4096-private', + # which allows this check to pass on decrypt. + # This "works" because all of the test vectors pass with these two hacks. + # But this seems weird. import _dafny import UTF8 @@ -126,9 +149,6 @@ def keyring(self, keys, keys_uri, mode): if keyring._impl._keyName == UTF8.default__.Encode(_dafny.Seq("rsa-4096-public")).value \ and (mode == "decrypt-generate" or mode == "encrypt"): if changed: - print("YES") - # input() - print(f"changed public to private") keyring._impl._keyName = UTF8.default__.Encode(_dafny.Seq("rsa-4096-private")).value @@ -146,13 +166,7 @@ def keyring_from_master_key_specs(keys, keys_uri, master_key_specs, mode): :return: Master key provider combining all loaded master keys :rtype: IKeyring """ - # print(f"{master_key_specs=}") - # input() keyrings = [spec.keyring(keys, keys_uri, mode) for spec in master_key_specs] - # print(f"speckeyrings {keyrings=}") - # input() - # print(f"speckeys {keys=}") - # input() primary = keyrings[0] others = keyrings[1:] From 3bf820cd0e85aa90da2feb7338d94a5403f2b657 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 12 Mar 2024 13:39:42 -0700 Subject: [PATCH 183/376] missing --- .../internal/keyvectors_provider.py | 25 +++++++++++++++++++ .../integration/integration_test_utils.py | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py diff --git a/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py b/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py new file mode 100644 index 000000000..12dc980e3 --- /dev/null +++ b/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py @@ -0,0 +1,25 @@ +from aws_cryptography_materialproviderstestvectorkeys.smithygenerated.\ + aws_cryptography_materialproviderstestvectorkeys.client import ( + KeyVectors, + ) +from aws_cryptography_materialproviderstestvectorkeys.smithygenerated.\ + aws_cryptography_materialproviderstestvectorkeys.config import ( + KeyVectorsConfig + ) + +keyvectors_instances = {} + +class KeyVectorsProvider: + """Singleton manager for the KeyVectors client. + + This is used because Dafny's JSON deserializer implementation is slow with large files. + It deserializes the file at keys_path and takes >1 minute to do this. + """ + + instance: KeyVectors + + @classmethod + def get_keyvectors(self, keys_path): + if not keys_path in keyvectors_instances: + keyvectors_instances[keys_path] = KeyVectors(KeyVectorsConfig(key_manifest_path=keys_path)) + return keyvectors_instances[keys_path] diff --git a/test_vector_handlers/test/integration/integration_test_utils.py b/test_vector_handlers/test/integration/integration_test_utils.py index 25efe6b79..fbe6cf7b7 100644 --- a/test_vector_handlers/test/integration/integration_test_utils.py +++ b/test_vector_handlers/test/integration/integration_test_utils.py @@ -33,5 +33,5 @@ def full_message_encrypt_vectors(): @pytest.fixture def full_message_decrypt_generation_vectors(): return os.path.join( - vectors_dir(), "features", "CANONICAL-GENERATED-MANIFESTS", "decrypt-generate-lite.json" + vectors_dir(), "features", "CANONICAL-GENERATED-MANIFESTS", "0006-awses-message-decryption-generation.v2.json" ) From eb40abb61fb16f1442e21fc620437f1f9b001c4a Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 12 Mar 2024 14:07:35 -0700 Subject: [PATCH 184/376] cleanup --- .../manifests/mpl_keyring.py | 35 +++++++++++-------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index 0f63ca143..d420a3be7 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -84,7 +84,9 @@ def keyring(self, keys, keys_uri, mode): if input_kwargs["type"] == "raw" \ and input_kwargs["encryption-algorithm"] == "rsa": - # Weird hack #1. + # Weird hack #1: + # Gets public key for encryption instead of private key. + # # If generating decrypt vectors (i.e. encrypting) # and the manifest specified an RSA private key, # change the input to KeyVectors to a public key. @@ -114,33 +116,36 @@ def keyring(self, keys, keys_uri, mode): ) ) - # Weird hack #2. - # Generating decrypt vectors for RSA keys. - # The MPL sets the encrypting keyring's keyName to "rsa-4096-private", - # somewhat undoing weird hack #1. + # Weird hack #2: + # Sets keyProviderInfo to "private" even though the material is "public". + # # Weird hack #1 allows the encrypting keyring to be created with a public key. - # However, it also changes the keyName of the encrypting keyring, - # which is changed back with this hack. + # However, it also changes the keyName of the encrypting keyring. + # This hack changes it back. + # # If this is not done, then decryption fails # (for BOTH native master keys and MPL keyrings) # with error # native master keys: "Unable to decrypt any data key" # MPL: "Raw RSA Key was unable to decrypt any encrypted data key" # - # digging, they key is unable to decrypt + # Digging, the keyring is unable to decrypt in the MPL # because the EDK keyProviderInfo differs from the keyring keyName, # and this check fails: # https://github.com/aws/aws-cryptographic-material-providers-library/blob/bd549c88cefc93ba8a2d204bd23134b3b12c69fb/AwsCryptographicMaterialProviders/dafny/AwsCryptographicMaterialProviders/src/Keyrings/RawRSAKeyring.dfy#L382 # due to the two variables not being equal: # edk.keyProviderInfo='rsa-4096-public' - # decrypting keyring.keyName='rsa-4096-private' + # keyring.keyName='rsa-4096-private' + # + # Changing the encrypting keyring's keyName back to 'rsa-4096-private' + # sets any EDKs this keyring encrypts to now have + # keyName="rsa-4096-private". + # However, keyvectors has still retrieved the public key material to encrypt with. + # So it any EDKs it encrypts will use the public material, but have keyName="rsa-4096-private". # - # changing the encrypting keyring's keyName back to 'rsa-4096-private' - # (somewhat undoing weird hack #1) - # sets edk.keyProviderInfo='rsa-4096-private', - # which allows this check to pass on decrypt. - # This "works" because all of the test vectors pass with these two hacks. - # But this seems weird. + # This configuration seems to be correct, because + # all of the test vectors (master keys and MPL) pass with these two hacks. + # But this seems weird, and we didn't have to do this in Java. import _dafny import UTF8 From bc0d5ff1e78e6af426826abe92e0ec144c597d66 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 12 Mar 2024 18:11:16 -0700 Subject: [PATCH 185/376] wip --- .../manifests/full_message/decrypt.py | 2 +- .../manifests/mpl_keyring.py | 1 - .../integration/integration_test_utils.py | 21 +++++++++++++++---- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index 0b8dfe3b8..752315ee6 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -56,7 +56,7 @@ CLIENT_NAME = "aws/aws-encryption-sdk-python" CURRENT_VERSION = 2 -SUPPORTED_VERSIONS = (2,) +SUPPORTED_VERSIONS = (2,4,) @attr.s(init=False) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index d420a3be7..7fd0f3323 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -156,7 +156,6 @@ def keyring(self, keys, keys_uri, mode): if changed: keyring._impl._keyName = UTF8.default__.Encode(_dafny.Seq("rsa-4096-private")).value - return keyring diff --git a/test_vector_handlers/test/integration/integration_test_utils.py b/test_vector_handlers/test/integration/integration_test_utils.py index fbe6cf7b7..4fee77378 100644 --- a/test_vector_handlers/test/integration/integration_test_utils.py +++ b/test_vector_handlers/test/integration/integration_test_utils.py @@ -18,20 +18,33 @@ import pytest -def vectors_dir(): - here = os.path.abspath(os.path.dirname(__file__)) +here = os.path.abspath(os.path.dirname(__file__)) + + +def legacy_vectors_dir(): return os.path.abspath(os.path.join(here, "..", "aws-crypto-tools-test-vector-framework")) +def mpl_vectors_dir(): + return os.path.abspath(os.path.join(here, "..", "golden-manifest-TODORENAMEANDGETFROMGHA")) + + @pytest.fixture def full_message_encrypt_vectors(): return os.path.join( - vectors_dir(), "features", "CANONICAL-GENERATED-MANIFESTS", "0003-awses-message-encryption.v2.json" + legacy_vectors_dir(), "features", "CANONICAL-GENERATED-MANIFESTS", "0003-awses-message-encryption.v2.json" + ) + + +@pytest.fixture +def full_message_decrypt_generation_vectors(): + return os.path.join( + legacy_vectors_dir(), "features", "CANONICAL-GENERATED-MANIFESTS", "0006-awses-message-decryption-generation.v2.json" ) @pytest.fixture def full_message_decrypt_generation_vectors(): return os.path.join( - vectors_dir(), "features", "CANONICAL-GENERATED-MANIFESTS", "0006-awses-message-decryption-generation.v2.json" + mpl_vectors_dir(), "manifest.json" ) From fb7d10c7cb547c401d2107900ad47ddff9024b22 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 11:11:04 -0700 Subject: [PATCH 186/376] requiredec working --- .../manifests/full_message/decrypt.py | 132 ++++++++++++++++-- 1 file changed, 124 insertions(+), 8 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index 752315ee6..1c04a83a2 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -200,6 +200,9 @@ class MessageDecryptionTestScenario(object): :type master_key_specs: iterable of :class:`MasterKeySpec` :param Callable master_key_provider_fn: :param bool keyrings: True if should decrypt with keyring interfaces; False otherwise + :param str cmm_type: `cmm` from test vector manifest; "Default" if not specified + :param str encryption_context: Any encryption context to validate on decrypt if using + keyrings AND the required encryption context CMM :param str description: Description of test scenario (optional) """ @@ -211,6 +214,7 @@ class MessageDecryptionTestScenario(object): master_key_provider_fn = attr.ib(validator=attr.validators.is_callable()) result = attr.ib(validator=attr.validators.instance_of(MessageDecryptionTestResult)) keyrings = attr.ib(validator=attr.validators.instance_of(bool)) + cmm_type = attr.ib(validator=attr.validators.instance_of(str)) decryption_method = attr.ib( default=None, validator=attr.validators.optional(attr.validators.instance_of(DecryptionMethod)) ) @@ -226,6 +230,8 @@ def __init__( master_key_specs, # type: Iterable[MasterKeySpec] master_key_provider_fn, # type: Callable keyrings, # type: bool + cmm_type, # type: str + encryption_context, # type: Dict[str, str] decryption_method=None, # type: Optional[DecryptionMethod] description=None, # type: Optional[str] ): # noqa=D107 @@ -239,9 +245,11 @@ def __init__( self.result = result self.master_key_specs = master_key_specs self.master_key_provider_fn = master_key_provider_fn + self.keyrings = keyrings + self.cmm_type = cmm_type + self.encryption_context = encryption_context self.decryption_method = decryption_method self.description = description - self.keyrings = keyrings attr.validate(self) @classmethod @@ -284,6 +292,26 @@ def master_key_provider_fn(): result_spec = scenario["result"] result = MessageDecryptionTestResult.from_result_spec(result_spec, plaintext_reader) + encryption_context = scenario["encryption-context"] + + # MPL test vectors add CMM types to the test vectors manifests + if "cmm" in scenario: + if scenario["cmm"] == "Default": + # Master keys and keyrings can handle default CMM + cmm_type = scenario["cmm"] + elif scenario["cmm"] == "RequiredEncryptionContext": + # Skip RequiredEncryptionContext CMM for master keys; + # This is unsupported for master keys + if keyrings: + cmm_type = scenario["cmm"] + else: + return None + else: + raise ValueError("Unrecognized cmm_type: " + cmm_type) + else: + # If unspecified, set "Default" as the default + cmm_type = "Default" + return cls( ciphertext_uri=scenario["ciphertext"], ciphertext=ciphertext_reader(scenario["ciphertext"]), @@ -291,6 +319,8 @@ def master_key_provider_fn(): master_key_provider_fn=master_key_provider_fn, result=result, keyrings=keyrings, + encryption_context=encryption_context, + cmm_type=cmm_type, decryption_method=decryption_method, description=scenario.get("description"), ) @@ -316,9 +346,50 @@ def scenario_spec(self): def _one_shot_decrypt(self): client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - if self.keyrings: - return client.decrypt(source=self.ciphertext, keyring=self.master_key_provider_fn()) - return client.decrypt(source=self.ciphertext, key_provider=self.master_key_provider_fn()) + if self.cmm_type == "Default": + if self.keyrings: + return client.decrypt(source=self.ciphertext, keyring=self.master_key_provider_fn()) + return client.decrypt(source=self.ciphertext, key_provider=self.master_key_provider_fn()) + elif self.cmm_type == "RequiredEncryptionContext": + # We need to make a custom CMM and pass it into the client + assert self.keyrings + + from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders + from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig + from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager + from aws_cryptographic_materialproviders.mpl.models import ( + CreateDefaultCryptographicMaterialsManagerInput, + CreateRequiredEncryptionContextCMMInput, + ) + + + mpl: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + config=MaterialProvidersConfig() + ) + + underlying_cmm: ICryptographicMaterialsManager = \ + mpl.create_default_cryptographic_materials_manager( + CreateDefaultCryptographicMaterialsManagerInput( + keyring=self.master_key_provider_fn() + ) + ) + + required_ec_cmm: ICryptographicMaterialsManager = \ + mpl.create_required_encryption_context_cmm( + CreateRequiredEncryptionContextCMMInput( + # Currently, the test vector manifest requires that + # if using the required encryption context CMM, + # both and only "key1" and "key2" are required. + required_encryption_context_keys=["key1", "key2"], + underlying_cmm=underlying_cmm, + ) + ) + + return client.decrypt( + source=self.ciphertext, + materials_manager=required_ec_cmm, + encryption_context = self.encryption_context, + ) def _streaming_decrypt(self): result = bytearray() @@ -328,10 +399,48 @@ def _streaming_decrypt(self): "source": self.ciphertext, "mode": "d" } - if self.keyrings: - kwargs["keyring"] = self.master_key_provider_fn() - else: - kwargs["key_provider"] = self.master_key_provider_fn() + if self.cmm_type == "Default": + if self.keyrings: + kwargs["keyring"] = self.master_key_provider_fn() + else: + kwargs["key_provider"] = self.master_key_provider_fn() + elif self.cmm_type == "RequiredEncryptionContext": + # We need to make a custom CMM and pass it into the client + assert self.keyrings + + from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders + from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig + from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager + from aws_cryptographic_materialproviders.mpl.models import ( + CreateDefaultCryptographicMaterialsManagerInput, + CreateRequiredEncryptionContextCMMInput, + ) + + + mpl: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + config=MaterialProvidersConfig() + ) + + underlying_cmm: ICryptographicMaterialsManager = \ + mpl.create_default_cryptographic_materials_manager( + CreateDefaultCryptographicMaterialsManagerInput( + keyring=self.master_key_provider_fn() + ) + ) + + required_ec_cmm: ICryptographicMaterialsManager = \ + mpl.create_required_encryption_context_cmm( + CreateRequiredEncryptionContextCMMInput( + # Currently, the test vector manifest requires that + # if using the required encryption context CMM, + # both and only "key1" and "key2" are required. + required_encryption_context_keys=["key1", "key2"], + underlying_cmm=underlying_cmm, + ) + ) + + kwargs["materials_manager"] = required_ec_cmm + kwargs["encryption_context"] = self.encryption_context with client.stream(**kwargs) as decryptor: for chunk in decryptor: @@ -483,6 +592,13 @@ def from_file(cls, input_file, keyrings): # Merge keyring scenarios into test_scenarios test_scenarios = {**keyrings_test_scenarios, **test_scenarios} + # Remove any `None` scenarios from test scenarios. + # `None` scenarios indicate the loader determined the scenario is invalid. + # e.g. cmm_type = "RequiredEncryptionContext" with master keys + for name in list(test_scenarios.keys()): + if test_scenarios[name] is None: + del test_scenarios[name] + return cls( keys_uri=keys_uri, keys=keys, From 344824b1250b73f193527065b93121fb13b31645 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 11:45:08 -0700 Subject: [PATCH 187/376] debug cb --- .../src/awses_test_vectors/manifests/mpl_keyring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index 7fd0f3323..8b3fa98d8 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -77,7 +77,7 @@ def keyring(self, keys, keys_uri, mode): "encryption-algorithm": self.encryption_algorithm, } - if self.padding_algorithm is not None and self.padding_algorithm is not "": + if self.padding_algorithm is not None and self.padding_algorithm != "": input_kwargs["padding-algorithm"] = self.padding_algorithm if self.padding_hash is not None: input_kwargs["padding-hash"] = self.padding_hash From 697f2ffbb74d7dfb18b508f299fc261952b3d7ac Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 12:06:31 -0700 Subject: [PATCH 188/376] fix cb --- .../full_message/decrypt_generation.py | 2 ++ .../integration/integration_test_utils.py | 13 +++++++++++- .../test_i_full_message_encrypt_keyrings.py | 20 +++++++++---------- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 5536fc845..a048dcc32 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -451,6 +451,8 @@ def decryption_test_scenario_pair(self, ciphertext_writer, ciphertext_to_decrypt decryption_method=self.decryption_method, result=expected_result, keyrings=self.keyrings, + cmm_type="Default", + encryption_context={} ), ) diff --git a/test_vector_handlers/test/integration/integration_test_utils.py b/test_vector_handlers/test/integration/integration_test_utils.py index 4fee77378..696dc8648 100644 --- a/test_vector_handlers/test/integration/integration_test_utils.py +++ b/test_vector_handlers/test/integration/integration_test_utils.py @@ -29,6 +29,10 @@ def mpl_vectors_dir(): return os.path.abspath(os.path.join(here, "..", "golden-manifest-TODORENAMEANDGETFROMGHA")) +def required_ec_vectors_dir(): + return os.path.abspath(os.path.join(here, "..", "required-ec-TODORENAMEANDGETFROMGHA")) + + @pytest.fixture def full_message_encrypt_vectors(): return os.path.join( @@ -44,7 +48,14 @@ def full_message_decrypt_generation_vectors(): @pytest.fixture -def full_message_decrypt_generation_vectors(): +def mpl_decrypt_vectors(): return os.path.join( mpl_vectors_dir(), "manifest.json" ) + + +@pytest.fixture +def required_encryption_context_cmm_decrypt_vectors(): + return os.path.join( + required_ec_vectors_dir(), "manifest.json" + ) \ No newline at end of file diff --git a/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py b/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py index de5f10299..37c33e417 100644 --- a/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py +++ b/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py @@ -34,16 +34,16 @@ def test_full_message_cycle_canonical_full(tmpdir, full_message_decrypt_generation_vectors): # Generate vectors using keyring interfaces - keyring_output_dir = tmpdir.join("output-keyrings") - print("Generating vectors with keyrings... ", end="") - full_message_decrypt_generate.cli([ - "--output", - str(keyring_output_dir), - "--input", - full_message_decrypt_generation_vectors, - "--keyrings" - ]) - print("done") + # keyring_output_dir = tmpdir.join("output-keyrings") + # print("Generating vectors with keyrings... ", end="") + # full_message_decrypt_generate.cli([ + # "--output", + # str(keyring_output_dir), + # "--input", + # full_message_decrypt_generation_vectors, + # "--keyrings" + # ]) + # print("done") # print("Generating vectors with master keys... ", end="") # # Generate vectors using master key interfaces From 7dbc00ad796ab4239761542d11c5adce470df5f2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 12:12:20 -0700 Subject: [PATCH 189/376] fix cb --- .../test_i_full_message_encrypt_keyrings.py | 82 +++++++++---------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py b/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py index 37c33e417..c86a23ab9 100644 --- a/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py +++ b/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py @@ -27,54 +27,54 @@ pytestmark = [pytest.mark.integ] -# def test_full_message_encrypt_canonical_full(full_message_encrypt_vectors): -# full_message_encrypt.cli(["--input", full_message_encrypt_vectors]) -# full_message_encrypt.cli(["--input", full_message_encrypt_vectors], "--keyrings") +def test_full_message_encrypt_canonical_full(full_message_encrypt_vectors): + full_message_encrypt.cli(["--input", full_message_encrypt_vectors]) + full_message_encrypt.cli(["--input", full_message_encrypt_vectors], "--keyrings") def test_full_message_cycle_canonical_full(tmpdir, full_message_decrypt_generation_vectors): # Generate vectors using keyring interfaces - # keyring_output_dir = tmpdir.join("output-keyrings") - # print("Generating vectors with keyrings... ", end="") - # full_message_decrypt_generate.cli([ - # "--output", - # str(keyring_output_dir), - # "--input", - # full_message_decrypt_generation_vectors, - # "--keyrings" - # ]) - # print("done") + keyring_output_dir = tmpdir.join("output-keyrings") + print("Generating vectors with keyrings... ", end="") + full_message_decrypt_generate.cli([ + "--output", + str(keyring_output_dir), + "--input", + full_message_decrypt_generation_vectors, + "--keyrings" + ]) + print("done") - # print("Generating vectors with master keys... ", end="") - # # Generate vectors using master key interfaces - # master_key_output_dir = tmpdir.join("output-master-key") - # full_message_decrypt_generate.cli([ - # "--output", - # str(master_key_output_dir), - # "--input", - # full_message_decrypt_generation_vectors - # ]) - # print("done") + print("Generating vectors with master keys... ", end="") + # Generate vectors using master key interfaces + master_key_output_dir = tmpdir.join("output-master-key") + full_message_decrypt_generate.cli([ + "--output", + str(master_key_output_dir), + "--input", + full_message_decrypt_generation_vectors + ]) + print("done") - # # Validate that vectors generated using keyring interfaces - # # can be decrypted by BOTH keyring and master key interfaces - # keyring_decrypt_manifest_file = keyring_output_dir.join("manifest.json") - # print("Decrypting keyring-encrypted vectors with keyrings... ", end="") - # full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file), "--keyrings"]) - # print("done") + # Validate that vectors generated using keyring interfaces + # can be decrypted by BOTH keyring and master key interfaces + keyring_decrypt_manifest_file = keyring_output_dir.join("manifest.json") + print("Decrypting keyring-encrypted vectors with keyrings... ", end="") + full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file), "--keyrings"]) + print("done") - # print("Decrypting keyring-encrypted vectors with master keys... ", end="") - # full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file)]) - # print("done") + print("Decrypting keyring-encrypted vectors with master keys... ", end="") + full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file)]) + print("done") - # # Validate that vectors generated using master key interfaces - # # can be decrypted by BOTH keyring and master key interfaces - # master_key_decrypt_manifest_file = keyring_output_dir.join("manifest.json") + # Validate that vectors generated using master key interfaces + # can be decrypted by BOTH keyring and master key interfaces + master_key_decrypt_manifest_file = keyring_output_dir.join("manifest.json") - # print("Decrypting master key-encrypted vectors with keyrings... ", end="") - # full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file), "--keyrings"]) - # print("done") + print("Decrypting master key-encrypted vectors with keyrings... ", end="") + full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file), "--keyrings"]) + print("done") - # print("Decrypting master key-encrypted vectors with master keys... ", end="") - # full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file)]) - # print("done") + print("Decrypting master key-encrypted vectors with master keys... ", end="") + full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file)]) + print("done") From 7036337ce16b1017bff3d54e4f8bbf40ab92e9d8 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 12:41:00 -0700 Subject: [PATCH 190/376] debug cb --- .../integration/commands/test_i_full_message_encrypt_keyrings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py b/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py index c86a23ab9..6d037a586 100644 --- a/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py +++ b/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py @@ -22,7 +22,6 @@ full_message_encrypt_vectors, ) -import cProfile pytestmark = [pytest.mark.integ] From 7a44191d33f95f2f9b7e6c0fddd89a134c03b228 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 12:46:27 -0700 Subject: [PATCH 191/376] debug gha --- src/aws_encryption_sdk/materials_managers/mpl/cmm.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index 1655e0746..c4a1a1fb6 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -70,8 +70,6 @@ def get_encryption_materials( mpl_output: MPL_GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) - print(f"get {mpl_output=}") - # ???????????????????????????? # kpis = set() # for edk in mpl_output.encryption_materials.encrypted_data_keys: From 298235ab8ce31b0ac8a7e26af56f18b3f424fe7c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 12:50:01 -0700 Subject: [PATCH 192/376] temp rm cov --- test_vector_handlers/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index 18b3710e5..5c96def2b 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -37,7 +37,7 @@ envlist = # release :: Builds dist files and uploads to pypi pypirc profile. [testenv:base-command] -commands = python3 -m cProfile -o profile.txt -m pytest --basetemp={envtmpdir} -l --cov awses_test_vectors {posargs} +commands = python3 -m cProfile -o profile.txt -m pytest --basetemp={envtmpdir} -l {posargs} [testenv] passenv = From 4c1d0a06e4e4e5ccf887f63d66c579cdc2edcde0 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 12:51:10 -0700 Subject: [PATCH 193/376] temp rm cov --- src/aws_encryption_sdk/materials_managers/mpl/cmm.py | 7 ++++--- src/aws_encryption_sdk/materials_managers/mpl/materials.py | 2 +- .../src/awses_test_vectors/manifests/mpl_keyring.py | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index c4a1a1fb6..8edf52151 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -139,10 +139,10 @@ def decrypt_materials( try: mpl_input: 'MPL_DecryptMaterialsInput' = \ CryptoMaterialsManagerFromMPL._create_mpl_decrypt_materials_input_from_request(request) - print(f"{mpl_input.as_dict()=}") + # print(f"{mpl_input.as_dict()=}") # input() mpl_output: 'MPL_DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(mpl_input) - print(f"{mpl_output.as_dict()=}") + # print(f"{mpl_output.as_dict()=}") # input() return DecryptionMaterialsFromMPL(mpl_output.decryption_materials) except AwsCryptographicMaterialProvidersException as mpl_exception: @@ -150,7 +150,8 @@ def decrypt_materials( # so customers only have to catch ESDK error types. raise AWSEncryptionSDKClientError(mpl_exception) except COE as coe: - print(f"{coe.list=}") + # print(f"{coe.list=}") + pass # raise AWSEncryptionSDKClientError(coe) @staticmethod diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index 9f3d4f0fb..b70e48efe 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -75,7 +75,7 @@ def encrypted_data_keys(self) -> List[Native_EncryptedDataKey]: ), encrypted_data_key=mpl_edk.ciphertext, ) for mpl_edk in mpl_edk_list} - print(f"{key_blob_list=}") + # print(f"{key_blob_list=}") # input() return key_blob_list diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index 8b3fa98d8..40421e931 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -63,7 +63,7 @@ def keyring(self, keys, keys_uri, mode): ''' - print(f"{keys=}") + # print(f"{keys=}") keyvectors = KeyVectorsProvider.get_keyvectors(keys_path=keys_uri) From 12e00605b27183772fffcb916963940ff4f31856 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 12:51:51 -0700 Subject: [PATCH 194/376] debug gha --- .../src/awses_test_vectors/manifests/mpl_keyring.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index 40421e931..d914b5501 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -63,7 +63,6 @@ def keyring(self, keys, keys_uri, mode): ''' - # print(f"{keys=}") keyvectors = KeyVectorsProvider.get_keyvectors(keys_path=keys_uri) From 9ca61e24c9b9b2563f04c6d251de0c33c5c5cd7d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 13:00:37 -0700 Subject: [PATCH 195/376] debug gha --- .../materials_managers/mpl/cmm.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index 8edf52151..c398904a9 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -67,7 +67,7 @@ def get_encryption_materials( CryptoMaterialsManagerFromMPL._native_to_mpl_get_encryption_materials( request ) - + mpl_output: MPL_GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) # ???????????????????????????? @@ -77,7 +77,7 @@ def get_encryption_materials( # print(kpis) # input - + # if len(kpis) == 1: # for edk in mpl_output.encryption_materials.encrypted_data_keys: # if edk.key_provider_info == b"rsa-4096-public": @@ -135,24 +135,15 @@ def decrypt_materials( Returns a DecryptionMaterialsFromMPL for the configured CMM. :param request: Request for decryption materials """ - from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.errors import CollectionOfErrors as COE try: mpl_input: 'MPL_DecryptMaterialsInput' = \ CryptoMaterialsManagerFromMPL._create_mpl_decrypt_materials_input_from_request(request) - # print(f"{mpl_input.as_dict()=}") - # input() mpl_output: 'MPL_DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(mpl_input) - # print(f"{mpl_output.as_dict()=}") - # input() return DecryptionMaterialsFromMPL(mpl_output.decryption_materials) except AwsCryptographicMaterialProvidersException as mpl_exception: # Wrap MPL error into the ESDK error type # so customers only have to catch ESDK error types. raise AWSEncryptionSDKClientError(mpl_exception) - except COE as coe: - # print(f"{coe.list=}") - pass - # raise AWSEncryptionSDKClientError(coe) @staticmethod def _native_algorithm_id_to_mpl_algorithm_id(native_algorithm_id: str) -> 'MPL_AlgorithmSuiteIdESDK': From 5deac12dc81b6988799e32e03560a6b2685349eb Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 13:13:35 -0700 Subject: [PATCH 196/376] debug cb --- .../awses_test_vectors/manifests/full_message/decrypt.py | 6 ++++-- test_vector_handlers/tox.ini | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index 1c04a83a2..430b81157 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -352,7 +352,8 @@ def _one_shot_decrypt(self): return client.decrypt(source=self.ciphertext, key_provider=self.master_key_provider_fn()) elif self.cmm_type == "RequiredEncryptionContext": # We need to make a custom CMM and pass it into the client - assert self.keyrings + if not self.keyrings: + raise ValueError("Must provide keyrings arg to use RequiredEncryptionContext") from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig @@ -406,7 +407,8 @@ def _streaming_decrypt(self): kwargs["key_provider"] = self.master_key_provider_fn() elif self.cmm_type == "RequiredEncryptionContext": # We need to make a custom CMM and pass it into the client - assert self.keyrings + if not self.keyrings: + raise ValueError("Must provide keyrings arg to use RequiredEncryptionContext") from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index 5c96def2b..497baed4b 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -37,7 +37,7 @@ envlist = # release :: Builds dist files and uploads to pypi pypirc profile. [testenv:base-command] -commands = python3 -m cProfile -o profile.txt -m pytest --basetemp={envtmpdir} -l {posargs} +commands = python3 -m cProfile -o profile.txt -m pytest --basetemp={envtmpdir} -l --cov awses_test_vectors {posargs} [testenv] passenv = @@ -55,7 +55,7 @@ deps = .. commands = awses_local: {[testenv:base-command]commands} test/integration - mplvectors: {[testenv:base-command]commands} test/keyrings -s -v + mplvectors: {[testenv:base-command]commands} test/keyrings [testenv:full-encrypt] basepython = python3 From ff99fe7990677d0914ae320acb50b0dd2856e1c9 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 14:37:03 -0700 Subject: [PATCH 197/376] debug cb --- .../manifests/full_message/decrypt.py | 51 +++++++++++++++++++ .../test_i_full_message_encrypt_keyrings.py | 2 +- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index 430b81157..cc4a6eac0 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -452,6 +452,57 @@ def _streaming_decrypt(self): def _streaming_decrypt_unsigned(self): result = bytearray() client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + + stream_kwargs = { + "source": self.ciphertext, + "mode": "decrypt-unsigned", + } + + if self.cmm_type == "Default": + if self.keyrings: + stream_kwargs["keyring"] = self.master_key_provider_fn() + else: + stream_kwargs["key_provider"] = self.master_key_provider_fn() + elif self.cmm_type == "RequiredEncryptionContext": + # We need to make a custom CMM and pass it into the client + if not self.keyrings: + raise ValueError("Must provide keyrings arg to use RequiredEncryptionContext") + + from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders + from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig + from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager + from aws_cryptographic_materialproviders.mpl.models import ( + CreateDefaultCryptographicMaterialsManagerInput, + CreateRequiredEncryptionContextCMMInput, + ) + + + mpl: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + config=MaterialProvidersConfig() + ) + + underlying_cmm: ICryptographicMaterialsManager = \ + mpl.create_default_cryptographic_materials_manager( + CreateDefaultCryptographicMaterialsManagerInput( + keyring=self.master_key_provider_fn() + ) + ) + + required_ec_cmm: ICryptographicMaterialsManager = \ + mpl.create_required_encryption_context_cmm( + CreateRequiredEncryptionContextCMMInput( + # Currently, the test vector manifest requires that + # if using the required encryption context CMM, + # both and only "key1" and "key2" are required. + required_encryption_context_keys=["key1", "key2"], + underlying_cmm=underlying_cmm, + ) + ) + + stream_kwargs["materials_manager"] = required_ec_cmm + stream_kwargs["encryption_context"] = self.encryption_context + + with client.stream( source=self.ciphertext, mode="decrypt-unsigned", key_provider=self.master_key_provider_fn() ) as decryptor: diff --git a/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py b/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py index 6d037a586..7ecbdb69f 100644 --- a/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py +++ b/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py @@ -28,7 +28,7 @@ def test_full_message_encrypt_canonical_full(full_message_encrypt_vectors): full_message_encrypt.cli(["--input", full_message_encrypt_vectors]) - full_message_encrypt.cli(["--input", full_message_encrypt_vectors], "--keyrings") + full_message_encrypt.cli(["--input", full_message_encrypt_vectors, "--keyrings"]) def test_full_message_cycle_canonical_full(tmpdir, full_message_decrypt_generation_vectors): From 6a4b7045c02cbec3afbed7c33e078ed13c10fafb Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 14:37:22 -0700 Subject: [PATCH 198/376] debug cb --- .../src/awses_test_vectors/manifests/full_message/decrypt.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index cc4a6eac0..1f214c89e 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -502,10 +502,7 @@ def _streaming_decrypt_unsigned(self): stream_kwargs["materials_manager"] = required_ec_cmm stream_kwargs["encryption_context"] = self.encryption_context - - with client.stream( - source=self.ciphertext, mode="decrypt-unsigned", key_provider=self.master_key_provider_fn() - ) as decryptor: + with client.stream(**stream_kwargs) as decryptor: for chunk in decryptor: result.extend(chunk) return result, decryptor.header From 0dbd4f636d7ff767cf56e02977a669c5775fcdbc Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 15:38:34 -0700 Subject: [PATCH 199/376] fix cb --- .../manifests/full_message/decrypt.py | 70 +++++++++---------- .../full_message/decrypt_generation.py | 19 ++++- .../manifests/full_message/encrypt.py | 2 +- .../manifests/mpl_keyring.py | 67 +++++++++--------- 4 files changed, 83 insertions(+), 75 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index 1f214c89e..a0f847762 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -37,6 +37,14 @@ try: from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_from_master_key_specs + from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders + from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig + from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager + from aws_cryptographic_materialproviders.mpl.models import ( + CreateDefaultCryptographicMaterialsManagerInput, + CreateRequiredEncryptionContextCMMInput, + ) + _HAS_MPL = True except ImportError: _HAS_MPL = False @@ -262,6 +270,7 @@ def from_scenario( keyrings, # type: bool keys_uri, # type: str ): + # pylint: disable=too-many-locals # type: (...) -> MessageDecryptionTestScenario """Load from a scenario specification. @@ -284,7 +293,7 @@ def from_scenario( def master_key_provider_fn(): if keyrings: - return keyring_from_master_key_specs(keys, keys_uri, master_key_specs, "decrypt") + return keyring_from_master_key_specs(keys_uri, master_key_specs, "decrypt") return master_key_provider_from_master_key_specs(keys, master_key_specs) decryption_method_spec = scenario.get("decryption-method") @@ -293,7 +302,7 @@ def master_key_provider_fn(): result = MessageDecryptionTestResult.from_result_spec(result_spec, plaintext_reader) encryption_context = scenario["encryption-context"] - + # MPL test vectors add CMM types to the test vectors manifests if "cmm" in scenario: if scenario["cmm"] == "Default": @@ -350,19 +359,13 @@ def _one_shot_decrypt(self): if self.keyrings: return client.decrypt(source=self.ciphertext, keyring=self.master_key_provider_fn()) return client.decrypt(source=self.ciphertext, key_provider=self.master_key_provider_fn()) - elif self.cmm_type == "RequiredEncryptionContext": + if self.cmm_type == "RequiredEncryptionContext": # We need to make a custom CMM and pass it into the client if not self.keyrings: raise ValueError("Must provide keyrings arg to use RequiredEncryptionContext") - - from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders - from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig - from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager - from aws_cryptographic_materialproviders.mpl.models import ( - CreateDefaultCryptographicMaterialsManagerInput, - CreateRequiredEncryptionContextCMMInput, - ) - + if not _HAS_MPL: + raise ValueError("Must install the aws-cryptographic-material-providers library" + "to use RequiredEncryptionContext") mpl: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( config=MaterialProvidersConfig() @@ -378,20 +381,23 @@ def _one_shot_decrypt(self): required_ec_cmm: ICryptographicMaterialsManager = \ mpl.create_required_encryption_context_cmm( CreateRequiredEncryptionContextCMMInput( - # Currently, the test vector manifest requires that + # Currently, the test vector manifest requires that # if using the required encryption context CMM, # both and only "key1" and "key2" are required. required_encryption_context_keys=["key1", "key2"], underlying_cmm=underlying_cmm, ) ) - + return client.decrypt( source=self.ciphertext, materials_manager=required_ec_cmm, encryption_context = self.encryption_context, ) + # If the cmm type was not in if/elif above, raise error + raise ValueError(f"Unrecognized cmm_type: {self.cmm_type}") + def _streaming_decrypt(self): result = bytearray() client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) @@ -409,15 +415,9 @@ def _streaming_decrypt(self): # We need to make a custom CMM and pass it into the client if not self.keyrings: raise ValueError("Must provide keyrings arg to use RequiredEncryptionContext") - - from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders - from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig - from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager - from aws_cryptographic_materialproviders.mpl.models import ( - CreateDefaultCryptographicMaterialsManagerInput, - CreateRequiredEncryptionContextCMMInput, - ) - + if not _HAS_MPL: + raise ValueError("Must install the aws-cryptographic-material-providers library" + "to use RequiredEncryptionContext") mpl: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( config=MaterialProvidersConfig() @@ -433,16 +433,18 @@ def _streaming_decrypt(self): required_ec_cmm: ICryptographicMaterialsManager = \ mpl.create_required_encryption_context_cmm( CreateRequiredEncryptionContextCMMInput( - # Currently, the test vector manifest requires that + # Currently, the test vector manifest requires that # if using the required encryption context CMM, # both and only "key1" and "key2" are required. required_encryption_context_keys=["key1", "key2"], underlying_cmm=underlying_cmm, ) ) - + kwargs["materials_manager"] = required_ec_cmm kwargs["encryption_context"] = self.encryption_context + else: + raise ValueError(f"Unrecognized cmm_type: {self.cmm_type}") with client.stream(**kwargs) as decryptor: for chunk in decryptor: @@ -467,15 +469,9 @@ def _streaming_decrypt_unsigned(self): # We need to make a custom CMM and pass it into the client if not self.keyrings: raise ValueError("Must provide keyrings arg to use RequiredEncryptionContext") - - from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders - from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig - from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager - from aws_cryptographic_materialproviders.mpl.models import ( - CreateDefaultCryptographicMaterialsManagerInput, - CreateRequiredEncryptionContextCMMInput, - ) - + if not _HAS_MPL: + raise ValueError("Must install the aws-cryptographic-material-providers library" + "to use RequiredEncryptionContext") mpl: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( config=MaterialProvidersConfig() @@ -491,16 +487,18 @@ def _streaming_decrypt_unsigned(self): required_ec_cmm: ICryptographicMaterialsManager = \ mpl.create_required_encryption_context_cmm( CreateRequiredEncryptionContextCMMInput( - # Currently, the test vector manifest requires that + # Currently, the test vector manifest requires that # if using the required encryption context CMM, # both and only "key1" and "key2" are required. required_encryption_context_keys=["key1", "key2"], underlying_cmm=underlying_cmm, ) ) - + stream_kwargs["materials_manager"] = required_ec_cmm stream_kwargs["encryption_context"] = self.encryption_context + else: + raise ValueError(f"Unrecognized cmm_type: {self.cmm_type}") with client.stream(**stream_kwargs) as decryptor: for chunk in decryptor: diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index a048dcc32..8d80c46e2 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -40,8 +40,10 @@ from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_from_master_key_specs + + _HAS_MPL = True except ImportError: - pass + _HAS_MPL = False from awses_test_vectors.internal.defaults import ENCODING @@ -314,7 +316,18 @@ def __init__(self, master_key_provider): Create a new CMM that wraps a new DefaultCryptoMaterialsManager based on the given master key provider. """ - self.wrapped_default_cmm = DefaultCryptoMaterialsManager(master_key_provider) + if isinstance(master_key_provider, MasterKeyProvider): + self.wrapped_default_cmm = DefaultCryptoMaterialsManager(master_key_provider) + elif _HAS_MPL and isinstance(master_key_provider, IKeyring): + mpl = AwsCryptographicMaterialProviders(MaterialProvidersConfig()) + mpl_cmm = mpl.create_default_cryptographic_materials_manager( + CreateDefaultCryptographicMaterialsManagerInput( + keyring=master_key_provider + ) + ) + self.wrapped_default_cmm = CryptoMaterialsManagerFromMPL(mpl_cmm=mpl_cmm) + else: + raise TypeError(f"Unrecognized master_key_provider type: {master_key_provider}") def get_encryption_materials(self, request): """ @@ -405,7 +418,7 @@ def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): def decryption_master_key_provider_fn(): if keyrings: - return keyring_from_master_key_specs(keys, keys_uri, decryption_master_key_specs, "decrypt-generation") + return keyring_from_master_key_specs(keys_uri, decryption_master_key_specs, "decrypt-generation") return master_key_provider_from_master_key_specs(keys, decryption_master_key_specs) else: diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index a3d351317..0c2580fa8 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -112,7 +112,7 @@ def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): def master_key_provider_fn(): if keyrings: - return keyring_from_master_key_specs(keys, keys_uri, master_key_specs, "encrypt") + return keyring_from_master_key_specs(keys_uri, master_key_specs, "encrypt") return master_key_provider_from_master_key_specs(keys, master_key_specs) return cls( diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index d914b5501..7094ec35c 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -14,8 +14,11 @@ This REQUIRES the aws-cryptographic-material-providers library. """ +import json import attr +# Ignore missing MPL for pylint, but the MPL is required for this example +# noqa pylint: disable=import-error from aws_cryptography_materialproviderstestvectorkeys.smithygenerated.\ aws_cryptography_materialproviderstestvectorkeys.models import ( GetKeyDescriptionInput, @@ -27,10 +30,12 @@ from aws_cryptographic_materialproviders.mpl.references import IKeyring from aws_cryptographic_materialproviders.mpl.models import CreateMultiKeyringInput +import _dafny +import UTF8 + from awses_test_vectors.internal.keyvectors_provider import KeyVectorsProvider -from awses_test_vectors.manifests.keys import KeysManifest # noqa pylint disable=unused-import +from awses_test_vectors.manifests.keys import KeysManifest -import json from .master_key import MasterKeySpec @@ -49,24 +54,18 @@ class KeyringSpec(MasterKeySpec): # pylint: disable=too-many-instance-attribute :param str padding_hash: Wrapping key padding hash (required for raw master keys) """ - def keyring(self, keys, keys_uri, mode): + def keyring(self, keys_uri, mode): # type: (KeysManifest) -> IKeyring """Build a keyring using this specification. :param str keys_uri: Path to the keys manifest """ - ''' - encryptmaterials keyProviderInfo = rsa-4096-public' - MUST be private. - somehow, it is writing "rsa-4096-public". - - ''' - - keyvectors = KeyVectorsProvider.get_keyvectors(keys_path=keys_uri) - changed = False + # Variable to flag whether we changed anything in weird hack #1. + # Signals to weird hack #2 whether it should execute. + changed_key_name_from_private_to_public = False # Construct the input to KeyVectorsConfig input_kwargs = { @@ -74,7 +73,7 @@ def keyring(self, keys, keys_uri, mode): "key": self.key_name, "provider-id": self.provider_id, "encryption-algorithm": self.encryption_algorithm, - + } if self.padding_algorithm is not None and self.padding_algorithm != "": input_kwargs["padding-algorithm"] = self.padding_algorithm @@ -85,7 +84,7 @@ def keyring(self, keys, keys_uri, mode): and input_kwargs["encryption-algorithm"] == "rsa": # Weird hack #1: # Gets public key for encryption instead of private key. - # + # # If generating decrypt vectors (i.e. encrypting) # and the manifest specified an RSA private key, # change the input to KeyVectors to a public key. @@ -93,8 +92,8 @@ def keyring(self, keys, keys_uri, mode): # If this is not done, then keyring.OnEncrypt fails with # "A RawRSAKeyring without a public key cannot provide OnEncrypt" if input_kwargs["key"] == "rsa-4096-private" \ - and (mode == "decrypt-generate" or mode == "encrypt"): - changed = True + and mode in ("decrypt-generate", "encrypt"): + changed_key_name_from_private_to_public = True input_kwargs["key"] = "rsa-4096-public" # Specify default padding-hash if "padding-hash" not in input_kwargs: @@ -117,17 +116,17 @@ def keyring(self, keys, keys_uri, mode): # Weird hack #2: # Sets keyProviderInfo to "private" even though the material is "public". - # + # # Weird hack #1 allows the encrypting keyring to be created with a public key. # However, it also changes the keyName of the encrypting keyring. # This hack changes it back. - # + # # If this is not done, then decryption fails # (for BOTH native master keys and MPL keyrings) - # with error + # with error # native master keys: "Unable to decrypt any data key" # MPL: "Raw RSA Key was unable to decrypt any encrypted data key" - # + # # Digging, the keyring is unable to decrypt in the MPL # because the EDK keyProviderInfo differs from the keyring keyName, # and this check fails: @@ -135,30 +134,28 @@ def keyring(self, keys, keys_uri, mode): # due to the two variables not being equal: # edk.keyProviderInfo='rsa-4096-public' # keyring.keyName='rsa-4096-private' - # - # Changing the encrypting keyring's keyName back to 'rsa-4096-private' - # sets any EDKs this keyring encrypts to now have + # + # Changing the encrypting keyring's keyName back to 'rsa-4096-private' + # sets any EDKs this keyring encrypts to now have # keyName="rsa-4096-private". # However, keyvectors has still retrieved the public key material to encrypt with. # So it any EDKs it encrypts will use the public material, but have keyName="rsa-4096-private". - # - # This configuration seems to be correct, because + # + # This configuration seems to be correct, because # all of the test vectors (master keys and MPL) pass with these two hacks. # But this seems weird, and we didn't have to do this in Java. - import _dafny - import UTF8 - - if hasattr(keyring, "_impl"): - if hasattr(keyring._impl, "_keyName"): + if hasattr(keyring, "_impl"): # pylint: disable=protected-access + if hasattr(keyring._impl, "_keyName"): # pylint: disable=protected-access if keyring._impl._keyName == UTF8.default__.Encode(_dafny.Seq("rsa-4096-public")).value \ - and (mode == "decrypt-generate" or mode == "encrypt"): - if changed: - keyring._impl._keyName = UTF8.default__.Encode(_dafny.Seq("rsa-4096-private")).value + and mode in ("decrypt-generate", "encrypt"): # pylint: disable=protected-access + if changed_key_name_from_private_to_public: + # pylint: disable=protected-access + keyring._impl._keyName = UTF8.default__.Encode(_dafny.Seq("rsa-4096-private")).value return keyring -def keyring_from_master_key_specs(keys, keys_uri, master_key_specs, mode): +def keyring_from_master_key_specs(keys_uri, master_key_specs, mode): # type: (str, list[KeyringSpec]) -> IKeyring """Build and combine all keyrings identified by the provided specs and using the provided keys. @@ -169,7 +166,7 @@ def keyring_from_master_key_specs(keys, keys_uri, master_key_specs, mode): :return: Master key provider combining all loaded master keys :rtype: IKeyring """ - keyrings = [spec.keyring(keys, keys_uri, mode) for spec in master_key_specs] + keyrings = [spec.keyring(keys_uri, mode) for spec in master_key_specs] primary = keyrings[0] others = keyrings[1:] From 357594b766e4d6c8a7c1815c92fdd271f85dcf95 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 16:37:48 -0700 Subject: [PATCH 200/376] debug cb --- .../manifests/full_message/decrypt_generation.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 8d80c46e2..b6806d8ca 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -41,6 +41,11 @@ from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_from_master_key_specs + from aws_encryption_sdk.materials_managers.mpl.materials import ( + EncryptionMaterialsFromMPL + ) + from awses_test_vectors.internal.half_signing_mpl_materials import HalfSigningEncryptionMaterialsFromMPL + _HAS_MPL = True except ImportError: _HAS_MPL = False @@ -297,7 +302,6 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, _p generation_scenario.decryption_test_scenario_pair(ciphertext_writer, ciphertext_to_decrypt, expected_result) ] - class HalfSigningCryptoMaterialsManager(CryptoMaterialsManager): """ Custom CMM that generates materials for an unsigned algorithm suite @@ -340,6 +344,11 @@ def get_encryption_materials(self, request): signing_request.algorithm = AlgorithmSuite.AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384 result = self.wrapped_default_cmm.get_encryption_materials(signing_request) + + if _HAS_MPL: + if isinstance(result, EncryptionMaterialsFromMPL): + result = HalfSigningEncryptionMaterialsFromMPL(result) + result.algorithm = request.algorithm result.signing_key = None From 3760ebe3967909d8b2fc8b439f1a2725f3eefb34 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 16:47:08 -0700 Subject: [PATCH 201/376] debug gha --- .../internal/keyvectors_provider.py | 13 +++++++------ .../test/integration/integration_test_utils.py | 7 +++++-- .../test_i_full_message_encrypt_keyrings.py | 14 -------------- 3 files changed, 12 insertions(+), 22 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py b/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py index 12dc980e3..85b8ef9fa 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py @@ -1,3 +1,6 @@ +"""Singleton provider for the KeyVectors client.""" +# # Ignore missing MPL TestVectors for pylint, but the MPL TestVectors is required for this file +# pylint: disable=import-error from aws_cryptography_materialproviderstestvectorkeys.smithygenerated.\ aws_cryptography_materialproviderstestvectorkeys.client import ( KeyVectors, @@ -9,17 +12,15 @@ keyvectors_instances = {} +# pylint: disable=too-few-public-methods class KeyVectorsProvider: - """Singleton manager for the KeyVectors client. - - This is used because Dafny's JSON deserializer implementation is slow with large files. - It deserializes the file at keys_path and takes >1 minute to do this. - """ + """Singleton manager for the KeyVectors client.""" instance: KeyVectors @classmethod - def get_keyvectors(self, keys_path): + def get_keyvectors(cls, keys_path): + """Returns the singleton KeyVectors client.""" if not keys_path in keyvectors_instances: keyvectors_instances[keys_path] = KeyVectors(KeyVectorsConfig(key_manifest_path=keys_path)) return keyvectors_instances[keys_path] diff --git a/test_vector_handlers/test/integration/integration_test_utils.py b/test_vector_handlers/test/integration/integration_test_utils.py index 696dc8648..b8c8beb56 100644 --- a/test_vector_handlers/test/integration/integration_test_utils.py +++ b/test_vector_handlers/test/integration/integration_test_utils.py @@ -43,7 +43,10 @@ def full_message_encrypt_vectors(): @pytest.fixture def full_message_decrypt_generation_vectors(): return os.path.join( - legacy_vectors_dir(), "features", "CANONICAL-GENERATED-MANIFESTS", "0006-awses-message-decryption-generation.v2.json" + legacy_vectors_dir(), + "features", + "CANONICAL-GENERATED-MANIFESTS", + "0006-awses-message-decryption-generation.v2.json" ) @@ -58,4 +61,4 @@ def mpl_decrypt_vectors(): def required_encryption_context_cmm_decrypt_vectors(): return os.path.join( required_ec_vectors_dir(), "manifest.json" - ) \ No newline at end of file + ) diff --git a/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py b/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py index 7ecbdb69f..6ffd97b60 100644 --- a/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py +++ b/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py @@ -34,7 +34,6 @@ def test_full_message_encrypt_canonical_full(full_message_encrypt_vectors): def test_full_message_cycle_canonical_full(tmpdir, full_message_decrypt_generation_vectors): # Generate vectors using keyring interfaces keyring_output_dir = tmpdir.join("output-keyrings") - print("Generating vectors with keyrings... ", end="") full_message_decrypt_generate.cli([ "--output", str(keyring_output_dir), @@ -42,9 +41,7 @@ def test_full_message_cycle_canonical_full(tmpdir, full_message_decrypt_generati full_message_decrypt_generation_vectors, "--keyrings" ]) - print("done") - print("Generating vectors with master keys... ", end="") # Generate vectors using master key interfaces master_key_output_dir = tmpdir.join("output-master-key") full_message_decrypt_generate.cli([ @@ -53,27 +50,16 @@ def test_full_message_cycle_canonical_full(tmpdir, full_message_decrypt_generati "--input", full_message_decrypt_generation_vectors ]) - print("done") # Validate that vectors generated using keyring interfaces # can be decrypted by BOTH keyring and master key interfaces keyring_decrypt_manifest_file = keyring_output_dir.join("manifest.json") - print("Decrypting keyring-encrypted vectors with keyrings... ", end="") full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file), "--keyrings"]) - print("done") - - print("Decrypting keyring-encrypted vectors with master keys... ", end="") full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file)]) - print("done") # Validate that vectors generated using master key interfaces # can be decrypted by BOTH keyring and master key interfaces master_key_decrypt_manifest_file = keyring_output_dir.join("manifest.json") - print("Decrypting master key-encrypted vectors with keyrings... ", end="") full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file), "--keyrings"]) - print("done") - - print("Decrypting master key-encrypted vectors with master keys... ", end="") full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file)]) - print("done") From 7b984fe85fdb35d6f3b940a298752329774154f6 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 16:53:12 -0700 Subject: [PATCH 202/376] add missing file --- .../internal/half_signing_mpl_materials.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py diff --git a/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py b/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py new file mode 100644 index 000000000..80c96ad7a --- /dev/null +++ b/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py @@ -0,0 +1,27 @@ +from aws_encryption_sdk.materials_managers.mpl.materials import ( + EncryptionMaterialsFromMPL +) + + +class HalfSigningEncryptionMaterialsFromMPL(EncryptionMaterialsFromMPL): + @EncryptionMaterialsFromMPL.algorithm.setter + def algorithm(self, algorithm): + self.set_algorithm = algorithm + + @EncryptionMaterialsFromMPL.algorithm.getter + def algorithm(self): + if hasattr(self, "set_algorithm"): + return self.set_algorithm + else: + return self.algorithm + + @EncryptionMaterialsFromMPL.signing_key.setter + def signing_key(self, signing_key): + self.set_signing_key = signing_key + + @EncryptionMaterialsFromMPL.signing_key.getter + def signing_key(self): + if hasattr(self, "set_signing_key"): + return self.set_signing_key + else: + return self.signing_key From 9b7a58d4c14c228573168b6473cf50766c7cf8f1 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 17:03:20 -0700 Subject: [PATCH 203/376] debug gha --- .../internal/half_signing_mpl_materials.py | 34 ++++++++++++------- .../internal/keyvectors_provider.py | 3 +- .../manifests/full_message/decrypt.py | 4 +-- .../full_message/decrypt_generation.py | 1 + .../manifests/mpl_keyring.py | 6 ++-- 5 files changed, 30 insertions(+), 18 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py b/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py index 80c96ad7a..481167684 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py @@ -1,27 +1,37 @@ +"""Allows overriding the algorithm and signing_key for EncryptionMaterialsFromMPL. +This must ONLY be used in testing and NOT in production.. +This is used in testing malicious message modification (HalfSigningTampering). +""" from aws_encryption_sdk.materials_managers.mpl.materials import ( EncryptionMaterialsFromMPL ) class HalfSigningEncryptionMaterialsFromMPL(EncryptionMaterialsFromMPL): - @EncryptionMaterialsFromMPL.algorithm.setter - def algorithm(self, algorithm): - self.set_algorithm = algorithm - + """Allows overriding the algorithm and signing_key for EncryptionMaterialsFromMPL. + This must ONLY be used in testing and NOT in production.. + This is used in testing malicious message modification (HalfSigningTampering). + """ + # pylint thinks EncryptionMaterialsFromMPL.algorithm is a method + # pylint: disable=invalid-overridden-method @EncryptionMaterialsFromMPL.algorithm.getter def algorithm(self): if hasattr(self, "set_algorithm"): return self.set_algorithm - else: - return self.algorithm - - @EncryptionMaterialsFromMPL.signing_key.setter - def signing_key(self, signing_key): - self.set_signing_key = signing_key + return self.algorithm + @algorithm.setter + def algorithm(self, algorithm): + self.set_algorithm = algorithm + + # pylint thinks EncryptionMaterialsFromMPL.signing_key is a method + # pylint: disable=invalid-overridden-method @EncryptionMaterialsFromMPL.signing_key.getter def signing_key(self): if hasattr(self, "set_signing_key"): return self.set_signing_key - else: - return self.signing_key + return self.signing_key + + @signing_key.setter + def signing_key(self, signing_key): + self.set_signing_key = signing_key diff --git a/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py b/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py index 85b8ef9fa..71e75c025 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py @@ -12,6 +12,7 @@ keyvectors_instances = {} + # pylint: disable=too-few-public-methods class KeyVectorsProvider: """Singleton manager for the KeyVectors client.""" @@ -21,6 +22,6 @@ class KeyVectorsProvider: @classmethod def get_keyvectors(cls, keys_path): """Returns the singleton KeyVectors client.""" - if not keys_path in keyvectors_instances: + if keys_path not in keyvectors_instances: keyvectors_instances[keys_path] = KeyVectors(KeyVectorsConfig(key_manifest_path=keys_path)) return keyvectors_instances[keys_path] diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index a0f847762..91628d1ee 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -64,7 +64,7 @@ CLIENT_NAME = "aws/aws-encryption-sdk-python" CURRENT_VERSION = 2 -SUPPORTED_VERSIONS = (2,4,) +SUPPORTED_VERSIONS = (2, 4,) @attr.s(init=False) @@ -392,7 +392,7 @@ def _one_shot_decrypt(self): return client.decrypt( source=self.ciphertext, materials_manager=required_ec_cmm, - encryption_context = self.encryption_context, + encryption_context=self.encryption_context, ) # If the cmm type was not in if/elif above, raise error diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index b6806d8ca..d48285ef2 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -302,6 +302,7 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, _p generation_scenario.decryption_test_scenario_pair(ciphertext_writer, ciphertext_to_decrypt, expected_result) ] + class HalfSigningCryptoMaterialsManager(CryptoMaterialsManager): """ Custom CMM that generates materials for an unsigned algorithm suite diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index 7094ec35c..02a41f1d1 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -34,7 +34,7 @@ import UTF8 from awses_test_vectors.internal.keyvectors_provider import KeyVectorsProvider -from awses_test_vectors.manifests.keys import KeysManifest +from awses_test_vectors.manifests.keys import KeysManifest # noqa: disable=F401 from .master_key import MasterKeySpec @@ -92,7 +92,7 @@ def keyring(self, keys_uri, mode): # If this is not done, then keyring.OnEncrypt fails with # "A RawRSAKeyring without a public key cannot provide OnEncrypt" if input_kwargs["key"] == "rsa-4096-private" \ - and mode in ("decrypt-generate", "encrypt"): + and mode in ("decrypt-generate", "encrypt"): changed_key_name_from_private_to_public = True input_kwargs["key"] = "rsa-4096-public" # Specify default padding-hash @@ -147,7 +147,7 @@ def keyring(self, keys_uri, mode): if hasattr(keyring, "_impl"): # pylint: disable=protected-access if hasattr(keyring._impl, "_keyName"): # pylint: disable=protected-access if keyring._impl._keyName == UTF8.default__.Encode(_dafny.Seq("rsa-4096-public")).value \ - and mode in ("decrypt-generate", "encrypt"): # pylint: disable=protected-access + and mode in ("decrypt-generate", "encrypt"): # pylint: disable=protected-access if changed_key_name_from_private_to_public: # pylint: disable=protected-access keyring._impl._keyName = UTF8.default__.Encode(_dafny.Seq("rsa-4096-private")).value From 93fee671031ed511fd14c569ab691b028d42dd82 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 17:06:44 -0700 Subject: [PATCH 204/376] debug cb --- .../src/awses_test_vectors/manifests/full_message/decrypt.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index 91628d1ee..4432502c5 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -301,7 +301,10 @@ def master_key_provider_fn(): result_spec = scenario["result"] result = MessageDecryptionTestResult.from_result_spec(result_spec, plaintext_reader) - encryption_context = scenario["encryption-context"] + if "encryption-context" in scenario: + encryption_context = scenario["encryption-context"] + else: + encryption_context = {} # MPL test vectors add CMM types to the test vectors manifests if "cmm" in scenario: From e65fec4878d141ee18286acc74ae4c387f699182 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 17:16:17 -0700 Subject: [PATCH 205/376] debug gha --- .../internal/half_signing_mpl_materials.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py b/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py index 481167684..b70a901d6 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py @@ -2,6 +2,8 @@ This must ONLY be used in testing and NOT in production.. This is used in testing malicious message modification (HalfSigningTampering). """ +# Ignore missing MPL for pylint, but the MPL is required for this class +# pylint: disable=import-error,no-name-in-module from aws_encryption_sdk.materials_managers.mpl.materials import ( EncryptionMaterialsFromMPL ) @@ -16,6 +18,9 @@ class HalfSigningEncryptionMaterialsFromMPL(EncryptionMaterialsFromMPL): # pylint: disable=invalid-overridden-method @EncryptionMaterialsFromMPL.algorithm.getter def algorithm(self): + """Returns any previously-provided overriden algorithm; + if none was provided, returns underlying algorithm from encryption materials. + """ if hasattr(self, "set_algorithm"): return self.set_algorithm return self.algorithm @@ -28,6 +33,9 @@ def algorithm(self, algorithm): # pylint: disable=invalid-overridden-method @EncryptionMaterialsFromMPL.signing_key.getter def signing_key(self): + """Returns any previously-provided overriden signing_key; + if none was provided, returns underlying signing_key from encryption materials. + """ if hasattr(self, "set_signing_key"): return self.set_signing_key return self.signing_key From f4ebbba80a6e71ba816b6cabf504129d52be590d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 17:21:28 -0700 Subject: [PATCH 206/376] imperative mood --- .../internal/half_signing_mpl_materials.py | 5 +++-- .../src/awses_test_vectors/internal/keyvectors_provider.py | 2 +- .../src/awses_test_vectors/manifests/mpl_keyring.py | 1 - 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py b/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py index b70a901d6..b782e029c 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py @@ -14,11 +14,12 @@ class HalfSigningEncryptionMaterialsFromMPL(EncryptionMaterialsFromMPL): This must ONLY be used in testing and NOT in production.. This is used in testing malicious message modification (HalfSigningTampering). """ + # pylint thinks EncryptionMaterialsFromMPL.algorithm is a method # pylint: disable=invalid-overridden-method @EncryptionMaterialsFromMPL.algorithm.getter def algorithm(self): - """Returns any previously-provided overriden algorithm; + """Return any previously-provided overriden algorithm; if none was provided, returns underlying algorithm from encryption materials. """ if hasattr(self, "set_algorithm"): @@ -33,7 +34,7 @@ def algorithm(self, algorithm): # pylint: disable=invalid-overridden-method @EncryptionMaterialsFromMPL.signing_key.getter def signing_key(self): - """Returns any previously-provided overriden signing_key; + """Return any previously-provided overriden signing_key; if none was provided, returns underlying signing_key from encryption materials. """ if hasattr(self, "set_signing_key"): diff --git a/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py b/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py index 71e75c025..305459026 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py @@ -21,7 +21,7 @@ class KeyVectorsProvider: @classmethod def get_keyvectors(cls, keys_path): - """Returns the singleton KeyVectors client.""" + """Return the singleton KeyVectors client.""" if keys_path not in keyvectors_instances: keyvectors_instances[keys_path] = KeyVectors(KeyVectorsConfig(key_manifest_path=keys_path)) return keyvectors_instances[keys_path] diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index 02a41f1d1..ea702e4eb 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -57,7 +57,6 @@ class KeyringSpec(MasterKeySpec): # pylint: disable=too-many-instance-attribute def keyring(self, keys_uri, mode): # type: (KeysManifest) -> IKeyring """Build a keyring using this specification. - :param str keys_uri: Path to the keys manifest """ From 05511a820656d2363c303b791acc1e7dddec3fb2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 17:24:23 -0700 Subject: [PATCH 207/376] fix gha --- .../src/awses_test_vectors/manifests/mpl_keyring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index ea702e4eb..74e6a5737 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -57,9 +57,9 @@ class KeyringSpec(MasterKeySpec): # pylint: disable=too-many-instance-attribute def keyring(self, keys_uri, mode): # type: (KeysManifest) -> IKeyring """Build a keyring using this specification. + :param str keys_uri: Path to the keys manifest """ - keyvectors = KeyVectorsProvider.get_keyvectors(keys_path=keys_uri) # Variable to flag whether we changed anything in weird hack #1. From c9c58e6ef45aea9f6c94a24dee02d489c41bffa4 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 17:27:24 -0700 Subject: [PATCH 208/376] fix gha --- .../src/awses_test_vectors/manifests/mpl_keyring.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index 74e6a5737..55a9276c9 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -57,7 +57,6 @@ class KeyringSpec(MasterKeySpec): # pylint: disable=too-many-instance-attribute def keyring(self, keys_uri, mode): # type: (KeysManifest) -> IKeyring """Build a keyring using this specification. - :param str keys_uri: Path to the keys manifest """ keyvectors = KeyVectorsProvider.get_keyvectors(keys_path=keys_uri) From a62f1b4d2a1c824da5796409d93e43b6fcb03075 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 18:18:19 -0700 Subject: [PATCH 209/376] debug cb --- .../internal/half_signing_mpl_materials.py | 29 ++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py b/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py index b782e029c..b8f405fde 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py @@ -15,16 +15,21 @@ class HalfSigningEncryptionMaterialsFromMPL(EncryptionMaterialsFromMPL): This is used in testing malicious message modification (HalfSigningTampering). """ + _underlying_materials: EncryptionMaterialsFromMPL + + def __init__(self, underling_materials): + self._underlying_materials = underling_materials + # pylint thinks EncryptionMaterialsFromMPL.algorithm is a method # pylint: disable=invalid-overridden-method - @EncryptionMaterialsFromMPL.algorithm.getter + @property def algorithm(self): """Return any previously-provided overriden algorithm; if none was provided, returns underlying algorithm from encryption materials. """ if hasattr(self, "set_algorithm"): return self.set_algorithm - return self.algorithm + return self._underlying_materials.algorithm @algorithm.setter def algorithm(self, algorithm): @@ -32,15 +37,31 @@ def algorithm(self, algorithm): # pylint thinks EncryptionMaterialsFromMPL.signing_key is a method # pylint: disable=invalid-overridden-method - @EncryptionMaterialsFromMPL.signing_key.getter + @property def signing_key(self): """Return any previously-provided overriden signing_key; if none was provided, returns underlying signing_key from encryption materials. """ if hasattr(self, "set_signing_key"): return self.set_signing_key - return self.signing_key + return self._underlying_materials.algorithm @signing_key.setter def signing_key(self, signing_key): self.set_signing_key = signing_key + + @property + def encryption_context(self): + return self._underlying_materials.encryption_context + + @property + def encrypted_data_keys(self): + return self._underlying_materials.encrypted_data_keys + + @property + def data_encryption_key(self): + return self._underlying_materials.data_encryption_key + + @property + def required_encryption_context_keys(self): + return self._underlying_materials.required_encryption_context_keys From 752c98c4e8fd6bc0a8ee5d19110b3cd0b8b8e9f5 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 18:28:30 -0700 Subject: [PATCH 210/376] no more profile --- test_vector_handlers/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index 497baed4b..c2ff913c2 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -37,7 +37,7 @@ envlist = # release :: Builds dist files and uploads to pypi pypirc profile. [testenv:base-command] -commands = python3 -m cProfile -o profile.txt -m pytest --basetemp={envtmpdir} -l --cov awses_test_vectors {posargs} +commands = pytest --basetemp={envtmpdir} -l --cov awses_test_vectors {posargs} [testenv] passenv = From d2897e5cb0023a7b4624a4ef4ef9325d0dc58a8a Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 19:33:33 -0700 Subject: [PATCH 211/376] debug cb --- .../internal/half_signing_mpl_materials.py | 2 +- .../full_message/decrypt_generation.py | 34 ++++++++++++++----- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py b/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py index b8f405fde..0f16d57f6 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py @@ -1,6 +1,6 @@ """Allows overriding the algorithm and signing_key for EncryptionMaterialsFromMPL. This must ONLY be used in testing and NOT in production.. -This is used in testing malicious message modification (HalfSigningTampering). +This is used in message tampering testing. """ # Ignore missing MPL for pylint, but the MPL is required for this class # pylint: disable=import-error,no-name-in-module diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index d48285ef2..d002c67ad 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -169,16 +169,34 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, _p master_key_provider = generation_scenario.encryption_scenario.master_key_provider_fn() # Use a caching CMM to avoid generating a new data key every time. - cache = LocalCryptoMaterialsCache(10) - caching_cmm = CachingCryptoMaterialsManager( - master_key_provider=master_key_provider, - cache=cache, - max_age=60.0, - max_messages_encrypted=100, - ) + if isinstance(master_key_provider, MasterKeyProvider): + cache = LocalCryptoMaterialsCache(10) + cmm = CachingCryptoMaterialsManager( + master_key_provider=master_key_provider, + cache=cache, + max_age=60.0, + max_messages_encrypted=100, + ) + cmm = caching_cmm + elif _HAS_MPL and isinstance(master_key_provider, IKeyring): + mpl = AwsCryptographicMaterialProviders(MaterialProvidersConfig()) + mpl_caching_cmm = mpl.create_default_cryptographic_materials_manager( + CreateDefaultCryptographicMaterialsManagerInput( + + ) + ) + mpl_cmm = mpl.create_default_cryptographic_materials_manager( + CreateDefaultCryptographicMaterialsManagerInput( + keyring=master_key_provider + ) + ) + cmm = CryptoMaterialsManagerFromMPL(mpl_cmm=mpl_cmm) + else: + raise TypeError(f"Unrecognized master_key_provider type: {master_key_provider}") + return [ self.run_scenario_with_new_provider_info( - ciphertext_writer, generation_scenario, caching_cmm, new_provider_info + ciphertext_writer, generation_scenario, cmm, new_provider_info ) for new_provider_info in self.new_provider_infos ] From dc7887df8ef2bca11700f6a6027c12a7dada6d41 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 20:51:59 -0700 Subject: [PATCH 212/376] debug cb --- .../manifests/full_message/decrypt_generation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index d002c67ad..8dcda0eb6 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -171,7 +171,7 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, _p # Use a caching CMM to avoid generating a new data key every time. if isinstance(master_key_provider, MasterKeyProvider): cache = LocalCryptoMaterialsCache(10) - cmm = CachingCryptoMaterialsManager( + caching_cmm = CachingCryptoMaterialsManager( master_key_provider=master_key_provider, cache=cache, max_age=60.0, From 36a46303b1fe30c8eb811764c29cd3f2ffe4054c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 21:16:30 -0700 Subject: [PATCH 213/376] debug cb --- .../manifests/full_message/decrypt_generation.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 8dcda0eb6..847229b84 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -180,11 +180,6 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, _p cmm = caching_cmm elif _HAS_MPL and isinstance(master_key_provider, IKeyring): mpl = AwsCryptographicMaterialProviders(MaterialProvidersConfig()) - mpl_caching_cmm = mpl.create_default_cryptographic_materials_manager( - CreateDefaultCryptographicMaterialsManagerInput( - - ) - ) mpl_cmm = mpl.create_default_cryptographic_materials_manager( CreateDefaultCryptographicMaterialsManagerInput( keyring=master_key_provider From 736c1f4a30b96ffa3183d7d195c44f1e727eacb9 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 09:15:19 -0700 Subject: [PATCH 214/376] debug cb --- .../awses_test_vectors/manifests/full_message/encrypt.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 0c2580fa8..25697e15c 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -41,8 +41,10 @@ try: from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_from_master_key_specs + + _HAS_MPL = True except ImportError: - pass + _HAS_MPL = False try: # Python 3.5.0 and 3.5.1 have incompatible typing modules @@ -149,9 +151,9 @@ def run(self, materials_manager=None): ) if materials_manager: encrypt_kwargs["materials_manager"] = materials_manager - elif self.keyrings: + elif isinstance(self.master_key_provider_fn(), MasterKeySpec): encrypt_kwargs["keyring"] = self.master_key_provider_fn() - else: + elif _HAS_MPL and isinstance(self.master_key_provider_fn(), KeyringSpec): encrypt_kwargs["key_provider"] = self.master_key_provider_fn() ciphertext, _header = client.encrypt(**encrypt_kwargs) return ciphertext From 1adfb12d56eb00ef9d0c6a9b851fc071e7826d7f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 09:44:50 -0700 Subject: [PATCH 215/376] debug cb --- .../awses_test_vectors/manifests/full_message/encrypt.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 25697e15c..3415bb1f5 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -152,9 +152,11 @@ def run(self, materials_manager=None): if materials_manager: encrypt_kwargs["materials_manager"] = materials_manager elif isinstance(self.master_key_provider_fn(), MasterKeySpec): - encrypt_kwargs["keyring"] = self.master_key_provider_fn() - elif _HAS_MPL and isinstance(self.master_key_provider_fn(), KeyringSpec): encrypt_kwargs["key_provider"] = self.master_key_provider_fn() + elif _HAS_MPL and isinstance(self.master_key_provider_fn(), KeyringSpec): + encrypt_kwargs["keyring"] = self.master_key_provider_fn() + else: + raise TypeError(f"Unrecognized master_key_provider_fn return type: {self.master_key_provider_fn()}") ciphertext, _header = client.encrypt(**encrypt_kwargs) return ciphertext From dd4b495b625a5a42bbbd592c2f893b7246c0a273 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 10:15:37 -0700 Subject: [PATCH 216/376] debug cb --- codebuild/py312/awses_local_mpl.yml | 4 ++-- codebuild/py312/mplawses_local_mpl.yml | 3 +-- .../manifests/full_message/encrypt.py | 10 ++++++++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/codebuild/py312/awses_local_mpl.yml b/codebuild/py312/awses_local_mpl.yml index 1d0f80319..96ca5bc28 100644 --- a/codebuild/py312/awses_local_mpl.yml +++ b/codebuild/py312/awses_local_mpl.yml @@ -1,5 +1,5 @@ -# Runs the same tests as awses_local in an environment with the MPL installed. -# This asserts existing tests continue to pass with the MPL installed. +# Runs test vectors using native constructs in an environment with the MPL installed. +# This asserts that installing the MPL does not change existing behavior. version: 0.2 env: diff --git a/codebuild/py312/mplawses_local_mpl.yml b/codebuild/py312/mplawses_local_mpl.yml index e3f06e7f6..d932f0461 100644 --- a/codebuild/py312/mplawses_local_mpl.yml +++ b/codebuild/py312/mplawses_local_mpl.yml @@ -1,5 +1,4 @@ -# Runs the same tests as awses_local in an environment with the MPL installed. -# This asserts existing tests continue to pass with the MPL installed. +# Runs MPL-specific test vectors. version: 0.2 env: diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 3415bb1f5..6343b7044 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -31,15 +31,21 @@ membership_validator, validate_manifest_type, ) +from aws_encryption_sdk.key_providers.base import MasterKeyProvider from awses_test_vectors.manifests.keys import KeysManifest from awses_test_vectors.manifests.master_key import MasterKeySpec, master_key_provider_from_master_key_specs + try: from aws_encryption_sdk.identifiers import AlgorithmSuite, CommitmentPolicy except ImportError: from aws_encryption_sdk.identifiers import Algorithm as AlgorithmSuite try: + from aws_cryptographic_materialproviders.mpl.references import ( + IKeyring, + ) + from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_from_master_key_specs _HAS_MPL = True @@ -151,9 +157,9 @@ def run(self, materials_manager=None): ) if materials_manager: encrypt_kwargs["materials_manager"] = materials_manager - elif isinstance(self.master_key_provider_fn(), MasterKeySpec): + elif isinstance(self.master_key_provider_fn(), MasterKeyProvider): encrypt_kwargs["key_provider"] = self.master_key_provider_fn() - elif _HAS_MPL and isinstance(self.master_key_provider_fn(), KeyringSpec): + elif _HAS_MPL and isinstance(self.master_key_provider_fn(), IKeyring): encrypt_kwargs["keyring"] = self.master_key_provider_fn() else: raise TypeError(f"Unrecognized master_key_provider_fn return type: {self.master_key_provider_fn()}") From 76b1f29c8ff29e98305c82c5538475d90a7fe01b Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 11:08:09 -0700 Subject: [PATCH 217/376] debug cb --- buildspec.yml | 142 ++++++++++++++++++ codebuild/py312/mplawses_local_mpl.yml | 2 +- .../full_message/decrypt_generation.py | 36 ++++- .../test/keyrings/__init__.py | 0 .../test/keyrings/integration/__init__.py | 0 .../keyrings/integration/commands/__init__.py | 0 .../test_i_full_message_encrypt_keyrings.py | 65 -------- test_vector_handlers/tox.ini | 2 +- 8 files changed, 179 insertions(+), 68 deletions(-) delete mode 100644 test_vector_handlers/test/keyrings/__init__.py delete mode 100644 test_vector_handlers/test/keyrings/integration/__init__.py delete mode 100644 test_vector_handlers/test/keyrings/integration/commands/__init__.py delete mode 100644 test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py diff --git a/buildspec.yml b/buildspec.yml index fff7c68d1..89526ba64 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -15,6 +15,28 @@ batch: buildspec: codebuild/py37/awses_local.yml env: image: aws/codebuild/standard:5.0 + - identifier: py37_decrypt_dafny_esdk_vectors + buildspec: codebuild/py37/decrypt_dafny_esdk_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py37_decrypt_net_401_vectors + buildspec: codebuild/py37/decrypt_net_401_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py37_generate_decrypt_vectors + buildspec: codebuild/py37/generate_decrypt_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py37_decrypt_generated + depend-on: py37_generate_decrypt_vectors + buildspec: codebuild/py37/decrypt_generated_with_python.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py37_decrypt_generated_js + depend-on: py37_generate_decrypt_vectors + buildspec: codebuild/py37/decrypt_generated_with_js.yml + env: + image: aws/codebuild/standard:5.0 - identifier: py38_integ buildspec: codebuild/py38/integ.yml @@ -28,6 +50,28 @@ batch: buildspec: codebuild/py38/awses_local.yml env: image: aws/codebuild/standard:5.0 + - identifier: py38_decrypt_dafny_esdk_vectors + buildspec: codebuild/py38/decrypt_dafny_esdk_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py38_decrypt_net_401_vectors + buildspec: codebuild/py38/decrypt_net_401_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py38_generate_decrypt_vectors + buildspec: codebuild/py38/generate_decrypt_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py38_decrypt_generated + depend-on: py38_generate_decrypt_vectors + buildspec: codebuild/py38/decrypt_generated_with_python.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py38_decrypt_generated_js + depend-on: py38_generate_decrypt_vectors + buildspec: codebuild/py38/decrypt_generated_with_js.yml + env: + image: aws/codebuild/standard:5.0 - identifier: py39_integ buildspec: codebuild/py39/integ.yml @@ -40,6 +84,28 @@ batch: - identifier: py39_awses_latest env: image: aws/codebuild/standard:5.0 + - identifier: py39_decrypt_dafny_esdk_vectors + buildspec: codebuild/py39/decrypt_dafny_esdk_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_decrypt_net_401_vectors + buildspec: codebuild/py39/decrypt_net_401_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_generate_decrypt_vectors + buildspec: codebuild/py39/generate_decrypt_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_decrypt_generated + depend-on: py39_generate_decrypt_vectors + buildspec: codebuild/py39/decrypt_generated_with_python.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_decrypt_generated_js + depend-on: py39_generate_decrypt_vectors + buildspec: codebuild/py39/decrypt_generated_with_js.yml + env: + image: aws/codebuild/standard:5.0 - identifier: py310_integ buildspec: codebuild/py310/integ.yml @@ -53,6 +119,28 @@ batch: buildspec: codebuild/py310/awses_local.yml env: image: aws/codebuild/standard:6.0 + - identifier: py310_decrypt_dafny_esdk_vectors + buildspec: codebuild/py310/decrypt_dafny_esdk_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py310_decrypt_net_401_vectors + buildspec: codebuild/py310/decrypt_net_401_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py310_generate_decrypt_vectors + buildspec: codebuild/py310/generate_decrypt_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py310_decrypt_generated + depend-on: py310_generate_decrypt_vectors + buildspec: codebuild/py310/decrypt_generated_with_python.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py310_decrypt_generated_js + depend-on: py310_generate_decrypt_vectors + buildspec: codebuild/py310/decrypt_generated_with_js.yml + env: + image: aws/codebuild/standard:5.0 - identifier: py311_integ buildspec: codebuild/py311/integ.yml @@ -82,6 +170,60 @@ batch: buildspec: codebuild/py311/mplawses_local_mpl.yml env: image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_dafny_esdk_vectors_masterkey + buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_dafny_esdk_vectors_keyrings + buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_net_401_vectors_masterkey + buildspec: codebuild/py311/decrypt_net_401_vectors_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_net_401_vectors_keyrings + buildspec: codebuild/py311/decrypt_net_401_vectors_keyrings.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_generate_decrypt_vectors_masterkey + buildspec: codebuild/py311/generate_decrypt_vectors_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_masterkey_with_masterkey + depend-on: py311_generate_decrypt_vectors_masterkey + buildspec: codebuild/py311/decrypt_masterkey_with_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_masterkey_with_keyrings + depend-on: py311_generate_decrypt_vectors_masterkey + buildspec: codebuild/py311/decrypt_masterkey_with_keyrings.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_masterkey_with_js + depend-on: py311_generate_decrypt_vectors_masterkey + buildspec: codebuild/py311/decrypt_masterkey_with_js.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_generate_decrypt_vectors_keyrings + buildspec: codebuild/py311/generate_decrypt_vectors_keyrings.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_keyrings_with_masterkey + depend-on: py311_generate_decrypt_vectors_keyrings + buildspec: codebuild/py311/decrypt_keyrings_with_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_keyrings_with_keyrings + depend-on: py311_generate_decrypt_vectors_keyrings + buildspec: codebuild/py311/decrypt_keyrings_with_keyrings.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_keyrings_with_js + depend-on: py311_generate_decrypt_vectors_keyrings + buildspec: codebuild/py311/decrypt_keyrings_with_js.yml + env: + image: aws/codebuild/standard:5.0 - identifier: py312_integ buildspec: codebuild/py312/integ.yml diff --git a/codebuild/py312/mplawses_local_mpl.yml b/codebuild/py312/mplawses_local_mpl.yml index d932f0461..e11f7523b 100644 --- a/codebuild/py312/mplawses_local_mpl.yml +++ b/codebuild/py312/mplawses_local_mpl.yml @@ -1,4 +1,4 @@ -# Runs MPL-specific test vectors. +# Runs test vectors using MPL constructs. version: 0.2 env: diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 847229b84..1db214525 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -200,7 +200,10 @@ def run_scenario_with_new_provider_info( self, ciphertext_writer, generation_scenario, materials_manager, new_provider_info ): """Run with tampering for a specific new provider info value""" - tampering_materials_manager = ProviderInfoChangingCryptoMaterialsManager(materials_manager, new_provider_info) + if isinstance(materials_manager, CryptoMaterialsManagerFromMPL): + tampering_materials_manager = ProviderInfoChangingCryptoMaterialsManagerFromMPL(materials_manager, new_provider_info) + else: + tampering_materials_manager = ProviderInfoChangingCryptoMaterialsManager(materials_manager, new_provider_info) ciphertext_to_decrypt = generation_scenario.encryption_scenario.run(tampering_materials_manager) expected_result = MessageDecryptionTestResult.expect_error( "Incorrect encrypted data key provider info: " + new_provider_info @@ -239,6 +242,37 @@ def get_encryption_materials(self, request): def decrypt_materials(self, request): """Thunks to the wrapped CMM""" return self.wrapped_cmm.decrypt_materials(request) + + +class ProviderInfoChangingCryptoMaterialsManagerFromMPL(CryptoMaterialsManagerFromMPL): + """ + Custom CMM that modifies the provider info field on EDKS. + + THIS IS ONLY USED TO CREATE INVALID MESSAGES and should never be used in + production! + """ + + wrapped_cmm = attr.ib(validator=attr.validators.instance_of(CryptoMaterialsManager)) + new_provider_info = attr.ib(validator=attr.validators.instance_of(six.string_types)) + + def __init__(self, materials_manager, new_provider_info): + """Create a new CMM that wraps a the given CMM.""" + self.wrapped_cmm = materials_manager + self.new_provider_info = new_provider_info + + def get_encryption_materials(self, request): + """ + Request materials from the wrapped CMM, and then change the provider info + on each EDK. + """ + result = self.wrapped_cmm.get_encryption_materials(request) + for encrypted_data_key in result.encrypted_data_keys: + encrypted_data_key.key_provider.key_info = self.new_provider_info + return result + + def decrypt_materials(self, request): + """Thunks to the wrapped CMM""" + return self.wrapped_cmm.decrypt_materials(request) BITS_PER_BYTE = 8 diff --git a/test_vector_handlers/test/keyrings/__init__.py b/test_vector_handlers/test/keyrings/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/test_vector_handlers/test/keyrings/integration/__init__.py b/test_vector_handlers/test/keyrings/integration/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/test_vector_handlers/test/keyrings/integration/commands/__init__.py b/test_vector_handlers/test/keyrings/integration/commands/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py b/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py deleted file mode 100644 index 6ffd97b60..000000000 --- a/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. -""" -Integration tests for `awses_test_vectors.commands` with keyrings. -""" -import pytest - -from awses_test_vectors.commands import full_message_decrypt, full_message_decrypt_generate, full_message_encrypt - -from ....integration.integration_test_utils import ( # noqa pylint: disable=unused-import - full_message_decrypt_generation_vectors, - full_message_encrypt_vectors, -) - - -pytestmark = [pytest.mark.integ] - - -def test_full_message_encrypt_canonical_full(full_message_encrypt_vectors): - full_message_encrypt.cli(["--input", full_message_encrypt_vectors]) - full_message_encrypt.cli(["--input", full_message_encrypt_vectors, "--keyrings"]) - - -def test_full_message_cycle_canonical_full(tmpdir, full_message_decrypt_generation_vectors): - # Generate vectors using keyring interfaces - keyring_output_dir = tmpdir.join("output-keyrings") - full_message_decrypt_generate.cli([ - "--output", - str(keyring_output_dir), - "--input", - full_message_decrypt_generation_vectors, - "--keyrings" - ]) - - # Generate vectors using master key interfaces - master_key_output_dir = tmpdir.join("output-master-key") - full_message_decrypt_generate.cli([ - "--output", - str(master_key_output_dir), - "--input", - full_message_decrypt_generation_vectors - ]) - - # Validate that vectors generated using keyring interfaces - # can be decrypted by BOTH keyring and master key interfaces - keyring_decrypt_manifest_file = keyring_output_dir.join("manifest.json") - full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file), "--keyrings"]) - full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file)]) - - # Validate that vectors generated using master key interfaces - # can be decrypted by BOTH keyring and master key interfaces - master_key_decrypt_manifest_file = keyring_output_dir.join("manifest.json") - - full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file), "--keyrings"]) - full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file)]) diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index c2ff913c2..95dc2c9ba 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -55,7 +55,7 @@ deps = .. commands = awses_local: {[testenv:base-command]commands} test/integration - mplvectors: {[testenv:base-command]commands} test/keyrings + mplvectors: {[testenv:base-command]commands} test/mpl [testenv:full-encrypt] basepython = python3 From 33d7bdba32df553cc62438716784dbfae3622f16 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 11:13:34 -0700 Subject: [PATCH 218/376] debug cb --- buildspec.yml | 42 ++++++++++++------- .../full_message/decrypt_generation.py | 2 + 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index 89526ba64..bb5f8767f 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -28,12 +28,14 @@ batch: env: image: aws/codebuild/standard:5.0 - identifier: py37_decrypt_generated - depend-on: py37_generate_decrypt_vectors + depend-on: + - py37_generate_decrypt_vectors buildspec: codebuild/py37/decrypt_generated_with_python.yml env: image: aws/codebuild/standard:5.0 - identifier: py37_decrypt_generated_js - depend-on: py37_generate_decrypt_vectors + depend-on: + - py37_generate_decrypt_vectors buildspec: codebuild/py37/decrypt_generated_with_js.yml env: image: aws/codebuild/standard:5.0 @@ -63,12 +65,14 @@ batch: env: image: aws/codebuild/standard:5.0 - identifier: py38_decrypt_generated - depend-on: py38_generate_decrypt_vectors + depend-on: + - py38_generate_decrypt_vectors buildspec: codebuild/py38/decrypt_generated_with_python.yml env: image: aws/codebuild/standard:5.0 - identifier: py38_decrypt_generated_js - depend-on: py38_generate_decrypt_vectors + depend-on: + - py38_generate_decrypt_vectors buildspec: codebuild/py38/decrypt_generated_with_js.yml env: image: aws/codebuild/standard:5.0 @@ -97,12 +101,14 @@ batch: env: image: aws/codebuild/standard:5.0 - identifier: py39_decrypt_generated - depend-on: py39_generate_decrypt_vectors + depend-on: + - py39_generate_decrypt_vectors buildspec: codebuild/py39/decrypt_generated_with_python.yml env: image: aws/codebuild/standard:5.0 - identifier: py39_decrypt_generated_js - depend-on: py39_generate_decrypt_vectors + depend-on: + - py39_generate_decrypt_vectors buildspec: codebuild/py39/decrypt_generated_with_js.yml env: image: aws/codebuild/standard:5.0 @@ -132,12 +138,14 @@ batch: env: image: aws/codebuild/standard:5.0 - identifier: py310_decrypt_generated - depend-on: py310_generate_decrypt_vectors + depend-on: + - py310_generate_decrypt_vectors buildspec: codebuild/py310/decrypt_generated_with_python.yml env: image: aws/codebuild/standard:5.0 - identifier: py310_decrypt_generated_js - depend-on: py310_generate_decrypt_vectors + depend-on: + - py310_generate_decrypt_vectors buildspec: codebuild/py310/decrypt_generated_with_js.yml env: image: aws/codebuild/standard:5.0 @@ -191,17 +199,20 @@ batch: env: image: aws/codebuild/standard:5.0 - identifier: py311_decrypt_masterkey_with_masterkey - depend-on: py311_generate_decrypt_vectors_masterkey + depend-on: + - py311_generate_decrypt_vectors_masterkey buildspec: codebuild/py311/decrypt_masterkey_with_masterkey.yml env: image: aws/codebuild/standard:5.0 - identifier: py311_decrypt_masterkey_with_keyrings - depend-on: py311_generate_decrypt_vectors_masterkey + depend-on: + - py311_generate_decrypt_vectors_masterkey buildspec: codebuild/py311/decrypt_masterkey_with_keyrings.yml env: image: aws/codebuild/standard:5.0 - identifier: py311_decrypt_masterkey_with_js - depend-on: py311_generate_decrypt_vectors_masterkey + depend-on: + - py311_generate_decrypt_vectors_masterkey buildspec: codebuild/py311/decrypt_masterkey_with_js.yml env: image: aws/codebuild/standard:5.0 @@ -210,17 +221,20 @@ batch: env: image: aws/codebuild/standard:5.0 - identifier: py311_decrypt_keyrings_with_masterkey - depend-on: py311_generate_decrypt_vectors_keyrings + depend-on: + - py311_generate_decrypt_vectors_keyrings buildspec: codebuild/py311/decrypt_keyrings_with_masterkey.yml env: image: aws/codebuild/standard:5.0 - identifier: py311_decrypt_keyrings_with_keyrings - depend-on: py311_generate_decrypt_vectors_keyrings + depend-on: + - py311_generate_decrypt_vectors_keyrings buildspec: codebuild/py311/decrypt_keyrings_with_keyrings.yml env: image: aws/codebuild/standard:5.0 - identifier: py311_decrypt_keyrings_with_js - depend-on: py311_generate_decrypt_vectors_keyrings + depend-on: + - py311_generate_decrypt_vectors_keyrings buildspec: codebuild/py311/decrypt_keyrings_with_js.yml env: image: aws/codebuild/standard:5.0 diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 1db214525..a57d20cb2 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -247,6 +247,8 @@ def decrypt_materials(self, request): class ProviderInfoChangingCryptoMaterialsManagerFromMPL(CryptoMaterialsManagerFromMPL): """ Custom CMM that modifies the provider info field on EDKS. + This extends CryptoMaterialsManagerFromMPL so ESDK-internal checks + follow MPL logic. THIS IS ONLY USED TO CREATE INVALID MESSAGES and should never be used in production! From 6a8a623119fd2626eab35eda68933dcff390b2a7 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 13:08:41 -0700 Subject: [PATCH 219/376] debug new toxes --- buildspec.yml | 530 ++++++++++-------- .../internal/half_signing_mpl_materials.py | 67 --- .../full_message/decrypt_generation.py | 38 +- test_vector_handlers/tox.ini | 2 + 4 files changed, 301 insertions(+), 336 deletions(-) delete mode 100644 test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py diff --git a/buildspec.yml b/buildspec.yml index bb5f8767f..a79eb5d48 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -27,253 +27,313 @@ batch: buildspec: codebuild/py37/generate_decrypt_vectors.yml env: image: aws/codebuild/standard:5.0 - - identifier: py37_decrypt_generated + - identifier: py37_decrypt_masterkey_with_masterkey depend-on: - py37_generate_decrypt_vectors - buildspec: codebuild/py37/decrypt_generated_with_python.yml + buildspec: codebuild/py37/decrypt_masterkey_with_masterkey.yml env: image: aws/codebuild/standard:5.0 - - identifier: py37_decrypt_generated_js + - identifier: py37_decrypt_masterkey_with_js depend-on: - py37_generate_decrypt_vectors - buildspec: codebuild/py37/decrypt_generated_with_js.yml + buildspec: codebuild/py37/decrypt_masterkey_with_js.yml env: image: aws/codebuild/standard:5.0 - - identifier: py38_integ - buildspec: codebuild/py38/integ.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py38_examples - buildspec: codebuild/py38/examples.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py38_awses_local - buildspec: codebuild/py38/awses_local.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py38_decrypt_dafny_esdk_vectors - buildspec: codebuild/py38/decrypt_dafny_esdk_vectors.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py38_decrypt_net_401_vectors - buildspec: codebuild/py38/decrypt_net_401_vectors.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py38_generate_decrypt_vectors - buildspec: codebuild/py38/generate_decrypt_vectors.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py38_decrypt_generated - depend-on: - - py38_generate_decrypt_vectors - buildspec: codebuild/py38/decrypt_generated_with_python.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py38_decrypt_generated_js - depend-on: - - py38_generate_decrypt_vectors - buildspec: codebuild/py38/decrypt_generated_with_js.yml - env: - image: aws/codebuild/standard:5.0 + # - identifier: py38_integ + # buildspec: codebuild/py38/integ.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py38_examples + # buildspec: codebuild/py38/examples.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py38_awses_local + # buildspec: codebuild/py38/awses_local.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py38_decrypt_dafny_esdk_vectors + # buildspec: codebuild/py38/decrypt_dafny_esdk_vectors.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py38_decrypt_net_401_vectors + # buildspec: codebuild/py38/decrypt_net_401_vectors.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py38_generate_decrypt_vectors + # buildspec: codebuild/py38/generate_decrypt_vectors.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py38_decrypt_masterkey_with_masterkey + # depend-on: + # - py38_generate_decrypt_vectors + # buildspec: codebuild/py38/decrypt_masterkey_with_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py38_decrypt_masterkey_with_js + # depend-on: + # - py38_generate_decrypt_vectors + # buildspec: codebuild/py38/decrypt_generated_with_js.yml + # env: + # image: aws/codebuild/standard:5.0 - - identifier: py39_integ - buildspec: codebuild/py39/integ.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py39_examples - buildspec: codebuild/py39/examples.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py39_awses_latest - env: - image: aws/codebuild/standard:5.0 - - identifier: py39_decrypt_dafny_esdk_vectors - buildspec: codebuild/py39/decrypt_dafny_esdk_vectors.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py39_decrypt_net_401_vectors - buildspec: codebuild/py39/decrypt_net_401_vectors.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py39_generate_decrypt_vectors - buildspec: codebuild/py39/generate_decrypt_vectors.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py39_decrypt_generated - depend-on: - - py39_generate_decrypt_vectors - buildspec: codebuild/py39/decrypt_generated_with_python.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py39_decrypt_generated_js - depend-on: - - py39_generate_decrypt_vectors - buildspec: codebuild/py39/decrypt_generated_with_js.yml - env: - image: aws/codebuild/standard:5.0 + # - identifier: py39_integ + # buildspec: codebuild/py39/integ.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py39_examples + # buildspec: codebuild/py39/examples.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py39_awses_latest + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py39_decrypt_dafny_esdk_vectors + # buildspec: codebuild/py39/decrypt_dafny_esdk_vectors.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py39_decrypt_net_401_vectors + # buildspec: codebuild/py39/decrypt_net_401_vectors.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py39_generate_decrypt_vectors + # buildspec: codebuild/py39/generate_decrypt_vectors.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py39_decrypt_masterkey_with_masterkey + # depend-on: + # - py39_generate_decrypt_vectors + # buildspec: codebuild/py39/decrypt_masterkey_with_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py39_decrypt_masterkey_with_js + # depend-on: + # - py39_generate_decrypt_vectors + # buildspec: codebuild/py39/decrypt_generated_with_js.yml + # env: + # image: aws/codebuild/standard:5.0 - - identifier: py310_integ - buildspec: codebuild/py310/integ.yml - env: - image: aws/codebuild/standard:6.0 - - identifier: py310_examples - buildspec: codebuild/py310/examples.yml - env: - image: aws/codebuild/standard:6.0 - - identifier: py310_awses_latest - buildspec: codebuild/py310/awses_local.yml - env: - image: aws/codebuild/standard:6.0 - - identifier: py310_decrypt_dafny_esdk_vectors - buildspec: codebuild/py310/decrypt_dafny_esdk_vectors.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py310_decrypt_net_401_vectors - buildspec: codebuild/py310/decrypt_net_401_vectors.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py310_generate_decrypt_vectors - buildspec: codebuild/py310/generate_decrypt_vectors.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py310_decrypt_generated - depend-on: - - py310_generate_decrypt_vectors - buildspec: codebuild/py310/decrypt_generated_with_python.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py310_decrypt_generated_js - depend-on: - - py310_generate_decrypt_vectors - buildspec: codebuild/py310/decrypt_generated_with_js.yml - env: - image: aws/codebuild/standard:5.0 + # - identifier: py310_integ + # buildspec: codebuild/py310/integ.yml + # env: + # image: aws/codebuild/standard:6.0 + # - identifier: py310_examples + # buildspec: codebuild/py310/examples.yml + # env: + # image: aws/codebuild/standard:6.0 + # - identifier: py310_awses_latest + # buildspec: codebuild/py310/awses_local.yml + # env: + # image: aws/codebuild/standard:6.0 + # - identifier: py310_decrypt_dafny_esdk_vectors + # buildspec: codebuild/py310/decrypt_dafny_esdk_vectors.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py310_decrypt_net_401_vectors + # buildspec: codebuild/py310/decrypt_net_401_vectors.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py310_generate_decrypt_vectors + # buildspec: codebuild/py310/generate_decrypt_vectors.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py310_decrypt_masterkey_with_masterkey + # depend-on: + # - py310_generate_decrypt_vectors + # buildspec: codebuild/py310/decrypt_masterkey_with_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py310_decrypt_masterkey_with_js + # depend-on: + # - py310_generate_decrypt_vectors + # buildspec: codebuild/py310/decrypt_generated_with_js.yml + # env: + # image: aws/codebuild/standard:5.0 - - identifier: py311_integ - buildspec: codebuild/py311/integ.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_integ_mpl - buildspec: codebuild/py311/integ_mpl.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_examples - buildspec: codebuild/py311/examples.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_examples_mpl - buildspec: codebuild/py311/examples_mpl.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_awses_latest - buildspec: codebuild/py311/awses_local.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_awses_latest_mpl - buildspec: codebuild/py311/awses_local_mpl.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_mplawses_latest_mpl - buildspec: codebuild/py311/mplawses_local_mpl.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_dafny_esdk_vectors_masterkey - buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py311_decrypt_dafny_esdk_vectors_keyrings - buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py311_decrypt_net_401_vectors_masterkey - buildspec: codebuild/py311/decrypt_net_401_vectors_masterkey.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py311_decrypt_net_401_vectors_keyrings - buildspec: codebuild/py311/decrypt_net_401_vectors_keyrings.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py311_generate_decrypt_vectors_masterkey - buildspec: codebuild/py311/generate_decrypt_vectors_masterkey.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py311_decrypt_masterkey_with_masterkey - depend-on: - - py311_generate_decrypt_vectors_masterkey - buildspec: codebuild/py311/decrypt_masterkey_with_masterkey.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py311_decrypt_masterkey_with_keyrings - depend-on: - - py311_generate_decrypt_vectors_masterkey - buildspec: codebuild/py311/decrypt_masterkey_with_keyrings.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py311_decrypt_masterkey_with_js - depend-on: - - py311_generate_decrypt_vectors_masterkey - buildspec: codebuild/py311/decrypt_masterkey_with_js.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py311_generate_decrypt_vectors_keyrings - buildspec: codebuild/py311/generate_decrypt_vectors_keyrings.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py311_decrypt_keyrings_with_masterkey - depend-on: - - py311_generate_decrypt_vectors_keyrings - buildspec: codebuild/py311/decrypt_keyrings_with_masterkey.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py311_decrypt_keyrings_with_keyrings - depend-on: - - py311_generate_decrypt_vectors_keyrings - buildspec: codebuild/py311/decrypt_keyrings_with_keyrings.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py311_decrypt_keyrings_with_js - depend-on: - - py311_generate_decrypt_vectors_keyrings - buildspec: codebuild/py311/decrypt_keyrings_with_js.yml - env: - image: aws/codebuild/standard:5.0 + # - identifier: py311_integ + # buildspec: codebuild/py311/integ.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_integ_mpl + # buildspec: codebuild/py311/integ_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_examples + # buildspec: codebuild/py311/examples.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_examples_mpl + # buildspec: codebuild/py311/examples_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_awses_latest + # buildspec: codebuild/py311/awses_local.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_awses_latest_mpl + # buildspec: codebuild/py311/awses_local_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_mplawses_latest_mpl + # buildspec: codebuild/py311/mplawses_local_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_dafny_esdk_vectors_masterkey + # buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py311_decrypt_dafny_esdk_vectors_keyrings + # buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py311_decrypt_net_401_vectors_masterkey + # buildspec: codebuild/py311/decrypt_net_401_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py311_decrypt_net_401_vectors_keyrings + # buildspec: codebuild/py311/decrypt_net_401_vectors_keyrings.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py311_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py311/generate_decrypt_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py311_decrypt_masterkey_with_masterkey + # depend-on: + # - py311_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py311/decrypt_masterkey_with_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py311_decrypt_masterkey_with_keyrings + # depend-on: + # - py311_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py311/decrypt_masterkey_with_keyrings.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py311_decrypt_masterkey_with_js + # depend-on: + # - py311_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py311/decrypt_masterkey_with_js.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py311_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py311/generate_decrypt_vectors_keyrings.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py311_decrypt_keyrings_with_masterkey + # depend-on: + # - py311_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py311/decrypt_keyrings_with_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py311_decrypt_keyrings_with_keyrings + # depend-on: + # - py311_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py311/decrypt_keyrings_with_keyrings.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py311_decrypt_keyrings_with_js + # depend-on: + # - py311_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py311/decrypt_keyrings_with_js.yml + # env: + # image: aws/codebuild/standard:5.0 - - identifier: py312_integ - buildspec: codebuild/py312/integ.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_integ_mpl - buildspec: codebuild/py312/integ_mpl.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_examples - buildspec: codebuild/py312/examples.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_examples_mpl - buildspec: codebuild/py312/examples_mpl.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_awses_latest - buildspec: codebuild/py312/awses_local.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_awses_latest_mpl - buildspec: codebuild/py312/awses_local_mpl.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_mplawses_latest_mpl - buildspec: codebuild/py312/mplawses_local_mpl.yml - env: - image: aws/codebuild/standard:7.0 + # - identifier: py312_integ + # buildspec: codebuild/py312/integ.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_integ_mpl + # buildspec: codebuild/py312/integ_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_examples + # buildspec: codebuild/py312/examples.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_examples_mpl + # buildspec: codebuild/py312/examples_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_awses_latest + # buildspec: codebuild/py312/awses_local.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_awses_latest_mpl + # buildspec: codebuild/py312/awses_local_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_mplawses_latest_mpl + # buildspec: codebuild/py312/mplawses_local_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_decrypt_dafny_esdk_vectors_masterkey + # buildspec: codebuild/py312/decrypt_dafny_esdk_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py312_decrypt_dafny_esdk_vectors_keyrings + # buildspec: codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py312_decrypt_net_401_vectors_masterkey + # buildspec: codebuild/py312/decrypt_net_401_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py312_decrypt_net_401_vectors_keyrings + # buildspec: codebuild/py312/decrypt_net_401_vectors_keyrings.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py312_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py312/generate_decrypt_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py312_decrypt_masterkey_with_masterkey + # depend-on: + # - py312_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py312/decrypt_masterkey_with_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py312_decrypt_masterkey_with_keyrings + # depend-on: + # - py312_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py312/decrypt_masterkey_with_keyrings.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py312_decrypt_masterkey_with_js + # depend-on: + # - py312_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py312/decrypt_masterkey_with_js.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py312_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py312/generate_decrypt_vectors_keyrings.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py312_decrypt_keyrings_with_masterkey + # depend-on: + # - py312_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py312/decrypt_keyrings_with_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py312_decrypt_keyrings_with_keyrings + # depend-on: + # - py312_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py312/decrypt_keyrings_with_keyrings.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py312_decrypt_keyrings_with_js + # depend-on: + # - py312_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py312/decrypt_keyrings_with_js.yml + # env: + # image: aws/codebuild/standard:5.0 - - identifier: code_coverage - buildspec: codebuild/coverage/coverage.yml - - identifier: code_coverage_mpl - buildspec: codebuild/coverage/coverage_mpl.yml - env: - image: aws/codebuild/standard:7.0 + # - identifier: code_coverage + # buildspec: codebuild/coverage/coverage.yml + # - identifier: code_coverage_mpl + # buildspec: codebuild/coverage/coverage_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 - - identifier: compliance - buildspec: codebuild/compliance/compliance.yml + # - identifier: compliance + # buildspec: codebuild/compliance/compliance.yml diff --git a/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py b/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py deleted file mode 100644 index 0f16d57f6..000000000 --- a/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Allows overriding the algorithm and signing_key for EncryptionMaterialsFromMPL. -This must ONLY be used in testing and NOT in production.. -This is used in message tampering testing. -""" -# Ignore missing MPL for pylint, but the MPL is required for this class -# pylint: disable=import-error,no-name-in-module -from aws_encryption_sdk.materials_managers.mpl.materials import ( - EncryptionMaterialsFromMPL -) - - -class HalfSigningEncryptionMaterialsFromMPL(EncryptionMaterialsFromMPL): - """Allows overriding the algorithm and signing_key for EncryptionMaterialsFromMPL. - This must ONLY be used in testing and NOT in production.. - This is used in testing malicious message modification (HalfSigningTampering). - """ - - _underlying_materials: EncryptionMaterialsFromMPL - - def __init__(self, underling_materials): - self._underlying_materials = underling_materials - - # pylint thinks EncryptionMaterialsFromMPL.algorithm is a method - # pylint: disable=invalid-overridden-method - @property - def algorithm(self): - """Return any previously-provided overriden algorithm; - if none was provided, returns underlying algorithm from encryption materials. - """ - if hasattr(self, "set_algorithm"): - return self.set_algorithm - return self._underlying_materials.algorithm - - @algorithm.setter - def algorithm(self, algorithm): - self.set_algorithm = algorithm - - # pylint thinks EncryptionMaterialsFromMPL.signing_key is a method - # pylint: disable=invalid-overridden-method - @property - def signing_key(self): - """Return any previously-provided overriden signing_key; - if none was provided, returns underlying signing_key from encryption materials. - """ - if hasattr(self, "set_signing_key"): - return self.set_signing_key - return self._underlying_materials.algorithm - - @signing_key.setter - def signing_key(self, signing_key): - self.set_signing_key = signing_key - - @property - def encryption_context(self): - return self._underlying_materials.encryption_context - - @property - def encrypted_data_keys(self): - return self._underlying_materials.encrypted_data_keys - - @property - def data_encryption_key(self): - return self._underlying_materials.data_encryption_key - - @property - def required_encryption_context_keys(self): - return self._underlying_materials.required_encryption_context_keys diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index a57d20cb2..74985b5d8 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -44,7 +44,10 @@ from aws_encryption_sdk.materials_managers.mpl.materials import ( EncryptionMaterialsFromMPL ) - from awses_test_vectors.internal.half_signing_mpl_materials import HalfSigningEncryptionMaterialsFromMPL + from awses_test_vectors.internal.tampering_mpl_materials import ( + HalfSigningEncryptionMaterialsFromMPL, + ProviderInfoChangingCryptoMaterialsManagerFromMPL, + ) _HAS_MPL = True except ImportError: @@ -244,39 +247,6 @@ def decrypt_materials(self, request): return self.wrapped_cmm.decrypt_materials(request) -class ProviderInfoChangingCryptoMaterialsManagerFromMPL(CryptoMaterialsManagerFromMPL): - """ - Custom CMM that modifies the provider info field on EDKS. - This extends CryptoMaterialsManagerFromMPL so ESDK-internal checks - follow MPL logic. - - THIS IS ONLY USED TO CREATE INVALID MESSAGES and should never be used in - production! - """ - - wrapped_cmm = attr.ib(validator=attr.validators.instance_of(CryptoMaterialsManager)) - new_provider_info = attr.ib(validator=attr.validators.instance_of(six.string_types)) - - def __init__(self, materials_manager, new_provider_info): - """Create a new CMM that wraps a the given CMM.""" - self.wrapped_cmm = materials_manager - self.new_provider_info = new_provider_info - - def get_encryption_materials(self, request): - """ - Request materials from the wrapped CMM, and then change the provider info - on each EDK. - """ - result = self.wrapped_cmm.get_encryption_materials(request) - for encrypted_data_key in result.encrypted_data_keys: - encrypted_data_key.key_provider.key_info = self.new_provider_info - return result - - def decrypt_materials(self, request): - """Thunks to the wrapped CMM""" - return self.wrapped_cmm.decrypt_materials(request) - - BITS_PER_BYTE = 8 diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index 95dc2c9ba..bf4b86724 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -56,6 +56,8 @@ deps = commands = awses_local: {[testenv:base-command]commands} test/integration mplvectors: {[testenv:base-command]commands} test/mpl + full_decrypt_generate: awses-full-message-decrypt-generate {posargs} + full_decrypt: awses-full-message-decrypt {posargs} [testenv:full-encrypt] basepython = python3 From 9fd746852267e19f1a32b1e7a24828b40d551fc5 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 13:11:11 -0700 Subject: [PATCH 220/376] debug new toxes --- buildspec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildspec.yml b/buildspec.yml index a79eb5d48..ca87c41c5 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -2,7 +2,7 @@ version: 0.2 batch: fast-fail: false - build-list: + build-graph: - identifier: py37_integ buildspec: codebuild/py37/integ.yml env: From 5affe9cdf24998e0fd4fd8fd20683aa961631bab Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 13:15:12 -0700 Subject: [PATCH 221/376] add missing files --- codebuild/py37/decrypt_masterkey_with_js.yml | 39 +++++++++++++++++++ .../py37/decrypt_masterkey_with_masterkey.yml | 26 +++++++++++++ codebuild/py37/generate_decrypt_vectors.yml | 27 +++++++++++++ 3 files changed, 92 insertions(+) create mode 100644 codebuild/py37/decrypt_masterkey_with_js.yml create mode 100644 codebuild/py37/decrypt_masterkey_with_masterkey.yml create mode 100644 codebuild/py37/generate_decrypt_vectors.yml diff --git a/codebuild/py37/decrypt_masterkey_with_js.yml b/codebuild/py37/decrypt_masterkey_with_js.yml new file mode 100644 index 000000000..32db1083e --- /dev/null +++ b/codebuild/py37/decrypt_masterkey_with_js.yml @@ -0,0 +1,39 @@ +version: 0.2 + +env: + variables: + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + GENERATE_OUTPUT_DIR: >- + $CODEBUILD_SRC_DIR/generated_vectors/ + + +phases: + install: + runtime-versions: + python: 3.7 + commands: + - n 16 + # Install the Javascript ESDK run test vectors + - npm install -g @aws-crypto/integration-node + + pre_build: + commands: + # Assume Role to access non-prod resources + - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Java-Role-us-west-2" --role-session-name "CB-TestVectorResources") + - export TMP_ROLE + - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') + - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') + - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') + - aws sts get-caller-identity + - cd $CODEBUILD_SRC_DIR + build: + commands: + # Decrypt generated vectors with Javascript ESDK + - integration-node decrypt -v ../tmp/generated/37_masterkey \ No newline at end of file diff --git a/codebuild/py37/decrypt_masterkey_with_masterkey.yml b/codebuild/py37/decrypt_masterkey_with_masterkey.yml new file mode 100644 index 000000000..df7067e60 --- /dev/null +++ b/codebuild/py37/decrypt_masterkey_with_masterkey.yml @@ -0,0 +1,26 @@ +version: 0.2 + +env: + variables: + TOXENV: "py37-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + python: 3.7 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - tox -- \ + --input ../tmp/generated/37_masterkey diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml new file mode 100644 index 000000000..849605b49 --- /dev/null +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -0,0 +1,27 @@ +version: 0.2 + +env: + variables: + TOXENV: "py37-full_decrypt_generate" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + python: 3.7 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ + --output ../tmp/generated/37_masterkey From 78b817d38296c6d08f8708a188a22a0d2bf5d50c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 13:17:36 -0700 Subject: [PATCH 222/376] debug tox --- codebuild/py37/generate_decrypt_vectors.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml index 849605b49..04a09a47d 100644 --- a/codebuild/py37/generate_decrypt_vectors.yml +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -22,6 +22,7 @@ phases: commands: - pip install "tox < 4.0" - cd test_vector_handlers - - tox -- \ - --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ - --output ../tmp/generated/37_masterkey + - | + tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ + --output ../tmp/generated/37_masterkey From c817996188a3cfef7c404483122be215b5a93fb1 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 13:24:57 -0700 Subject: [PATCH 223/376] add missing --- codebuild/py37/decrypt_net_401_vectors.yml | 48 ++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 codebuild/py37/decrypt_net_401_vectors.yml diff --git a/codebuild/py37/decrypt_net_401_vectors.yml b/codebuild/py37/decrypt_net_401_vectors.yml new file mode 100644 index 000000000..7053b6c96 --- /dev/null +++ b/codebuild/py37/decrypt_net_401_vectors.yml @@ -0,0 +1,48 @@ +version: 0.2 +# Runs Only the ESDK-NET v4.0.1 Decryption Vectors, testing Required EC CMM + +env: + variables: + TOXENV: "py37-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + java: $JAVA_ENV_VERSION + commands: + # Get Dafny + - curl https://github.com/dafny-lang/dafny/releases/download/v4.2.0/dafny-4.2.0-x64-ubuntu-20.04.zip -L -o dafny.zip + - unzip -qq dafny.zip && rm dafny.zip + - export PATH="$PWD/dafny:$PATH" + pre_build: + commands: + # Assume Role to access non-prod resources + - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Java-Role-us-west-2" --role-session-name "CB-TestVectorResources") + - export TMP_ROLE + - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') + - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') + - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') + - aws sts get-caller-identity + + # Fetch ESDK .NET v4.0.1 Test Vectors + - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip + - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip + - curl -s --output $VECTOR_ZIP --location $VECTORS_URL + build: + commands: + # NOTE: We need to pass the absolute path of the vectors + - pip install "tox < 4.0" + - cd $CODEBUILD_SRC_DIR/test_vector_handlers + - | + tox -- \ + --input $VECTOR_ZIP From c549e393cb7bbb307aa1334470cf27d9bb4ab09f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 13:27:45 -0700 Subject: [PATCH 224/376] debug cb --- codebuild/py37/decrypt_net_401_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_net_401_vectors.yml b/codebuild/py37/decrypt_net_401_vectors.yml index 7053b6c96..7f5c1c8e5 100644 --- a/codebuild/py37/decrypt_net_401_vectors.yml +++ b/codebuild/py37/decrypt_net_401_vectors.yml @@ -18,7 +18,7 @@ env: phases: install: runtime-versions: - java: $JAVA_ENV_VERSION + python: 3.7 commands: # Get Dafny - curl https://github.com/dafny-lang/dafny/releases/download/v4.2.0/dafny-4.2.0-x64-ubuntu-20.04.zip -L -o dafny.zip From 90a18d09460afab78b437fe9f53a3091219f030f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 13:32:05 -0700 Subject: [PATCH 225/376] debug cb --- codebuild/py37/decrypt_net_401_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_net_401_vectors.yml b/codebuild/py37/decrypt_net_401_vectors.yml index 7f5c1c8e5..5ec0365ef 100644 --- a/codebuild/py37/decrypt_net_401_vectors.yml +++ b/codebuild/py37/decrypt_net_401_vectors.yml @@ -27,7 +27,7 @@ phases: pre_build: commands: # Assume Role to access non-prod resources - - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Java-Role-us-west-2" --role-session-name "CB-TestVectorResources") + - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") - export TMP_ROLE - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') From 77fba50f7ed4c7bcee42cbcc7086434d8b74f5b9 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 13:33:07 -0700 Subject: [PATCH 226/376] debug cb --- .../manifests/full_message/decrypt_generation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 74985b5d8..c7cc2be90 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -124,7 +124,7 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, pl materials_manager = DefaultCryptoMaterialsManager( key_provider ) - elif isinstance(key_provider, IKeyring): + elif _HAS_MPL and isinstance(key_provider, IKeyring): mpl = AwsCryptographicMaterialProviders(MaterialProvidersConfig()) mpl_cmm = mpl.create_default_cryptographic_materials_manager( CreateDefaultCryptographicMaterialsManagerInput( @@ -203,7 +203,7 @@ def run_scenario_with_new_provider_info( self, ciphertext_writer, generation_scenario, materials_manager, new_provider_info ): """Run with tampering for a specific new provider info value""" - if isinstance(materials_manager, CryptoMaterialsManagerFromMPL): + if _HAS_MPL and isinstance(materials_manager, CryptoMaterialsManagerFromMPL): tampering_materials_manager = ProviderInfoChangingCryptoMaterialsManagerFromMPL(materials_manager, new_provider_info) else: tampering_materials_manager = ProviderInfoChangingCryptoMaterialsManager(materials_manager, new_provider_info) From 991c55f8795f25a31465814e8772e1a95576872a Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 13:38:12 -0700 Subject: [PATCH 227/376] debug cb --- codebuild/py37/decrypt_net_401_vectors.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/codebuild/py37/decrypt_net_401_vectors.yml b/codebuild/py37/decrypt_net_401_vectors.yml index 5ec0365ef..59e703024 100644 --- a/codebuild/py37/decrypt_net_401_vectors.yml +++ b/codebuild/py37/decrypt_net_401_vectors.yml @@ -38,11 +38,12 @@ phases: - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip - curl -s --output $VECTOR_ZIP --location $VECTORS_URL - build: + - UNZIPPED_VECTORS_DIR = $CODEBUILD_SRC_DIR/test_vector_handlers/tmp/net_401_vectors + - unzip $VECTOR_ZIP -d $UNZIPPED_VECTORS_DIR commands: # NOTE: We need to pass the absolute path of the vectors - pip install "tox < 4.0" - cd $CODEBUILD_SRC_DIR/test_vector_handlers - | tox -- \ - --input $VECTOR_ZIP + --input $UNZIPPED_VECTORS_DIR From 71efaa3653297103df4cd7aee0f53dbdd8183a96 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 13:40:53 -0700 Subject: [PATCH 228/376] debug cb --- codebuild/py37/decrypt_net_401_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_net_401_vectors.yml b/codebuild/py37/decrypt_net_401_vectors.yml index 59e703024..8b388463f 100644 --- a/codebuild/py37/decrypt_net_401_vectors.yml +++ b/codebuild/py37/decrypt_net_401_vectors.yml @@ -38,7 +38,7 @@ phases: - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip - curl -s --output $VECTOR_ZIP --location $VECTORS_URL - - UNZIPPED_VECTORS_DIR = $CODEBUILD_SRC_DIR/test_vector_handlers/tmp/net_401_vectors + - UNZIPPED_VECTORS_DIR=$CODEBUILD_SRC_DIR/test_vector_handlers/tmp/net_401_vectors - unzip $VECTOR_ZIP -d $UNZIPPED_VECTORS_DIR commands: # NOTE: We need to pass the absolute path of the vectors From b1cbf4b69886a1170fb885abc589508a03afc180 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 13:45:47 -0700 Subject: [PATCH 229/376] debug gha and cb --- codebuild/py37/decrypt_net_401_vectors.yml | 2 +- .../manifests/full_message/decrypt_generation.py | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/codebuild/py37/decrypt_net_401_vectors.yml b/codebuild/py37/decrypt_net_401_vectors.yml index 8b388463f..0c04225a6 100644 --- a/codebuild/py37/decrypt_net_401_vectors.yml +++ b/codebuild/py37/decrypt_net_401_vectors.yml @@ -38,7 +38,7 @@ phases: - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip - curl -s --output $VECTOR_ZIP --location $VECTORS_URL - - UNZIPPED_VECTORS_DIR=$CODEBUILD_SRC_DIR/test_vector_handlers/tmp/net_401_vectors + - UNZIPPED_VECTORS_DIR=$CODEBUILD_SRC_DIR/test_vector_handlers/net_401_vectors - unzip $VECTOR_ZIP -d $UNZIPPED_VECTORS_DIR commands: # NOTE: We need to pass the absolute path of the vectors diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index c7cc2be90..a746ac127 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -191,7 +191,7 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, _p cmm = CryptoMaterialsManagerFromMPL(mpl_cmm=mpl_cmm) else: raise TypeError(f"Unrecognized master_key_provider type: {master_key_provider}") - + return [ self.run_scenario_with_new_provider_info( ciphertext_writer, generation_scenario, cmm, new_provider_info @@ -203,10 +203,18 @@ def run_scenario_with_new_provider_info( self, ciphertext_writer, generation_scenario, materials_manager, new_provider_info ): """Run with tampering for a specific new provider info value""" - if _HAS_MPL and isinstance(materials_manager, CryptoMaterialsManagerFromMPL): - tampering_materials_manager = ProviderInfoChangingCryptoMaterialsManagerFromMPL(materials_manager, new_provider_info) + if isinstance(materials_manager, CryptoMaterialsManager): + tampering_materials_manager = ProviderInfoChangingCryptoMaterialsManager( + materials_manager, + new_provider_info + ) + elif _HAS_MPL and isinstance(materials_manager, CryptoMaterialsManagerFromMPL): + tampering_materials_manager = ProviderInfoChangingCryptoMaterialsManagerFromMPL( + materials_manager, + new_provider_info + ) else: - tampering_materials_manager = ProviderInfoChangingCryptoMaterialsManager(materials_manager, new_provider_info) + raise TypeError(f"Unrecognized materials_manager type: {materials_manager}") ciphertext_to_decrypt = generation_scenario.encryption_scenario.run(tampering_materials_manager) expected_result = MessageDecryptionTestResult.expect_error( "Incorrect encrypted data key provider info: " + new_provider_info From 39fcb9b44308602430c5e289ffa72d0fc66cad35 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 13:49:16 -0700 Subject: [PATCH 230/376] debug gha and cb --- codebuild/py37/decrypt_net_401_vectors.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/codebuild/py37/decrypt_net_401_vectors.yml b/codebuild/py37/decrypt_net_401_vectors.yml index 0c04225a6..3711e6130 100644 --- a/codebuild/py37/decrypt_net_401_vectors.yml +++ b/codebuild/py37/decrypt_net_401_vectors.yml @@ -40,6 +40,7 @@ phases: - curl -s --output $VECTOR_ZIP --location $VECTORS_URL - UNZIPPED_VECTORS_DIR=$CODEBUILD_SRC_DIR/test_vector_handlers/net_401_vectors - unzip $VECTOR_ZIP -d $UNZIPPED_VECTORS_DIR + build: commands: # NOTE: We need to pass the absolute path of the vectors - pip install "tox < 4.0" From 5962312488c7243893fc704be7cb9534ffad1c8b Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 13:50:17 -0700 Subject: [PATCH 231/376] debug gha and cb --- .../manifests/full_message/decrypt_generation.py | 2 +- .../src/awses_test_vectors/manifests/full_message/encrypt.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index a746ac127..cef786335 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -253,7 +253,7 @@ def get_encryption_materials(self, request): def decrypt_materials(self, request): """Thunks to the wrapped CMM""" return self.wrapped_cmm.decrypt_materials(request) - + BITS_PER_BYTE = 8 diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 6343b7044..c1ffcdaa0 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -22,6 +22,8 @@ import aws_encryption_sdk import six +from aws_encryption_sdk.key_providers.base import MasterKeyProvider + from awses_test_vectors.internal.defaults import ENCODING from awses_test_vectors.internal.util import ( algorithm_suite_from_string_id, @@ -31,7 +33,6 @@ membership_validator, validate_manifest_type, ) -from aws_encryption_sdk.key_providers.base import MasterKeyProvider from awses_test_vectors.manifests.keys import KeysManifest from awses_test_vectors.manifests.master_key import MasterKeySpec, master_key_provider_from_master_key_specs @@ -45,7 +46,7 @@ from aws_cryptographic_materialproviders.mpl.references import ( IKeyring, ) - + from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_from_master_key_specs _HAS_MPL = True From 0ac5e96d55453204e66cd98f3202f9d0acfe17a2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 13:51:33 -0700 Subject: [PATCH 232/376] debug gha and cb --- codebuild/py37/decrypt_net_401_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_net_401_vectors.yml b/codebuild/py37/decrypt_net_401_vectors.yml index 3711e6130..8188c5ede 100644 --- a/codebuild/py37/decrypt_net_401_vectors.yml +++ b/codebuild/py37/decrypt_net_401_vectors.yml @@ -47,4 +47,4 @@ phases: - cd $CODEBUILD_SRC_DIR/test_vector_handlers - | tox -- \ - --input $UNZIPPED_VECTORS_DIR + --input $UNZIPPED_VECTORS_DIR/manifest.json From 3dcab7fdba982f5ddc9481b46e91d9f23a5f8081 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 13:57:10 -0700 Subject: [PATCH 233/376] debug gha and cb --- src/aws_encryption_sdk/internal/formatting/deserialize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/internal/formatting/deserialize.py b/src/aws_encryption_sdk/internal/formatting/deserialize.py index b06b5ba11..1b46a8b7d 100644 --- a/src/aws_encryption_sdk/internal/formatting/deserialize.py +++ b/src/aws_encryption_sdk/internal/formatting/deserialize.py @@ -475,7 +475,7 @@ def deserialize_frame(stream, header, verifier=None): frame_data["iv"] = frame_iv if final_frame is True: (content_length,) = unpack_values(">I", stream, verifier) - if content_length >= header.frame_length: + if content_length > header.frame_length: raise SerializationError( "Invalid final frame length: {final} >= {normal}".format( final=content_length, normal=header.frame_length From ccb01a2e1d97ec374054c8ea08ce849383e46e48 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 14:59:25 -0700 Subject: [PATCH 234/376] debug cb --- codebuild/py37/decrypt_net_401_vectors.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/codebuild/py37/decrypt_net_401_vectors.yml b/codebuild/py37/decrypt_net_401_vectors.yml index 8188c5ede..5b3925890 100644 --- a/codebuild/py37/decrypt_net_401_vectors.yml +++ b/codebuild/py37/decrypt_net_401_vectors.yml @@ -19,11 +19,6 @@ phases: install: runtime-versions: python: 3.7 - commands: - # Get Dafny - - curl https://github.com/dafny-lang/dafny/releases/download/v4.2.0/dafny-4.2.0-x64-ubuntu-20.04.zip -L -o dafny.zip - - unzip -qq dafny.zip && rm dafny.zip - - export PATH="$PWD/dafny:$PATH" pre_build: commands: # Assume Role to access non-prod resources From 7074f8adc114f8ed904cb37874dd0689da8bb864 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 15:02:57 -0700 Subject: [PATCH 235/376] add missing --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 codebuild/py37/decrypt_dafny_esdk_vectors.yml diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml new file mode 100644 index 000000000..a66c6e2d6 --- /dev/null +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -0,0 +1,60 @@ +version: 0.2 +# Runs Only the ESDK-NET v4.0.1 Decryption Vectors, testing Required EC CMM + +env: + variables: + TOXENV: "py37-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + python: 3.7 + pre_build: + commands: + # Fetch test vectors from Dafny ESDK's most recent run + # (Assuming the first result is most recent; seems to be correct) + - | + MOST_RECENT_RUN_ID=curl -L \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ + | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)' + - | + MOST_RECENT_RUN_DOWNLOAD_URL=curl -L \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" \ + | jq '.artifacts[0].archive_download_url' + - | + curl -L \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }} \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "$MOST_RECENT_RUN_DOWNLOAD_URL" + - unzip ubuntu-latest_test_vector_artifact + + # Assume Role to access non-prod resources + - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") + - export TMP_ROLE + - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') + - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') + - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') + - aws sts get-caller-identity + build: + commands: + # NOTE: We need to pass the absolute path of the vectors + - pip install "tox < 4.0" + - cd $CODEBUILD_SRC_DIR/test_vector_handlers + - | + tox -- \ + --input $UNZIPPED_VECTORS_DIR/manifest.json From 2b36513d6324db308fd2035f550ec05d9754723d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 15:55:20 -0700 Subject: [PATCH 236/376] token --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 18 ++++++++++++++---- .../py37/decrypt_masterkey_with_masterkey.yml | 5 +++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index a66c6e2d6..178d16895 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -3,6 +3,9 @@ version: 0.2 env: variables: + git-credential-helper: yes + secrets-manager: + GH_TOKEN: Github/aws-crypto-tools-ci-bot:personal access token TOXENV: "py37-full_decrypt" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f @@ -21,25 +24,32 @@ phases: python: 3.7 pre_build: commands: + # Authenticate into the CI bot to allow session to download ESDK Dafny GHA artifact + - git config --global user.name "aws-crypto-tools-ci-bot" + - git config --global user.email "no-reply@noemail.local" + - echo $GH_TOKEN > token.txt + # Blank out the token; we're done with it + # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct) - | - MOST_RECENT_RUN_ID=curl -L \ + MOST_RECENT_RUN_ID=curl \ -H "Accept: application/vnd.github+json" \ -H "X-GitHub-Api-Version: 2022-11-28" \ "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)' - | - MOST_RECENT_RUN_DOWNLOAD_URL=curl -L \ + MOST_RECENT_RUN_DOWNLOAD_URL=curl \ -H "Accept: application/vnd.github+json" \ -H "X-GitHub-Api-Version: 2022-11-28" \ "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" \ | jq '.artifacts[0].archive_download_url' - | - curl -L \ + curl \ -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }} \ + -H "Authorization: Bearer $GH_TOKEN \ -H "X-GitHub-Api-Version: 2022-11-28" \ + -o ubuntu-latest_test_vector_artifact.zip "$MOST_RECENT_RUN_DOWNLOAD_URL" - unzip ubuntu-latest_test_vector_artifact diff --git a/codebuild/py37/decrypt_masterkey_with_masterkey.yml b/codebuild/py37/decrypt_masterkey_with_masterkey.yml index df7067e60..357482e1e 100644 --- a/codebuild/py37/decrypt_masterkey_with_masterkey.yml +++ b/codebuild/py37/decrypt_masterkey_with_masterkey.yml @@ -22,5 +22,6 @@ phases: commands: - pip install "tox < 4.0" - cd test_vector_handlers - - tox -- \ - --input ../tmp/generated/37_masterkey + - | + tox -- \ + --input ../tmp/generated/37_masterkey From be6a25363ea74a5fc67d86d37cd06f09438d0dd5 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 15:58:16 -0700 Subject: [PATCH 237/376] token --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 178d16895..1f765daf2 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -3,9 +3,6 @@ version: 0.2 env: variables: - git-credential-helper: yes - secrets-manager: - GH_TOKEN: Github/aws-crypto-tools-ci-bot:personal access token TOXENV: "py37-full_decrypt" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f @@ -17,6 +14,9 @@ env: arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + git-credential-helper: yes + secrets-manager: + GH_TOKEN: Github/aws-crypto-tools-ci-bot:personal access token phases: install: From a431365e91224bdfaf13169b3fff49a6b45458b2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 16:13:19 -0700 Subject: [PATCH 238/376] debug --- codebuild/py37/decrypt_masterkey_with_masterkey.yml | 2 +- codebuild/py37/generate_decrypt_vectors.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/codebuild/py37/decrypt_masterkey_with_masterkey.yml b/codebuild/py37/decrypt_masterkey_with_masterkey.yml index 357482e1e..8e6f916f5 100644 --- a/codebuild/py37/decrypt_masterkey_with_masterkey.yml +++ b/codebuild/py37/decrypt_masterkey_with_masterkey.yml @@ -24,4 +24,4 @@ phases: - cd test_vector_handlers - | tox -- \ - --input ../tmp/generated/37_masterkey + --input tmp/generated/37_masterkey/manifest.json \ No newline at end of file diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml index 04a09a47d..873aac2e6 100644 --- a/codebuild/py37/generate_decrypt_vectors.yml +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -25,4 +25,4 @@ phases: - | tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ - --output ../tmp/generated/37_masterkey + --output tmp/generated/37_masterkey From 4120be3b1ec3f46f4a5e291aaa236f0fa7e73350 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 16:33:26 -0700 Subject: [PATCH 239/376] debug cb --- buildspec.yml | 176 +++++++++++++++++++++++++------------------------- 1 file changed, 88 insertions(+), 88 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index ca87c41c5..3d0f7d684 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -150,94 +150,94 @@ batch: # env: # image: aws/codebuild/standard:5.0 - # - identifier: py311_integ - # buildspec: codebuild/py311/integ.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_integ_mpl - # buildspec: codebuild/py311/integ_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_examples - # buildspec: codebuild/py311/examples.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_examples_mpl - # buildspec: codebuild/py311/examples_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_awses_latest - # buildspec: codebuild/py311/awses_local.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_awses_latest_mpl - # buildspec: codebuild/py311/awses_local_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_mplawses_latest_mpl - # buildspec: codebuild/py311/mplawses_local_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_dafny_esdk_vectors_masterkey - # buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py311_decrypt_dafny_esdk_vectors_keyrings - # buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py311_decrypt_net_401_vectors_masterkey - # buildspec: codebuild/py311/decrypt_net_401_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py311_decrypt_net_401_vectors_keyrings - # buildspec: codebuild/py311/decrypt_net_401_vectors_keyrings.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py311_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py311/generate_decrypt_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py311_decrypt_masterkey_with_masterkey - # depend-on: - # - py311_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py311/decrypt_masterkey_with_masterkey.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py311_decrypt_masterkey_with_keyrings - # depend-on: - # - py311_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py311/decrypt_masterkey_with_keyrings.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py311_decrypt_masterkey_with_js - # depend-on: - # - py311_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py311/decrypt_masterkey_with_js.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py311_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py311/generate_decrypt_vectors_keyrings.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py311_decrypt_keyrings_with_masterkey - # depend-on: - # - py311_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py311/decrypt_keyrings_with_masterkey.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py311_decrypt_keyrings_with_keyrings - # depend-on: - # - py311_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py311/decrypt_keyrings_with_keyrings.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py311_decrypt_keyrings_with_js - # depend-on: - # - py311_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py311/decrypt_keyrings_with_js.yml - # env: - # image: aws/codebuild/standard:5.0 + - identifier: py311_integ + buildspec: codebuild/py311/integ.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_integ_mpl + buildspec: codebuild/py311/integ_mpl.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_examples + buildspec: codebuild/py311/examples.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_examples_mpl + buildspec: codebuild/py311/examples_mpl.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_awses_latest + buildspec: codebuild/py311/awses_local.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_awses_latest_mpl + buildspec: codebuild/py311/awses_local_mpl.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_mplawses_latest_mpl + buildspec: codebuild/py311/mplawses_local_mpl.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_dafny_esdk_vectors_masterkey + buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_dafny_esdk_vectors_keyrings + buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_net_401_vectors_masterkey + buildspec: codebuild/py311/decrypt_net_401_vectors_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_net_401_vectors_keyrings + buildspec: codebuild/py311/decrypt_net_401_vectors_keyrings.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_generate_decrypt_vectors_masterkey + buildspec: codebuild/py311/generate_decrypt_vectors_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_masterkey_with_masterkey + depend-on: + - py311_generate_decrypt_vectors_masterkey + buildspec: codebuild/py311/decrypt_masterkey_with_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_masterkey_with_keyrings + depend-on: + - py311_generate_decrypt_vectors_masterkey + buildspec: codebuild/py311/decrypt_masterkey_with_keyrings.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_masterkey_with_js + depend-on: + - py311_generate_decrypt_vectors_masterkey + buildspec: codebuild/py311/decrypt_masterkey_with_js.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_generate_decrypt_vectors_keyrings + buildspec: codebuild/py311/generate_decrypt_vectors_keyrings.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_keyrings_with_masterkey + depend-on: + - py311_generate_decrypt_vectors_keyrings + buildspec: codebuild/py311/decrypt_keyrings_with_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_keyrings_with_keyrings + depend-on: + - py311_generate_decrypt_vectors_keyrings + buildspec: codebuild/py311/decrypt_keyrings_with_keyrings.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_keyrings_with_js + depend-on: + - py311_generate_decrypt_vectors_keyrings + buildspec: codebuild/py311/decrypt_keyrings_with_js.yml + env: + image: aws/codebuild/standard:5.0 # - identifier: py312_integ # buildspec: codebuild/py312/integ.yml From 7391c783ef77073020bf7e50c4f1612207b6330e Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 16:36:11 -0700 Subject: [PATCH 240/376] missing --- .../decrypt_net_401_vectors_keyrings.yml | 46 +++++++++++++++++++ .../decrypt_net_401_vectors_masterkey.yml | 45 ++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 codebuild/py311/decrypt_net_401_vectors_keyrings.yml create mode 100644 codebuild/py311/decrypt_net_401_vectors_masterkey.yml diff --git a/codebuild/py311/decrypt_net_401_vectors_keyrings.yml b/codebuild/py311/decrypt_net_401_vectors_keyrings.yml new file mode 100644 index 000000000..cccf5eda6 --- /dev/null +++ b/codebuild/py311/decrypt_net_401_vectors_keyrings.yml @@ -0,0 +1,46 @@ +version: 0.2 +# Runs Only the ESDK-NET v4.0.1 Decryption Vectors, testing Required EC CMM + +env: + variables: + TOXENV: "py311-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + python: 3.11 + pre_build: + commands: + # Assume Role to access non-prod resources + - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") + - export TMP_ROLE + - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') + - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') + - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') + - aws sts get-caller-identity + + # Fetch ESDK .NET v4.0.1 Test Vectors + - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip + - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip + - curl -s --output $VECTOR_ZIP --location $VECTORS_URL + - UNZIPPED_VECTORS_DIR=$CODEBUILD_SRC_DIR/test_vector_handlers/net_401_vectors + - unzip $VECTOR_ZIP -d $UNZIPPED_VECTORS_DIR + build: + commands: + # NOTE: We need to pass the absolute path of the vectors + - pip install "tox < 4.0" + - cd $CODEBUILD_SRC_DIR/test_vector_handlers + - | + tox -- \ + --input $UNZIPPED_VECTORS_DIR/manifest.json \ + --keyrings diff --git a/codebuild/py311/decrypt_net_401_vectors_masterkey.yml b/codebuild/py311/decrypt_net_401_vectors_masterkey.yml new file mode 100644 index 000000000..f6f0482e7 --- /dev/null +++ b/codebuild/py311/decrypt_net_401_vectors_masterkey.yml @@ -0,0 +1,45 @@ +version: 0.2 +# Runs Only the ESDK-NET v4.0.1 Decryption Vectors, testing Required EC CMM + +env: + variables: + TOXENV: "py311-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + python: 3.11 + pre_build: + commands: + # Assume Role to access non-prod resources + - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") + - export TMP_ROLE + - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') + - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') + - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') + - aws sts get-caller-identity + + # Fetch ESDK .NET v4.0.1 Test Vectors + - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip + - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip + - curl -s --output $VECTOR_ZIP --location $VECTORS_URL + - UNZIPPED_VECTORS_DIR=$CODEBUILD_SRC_DIR/test_vector_handlers/net_401_vectors + - unzip $VECTOR_ZIP -d $UNZIPPED_VECTORS_DIR + build: + commands: + # NOTE: We need to pass the absolute path of the vectors + - pip install "tox < 4.0" + - cd $CODEBUILD_SRC_DIR/test_vector_handlers + - | + tox -- \ + --input $UNZIPPED_VECTORS_DIR/manifest.json \ No newline at end of file From a16be01f77f05bba6ba7864c748d22f709b593e4 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 16:39:15 -0700 Subject: [PATCH 241/376] missing --- buildspec.yml | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index 3d0f7d684..db168cc78 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -181,63 +181,63 @@ batch: - identifier: py311_decrypt_dafny_esdk_vectors_masterkey buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml env: - image: aws/codebuild/standard:5.0 + image: aws/codebuild/standard:7.0 - identifier: py311_decrypt_dafny_esdk_vectors_keyrings buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml env: - image: aws/codebuild/standard:5.0 + image: aws/codebuild/standard:7.0 - identifier: py311_decrypt_net_401_vectors_masterkey buildspec: codebuild/py311/decrypt_net_401_vectors_masterkey.yml env: - image: aws/codebuild/standard:5.0 + image: aws/codebuild/standard:7.0 - identifier: py311_decrypt_net_401_vectors_keyrings buildspec: codebuild/py311/decrypt_net_401_vectors_keyrings.yml env: - image: aws/codebuild/standard:5.0 + image: aws/codebuild/standard:7.0 - identifier: py311_generate_decrypt_vectors_masterkey buildspec: codebuild/py311/generate_decrypt_vectors_masterkey.yml env: - image: aws/codebuild/standard:5.0 + image: aws/codebuild/standard:7.0 - identifier: py311_decrypt_masterkey_with_masterkey depend-on: - py311_generate_decrypt_vectors_masterkey buildspec: codebuild/py311/decrypt_masterkey_with_masterkey.yml env: - image: aws/codebuild/standard:5.0 + image: aws/codebuild/standard:7.0 - identifier: py311_decrypt_masterkey_with_keyrings depend-on: - py311_generate_decrypt_vectors_masterkey buildspec: codebuild/py311/decrypt_masterkey_with_keyrings.yml env: - image: aws/codebuild/standard:5.0 + image: aws/codebuild/standard:7.0 - identifier: py311_decrypt_masterkey_with_js depend-on: - py311_generate_decrypt_vectors_masterkey buildspec: codebuild/py311/decrypt_masterkey_with_js.yml env: - image: aws/codebuild/standard:5.0 + image: aws/codebuild/standard:7.0 - identifier: py311_generate_decrypt_vectors_keyrings buildspec: codebuild/py311/generate_decrypt_vectors_keyrings.yml env: - image: aws/codebuild/standard:5.0 + image: aws/codebuild/standard:7.0 - identifier: py311_decrypt_keyrings_with_masterkey depend-on: - py311_generate_decrypt_vectors_keyrings buildspec: codebuild/py311/decrypt_keyrings_with_masterkey.yml env: - image: aws/codebuild/standard:5.0 + image: aws/codebuild/standard:7.0 - identifier: py311_decrypt_keyrings_with_keyrings depend-on: - py311_generate_decrypt_vectors_keyrings buildspec: codebuild/py311/decrypt_keyrings_with_keyrings.yml env: - image: aws/codebuild/standard:5.0 + image: aws/codebuild/standard:7.0 - identifier: py311_decrypt_keyrings_with_js depend-on: - py311_generate_decrypt_vectors_keyrings buildspec: codebuild/py311/decrypt_keyrings_with_js.yml env: - image: aws/codebuild/standard:5.0 + image: aws/codebuild/standard:7.0 # - identifier: py312_integ # buildspec: codebuild/py312/integ.yml @@ -270,63 +270,63 @@ batch: # - identifier: py312_decrypt_dafny_esdk_vectors_masterkey # buildspec: codebuild/py312/decrypt_dafny_esdk_vectors_masterkey.yml # env: - # image: aws/codebuild/standard:5.0 + # image: aws/codebuild/standard:7.0 # - identifier: py312_decrypt_dafny_esdk_vectors_keyrings # buildspec: codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml # env: - # image: aws/codebuild/standard:5.0 + # image: aws/codebuild/standard:7.0 # - identifier: py312_decrypt_net_401_vectors_masterkey # buildspec: codebuild/py312/decrypt_net_401_vectors_masterkey.yml # env: - # image: aws/codebuild/standard:5.0 + # image: aws/codebuild/standard:7.0 # - identifier: py312_decrypt_net_401_vectors_keyrings # buildspec: codebuild/py312/decrypt_net_401_vectors_keyrings.yml # env: - # image: aws/codebuild/standard:5.0 + # image: aws/codebuild/standard:7.0 # - identifier: py312_generate_decrypt_vectors_masterkey # buildspec: codebuild/py312/generate_decrypt_vectors_masterkey.yml # env: - # image: aws/codebuild/standard:5.0 + # image: aws/codebuild/standard:7.0 # - identifier: py312_decrypt_masterkey_with_masterkey # depend-on: # - py312_generate_decrypt_vectors_masterkey # buildspec: codebuild/py312/decrypt_masterkey_with_masterkey.yml # env: - # image: aws/codebuild/standard:5.0 + # image: aws/codebuild/standard:7.0 # - identifier: py312_decrypt_masterkey_with_keyrings # depend-on: # - py312_generate_decrypt_vectors_masterkey # buildspec: codebuild/py312/decrypt_masterkey_with_keyrings.yml # env: - # image: aws/codebuild/standard:5.0 + # image: aws/codebuild/standard:7.0 # - identifier: py312_decrypt_masterkey_with_js # depend-on: # - py312_generate_decrypt_vectors_masterkey # buildspec: codebuild/py312/decrypt_masterkey_with_js.yml # env: - # image: aws/codebuild/standard:5.0 + # image: aws/codebuild/standard:7.0 # - identifier: py312_generate_decrypt_vectors_keyrings # buildspec: codebuild/py312/generate_decrypt_vectors_keyrings.yml # env: - # image: aws/codebuild/standard:5.0 + # image: aws/codebuild/standard:7.0 # - identifier: py312_decrypt_keyrings_with_masterkey # depend-on: # - py312_generate_decrypt_vectors_keyrings # buildspec: codebuild/py312/decrypt_keyrings_with_masterkey.yml # env: - # image: aws/codebuild/standard:5.0 + # image: aws/codebuild/standard:7.0 # - identifier: py312_decrypt_keyrings_with_keyrings # depend-on: # - py312_generate_decrypt_vectors_keyrings # buildspec: codebuild/py312/decrypt_keyrings_with_keyrings.yml # env: - # image: aws/codebuild/standard:5.0 + # image: aws/codebuild/standard:7.0 # - identifier: py312_decrypt_keyrings_with_js # depend-on: # - py312_generate_decrypt_vectors_keyrings # buildspec: codebuild/py312/decrypt_keyrings_with_js.yml # env: - # image: aws/codebuild/standard:5.0 + # image: aws/codebuild/standard:7.0 # - identifier: code_coverage # buildspec: codebuild/coverage/coverage.yml From 4b2553be0f77628ed83673b629c75d3d52e304a4 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 16:41:05 -0700 Subject: [PATCH 242/376] perms --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 1f765daf2..8478d1c62 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -24,6 +24,14 @@ phases: python: 3.7 pre_build: commands: + # Assume Role to access non-prod resources + - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") + - export TMP_ROLE + - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') + - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') + - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') + - aws sts get-caller-identity + # Authenticate into the CI bot to allow session to download ESDK Dafny GHA artifact - git config --global user.name "aws-crypto-tools-ci-bot" - git config --global user.email "no-reply@noemail.local" @@ -52,14 +60,6 @@ phases: -o ubuntu-latest_test_vector_artifact.zip "$MOST_RECENT_RUN_DOWNLOAD_URL" - unzip ubuntu-latest_test_vector_artifact - - # Assume Role to access non-prod resources - - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") - - export TMP_ROLE - - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') - - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') - - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') - - aws sts get-caller-identity build: commands: # NOTE: We need to pass the absolute path of the vectors From 843ac28c519e5bce05fdc0a322e50ae0b06e101d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 16:50:27 -0700 Subject: [PATCH 243/376] debug cb --- codebuild/py311/decrypt_net_401_vectors_keyrings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py311/decrypt_net_401_vectors_keyrings.yml b/codebuild/py311/decrypt_net_401_vectors_keyrings.yml index cccf5eda6..2b17002e5 100644 --- a/codebuild/py311/decrypt_net_401_vectors_keyrings.yml +++ b/codebuild/py311/decrypt_net_401_vectors_keyrings.yml @@ -3,7 +3,7 @@ version: 0.2 env: variables: - TOXENV: "py311-full_decrypt" + TOXENV: "py311-full_decrypt-mpl" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- From 68c068ecb12931f13d80f16c594156500ca88ab8 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 17:02:37 -0700 Subject: [PATCH 244/376] debug cb --- codebuild/py311/decrypt_net_401_vectors_keyrings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py311/decrypt_net_401_vectors_keyrings.yml b/codebuild/py311/decrypt_net_401_vectors_keyrings.yml index 2b17002e5..6634470c3 100644 --- a/codebuild/py311/decrypt_net_401_vectors_keyrings.yml +++ b/codebuild/py311/decrypt_net_401_vectors_keyrings.yml @@ -21,7 +21,7 @@ phases: python: 3.11 pre_build: commands: - # Assume Role to access non-prod resources + # Assume Role to access non-prod resource - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") - export TMP_ROLE - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') From adf7198bd25d1c7dac13499353dbfbddc5ef32ca Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 11:49:25 -0700 Subject: [PATCH 245/376] files in s3 --- codebuild/py37/generate_decrypt_vectors.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml index 873aac2e6..5ca7d1a4a 100644 --- a/codebuild/py37/generate_decrypt_vectors.yml +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -26,3 +26,6 @@ phases: tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ --output tmp/generated/37_masterkey +artifacts: + files: + - tmp/generated/37_masterkey/**/* \ No newline at end of file From e2464637c3f2db1b04d3941e417edf731b099975 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 11:55:54 -0700 Subject: [PATCH 246/376] files in s3 --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 8478d1c62..9536aa771 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -57,7 +57,7 @@ phases: -H "Accept: application/vnd.github+json" \ -H "Authorization: Bearer $GH_TOKEN \ -H "X-GitHub-Api-Version: 2022-11-28" \ - -o ubuntu-latest_test_vector_artifact.zip + -o ubuntu-latest_test_vector_artifact.zip \ "$MOST_RECENT_RUN_DOWNLOAD_URL" - unzip ubuntu-latest_test_vector_artifact build: From 1aa07e5f1f515349ea27a944bd8d5233b707b343 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 11:58:54 -0700 Subject: [PATCH 247/376] files in s3 --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 9536aa771..bbb58ec30 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -55,7 +55,7 @@ phases: - | curl \ -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $GH_TOKEN \ + -H "Authorization: Bearer $GH_TOKEN" \ -H "X-GitHub-Api-Version: 2022-11-28" \ -o ubuntu-latest_test_vector_artifact.zip \ "$MOST_RECENT_RUN_DOWNLOAD_URL" From 1702822607422ce657abd6df0b6e6149b8119e4e Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 12:01:35 -0700 Subject: [PATCH 248/376] files in s3 --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index bbb58ec30..21bee5367 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -36,10 +36,10 @@ phases: - git config --global user.name "aws-crypto-tools-ci-bot" - git config --global user.email "no-reply@noemail.local" - echo $GH_TOKEN > token.txt - # Blank out the token; we're done with it # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct) + - curl -h - | MOST_RECENT_RUN_ID=curl \ -H "Accept: application/vnd.github+json" \ From 207401aae1660a21b30440472fd9d5b856025d40 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 12:06:36 -0700 Subject: [PATCH 249/376] files in s3 --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 21bee5367..9c9232138 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -39,20 +39,19 @@ phases: # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct) - - curl -h - - | - MOST_RECENT_RUN_ID=curl \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ + - > + MOST_RECENT_RUN_ID=curl + -H "Accept: application/vnd.github+json" + -H "X-GitHub-Api-Version: 2022-11-28" + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)' - - | + - > MOST_RECENT_RUN_DOWNLOAD_URL=curl \ -H "Accept: application/vnd.github+json" \ -H "X-GitHub-Api-Version: 2022-11-28" \ "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" \ | jq '.artifacts[0].archive_download_url' - - | + - > curl \ -H "Accept: application/vnd.github+json" \ -H "Authorization: Bearer $GH_TOKEN" \ From b3e0125577718ab1393399c3c970b13fad8f6823 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 12:09:04 -0700 Subject: [PATCH 250/376] files in s3 --- codebuild/py37/generate_decrypt_vectors.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml index 5ca7d1a4a..be85d73e7 100644 --- a/codebuild/py37/generate_decrypt_vectors.yml +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -25,7 +25,7 @@ phases: - | tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ - --output tmp/generated/37_masterkey + --output 37_masterkey artifacts: files: - - tmp/generated/37_masterkey/**/* \ No newline at end of file + - 37_masterkey/**/* \ No newline at end of file From 644c343d52db17ed87a04e7f27a0114d6eb556a8 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 12:10:37 -0700 Subject: [PATCH 251/376] files in s3 --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 25 +++++-------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 9c9232138..3d921e3fd 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -39,25 +39,12 @@ phases: # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct) - - > - MOST_RECENT_RUN_ID=curl - -H "Accept: application/vnd.github+json" - -H "X-GitHub-Api-Version: 2022-11-28" - "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" - | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)' - - > - MOST_RECENT_RUN_DOWNLOAD_URL=curl \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" \ - | jq '.artifacts[0].archive_download_url' - - > - curl \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $GH_TOKEN" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -o ubuntu-latest_test_vector_artifact.zip \ - "$MOST_RECENT_RUN_DOWNLOAD_URL" + - | + MOST_RECENT_RUN_ID=curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)' + - | + MOST_RECENT_RUN_DOWNLOAD_URL=curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url' + - | + curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GH_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip "$MOST_RECENT_RUN_DOWNLOAD_URL" - unzip ubuntu-latest_test_vector_artifact build: commands: From e72ab997647f67d3734becc808f4c22ea74163e4 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 13:18:47 -0700 Subject: [PATCH 252/376] files in s3 --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 3d921e3fd..991e3b50d 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -40,9 +40,9 @@ phases: # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct) - | - MOST_RECENT_RUN_ID=curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)' + MOST_RECENT_RUN_ID='curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq "first(.workflow_runs[] | select(.name=="Daily CI") | .id)"' - | - MOST_RECENT_RUN_DOWNLOAD_URL=curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url' + MOST_RECENT_RUN_DOWNLOAD_URL='curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq ".artifacts[0].archive_download_url"' - | curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GH_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip "$MOST_RECENT_RUN_DOWNLOAD_URL" - unzip ubuntu-latest_test_vector_artifact From 36fd56b97a9a288e592543442942eed3bd4d36f6 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 13:27:40 -0700 Subject: [PATCH 253/376] files in s3 --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 991e3b50d..3ee14d3c0 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -41,8 +41,12 @@ phases: # (Assuming the first result is most recent; seems to be correct) - | MOST_RECENT_RUN_ID='curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq "first(.workflow_runs[] | select(.name=="Daily CI") | .id)"' + - | + echo $MOST_RECENT_RUN_ID - | MOST_RECENT_RUN_DOWNLOAD_URL='curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq ".artifacts[0].archive_download_url"' + - | + echo $MOST_RECENT_RUN_DOWNLOAD_URL - | curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GH_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip "$MOST_RECENT_RUN_DOWNLOAD_URL" - unzip ubuntu-latest_test_vector_artifact From 6052b53fee0c225b3d74753ad1ee37b9a577a27d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 13:30:42 -0700 Subject: [PATCH 254/376] files in s3 --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 3ee14d3c0..a8973776f 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -40,11 +40,11 @@ phases: # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct) - | - MOST_RECENT_RUN_ID='curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq "first(.workflow_runs[] | select(.name=="Daily CI") | .id)"' + MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq "first(.workflow_runs[] | select(.name=="Daily CI") | .id)") - | echo $MOST_RECENT_RUN_ID - | - MOST_RECENT_RUN_DOWNLOAD_URL='curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq ".artifacts[0].archive_download_url"' + MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq ".artifacts[0].archive_download_url") - | echo $MOST_RECENT_RUN_DOWNLOAD_URL - | From 457aa8836f04f63830454c1061404406814475a2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 13:38:51 -0700 Subject: [PATCH 255/376] files in s3 --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index a8973776f..8a2483cd6 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -40,11 +40,11 @@ phases: # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct) - | - MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq "first(.workflow_runs[] | select(.name=="Daily CI") | .id)") + MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') - | echo $MOST_RECENT_RUN_ID - | - MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq ".artifacts[0].archive_download_url") + MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url') - | echo $MOST_RECENT_RUN_DOWNLOAD_URL - | From a8b65d3e5809637bd11fe6854f618b92616cf798 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 13:41:36 -0700 Subject: [PATCH 256/376] files in s3 --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 8a2483cd6..d2463e1b7 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -48,7 +48,7 @@ phases: - | echo $MOST_RECENT_RUN_DOWNLOAD_URL - | - curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GH_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip "$MOST_RECENT_RUN_DOWNLOAD_URL" + curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GH_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip $MOST_RECENT_RUN_DOWNLOAD_URL - unzip ubuntu-latest_test_vector_artifact build: commands: From 8ed6cca93c01bbf37f6c8c99ef380c28f6f66c5f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 13:48:02 -0700 Subject: [PATCH 257/376] debug gen --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 4 ++-- codebuild/py37/generate_decrypt_vectors.yml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index d2463e1b7..9294501e9 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -42,11 +42,11 @@ phases: - | MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') - | - echo $MOST_RECENT_RUN_ID + echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" - | MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url') - | - echo $MOST_RECENT_RUN_DOWNLOAD_URL + echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" - | curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GH_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip $MOST_RECENT_RUN_DOWNLOAD_URL - unzip ubuntu-latest_test_vector_artifact diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml index be85d73e7..517544815 100644 --- a/codebuild/py37/generate_decrypt_vectors.yml +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -26,6 +26,8 @@ phases: tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ --output 37_masterkey + - ls + - zip 37_master.zip 37_masterkey artifacts: files: - 37_masterkey/**/* \ No newline at end of file From ef4a9d7605d90718e4a458e4456618cbc5622b57 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 13:51:01 -0700 Subject: [PATCH 258/376] debug gen --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 9294501e9..1433ed114 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -48,7 +48,7 @@ phases: - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" - | - curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GH_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip $MOST_RECENT_RUN_DOWNLOAD_URL + curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip $MOST_RECENT_RUN_DOWNLOAD_URL - unzip ubuntu-latest_test_vector_artifact build: commands: From b564f77150d1d633869b88cab68b31a6d7ccca3f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 13:58:20 -0700 Subject: [PATCH 259/376] debug gen --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 1433ed114..d5e94a7d5 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -44,7 +44,7 @@ phases: - | echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" - | - MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url') + MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url[8:]') - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" - | From 31a58ccea0f86f8e4f2d657e855693ab8bc1deed Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 14:01:42 -0700 Subject: [PATCH 260/376] debug gen --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index d5e94a7d5..00195a437 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -44,11 +44,11 @@ phases: - | echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" - | - MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url[8:]') + MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url') - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" - | - curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip $MOST_RECENT_RUN_DOWNLOAD_URL + curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" $MOST_RECENT_RUN_DOWNLOAD_URL -o ubuntu-latest_test_vector_artifact.zip - unzip ubuntu-latest_test_vector_artifact build: commands: From ed879537d06f4bcbb2599a66a73d5197cabaf6f0 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 14:03:46 -0700 Subject: [PATCH 261/376] debug gen --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 00195a437..9ce80d4a8 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -48,7 +48,7 @@ phases: - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" - | - curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" $MOST_RECENT_RUN_DOWNLOAD_URL -o ubuntu-latest_test_vector_artifact.zip + $(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" $MOST_RECENT_RUN_DOWNLOAD_URL -o ubuntu-latest_test_vector_artifact.zip) - unzip ubuntu-latest_test_vector_artifact build: commands: From ac01f37c0b66962d13ff668323491e16ab8744ca Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 14:05:59 -0700 Subject: [PATCH 262/376] debug gen --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 9ce80d4a8..9ed28658d 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -48,7 +48,7 @@ phases: - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" - | - $(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" $MOST_RECENT_RUN_DOWNLOAD_URL -o ubuntu-latest_test_vector_artifact.zip) + $(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/artifacts/1326417479/zip" -o ubuntu-latest_test_vector_artifact.zip) - unzip ubuntu-latest_test_vector_artifact build: commands: From 69a934c36560b148f2fffa5fb73c45530902d458 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 14:09:58 -0700 Subject: [PATCH 263/376] debug gen --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 9ed28658d..99ee498b9 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -48,7 +48,7 @@ phases: - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" - | - $(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/artifacts/1326417479/zip" -o ubuntu-latest_test_vector_artifact.zip) + curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GH_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/artifacts/1326417479/zip" - unzip ubuntu-latest_test_vector_artifact build: commands: From 65d3acdd5e5e19d46bea860e7814064898cf01ea Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 14:12:44 -0700 Subject: [PATCH 264/376] debug gen --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 99ee498b9..c6401837d 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -38,7 +38,7 @@ phases: - echo $GH_TOKEN > token.txt # Fetch test vectors from Dafny ESDK's most recent run - # (Assuming the first result is most recent; seems to be correct) + # (Assuming the first result is most recent; seems to be correct...) - | MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') - | From 95e8a8bb804ea646065de280884057803914e4ad Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 14:16:39 -0700 Subject: [PATCH 265/376] debug gen --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index c6401837d..99ee498b9 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -38,7 +38,7 @@ phases: - echo $GH_TOKEN > token.txt # Fetch test vectors from Dafny ESDK's most recent run - # (Assuming the first result is most recent; seems to be correct...) + # (Assuming the first result is most recent; seems to be correct) - | MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') - | From 8d484e60cf97f6876ffcc5957c18dad4309af041 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 14:23:58 -0700 Subject: [PATCH 266/376] debug gen --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 99ee498b9..e0580a45e 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -40,7 +40,7 @@ phases: # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct) - | - MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') + MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true") - | echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" - | From 8dcfc3cfd8733960836505d2c6ff10b612826240 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 14:25:56 -0700 Subject: [PATCH 267/376] debug gen --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index e0580a45e..99ee498b9 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -40,7 +40,7 @@ phases: # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct) - | - MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true") + MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') - | echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" - | From a9306bc1a0f9a347d819d73a7fd61175ffce7838 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 14:38:25 -0700 Subject: [PATCH 268/376] debug gen --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 99ee498b9..dbdb1387c 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -35,10 +35,11 @@ phases: # Authenticate into the CI bot to allow session to download ESDK Dafny GHA artifact - git config --global user.name "aws-crypto-tools-ci-bot" - git config --global user.email "no-reply@noemail.local" - - echo $GH_TOKEN > token.txt + - | + echo "DEBUG: $GH_TOKEN" # Fetch test vectors from Dafny ESDK's most recent run - # (Assuming the first result is most recent; seems to be correct) + # (Assuming the first result is most recent; seems to be correct...) - | MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') - | @@ -48,7 +49,7 @@ phases: - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" - | - curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GH_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/artifacts/1326417479/zip" + curl -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GH_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip $MOST_RECENT_RUN_DOWNLOAD_URL - unzip ubuntu-latest_test_vector_artifact build: commands: From 894dcee5a2c3e81b78e03493f7db5a7d3a5aac88 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 14:45:49 -0700 Subject: [PATCH 269/376] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index dbdb1387c..3f1a7afee 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -35,15 +35,15 @@ phases: # Authenticate into the CI bot to allow session to download ESDK Dafny GHA artifact - git config --global user.name "aws-crypto-tools-ci-bot" - git config --global user.email "no-reply@noemail.local" - - | - echo "DEBUG: $GH_TOKEN" # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct...) - | - MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') + MOST_RECENT_RUN_STUFF=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') + - | + echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_STUFF" - | - echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" + MOST_RECENT_RUN_ID=$(echo $MOST_RECENT_RUN_STUFF | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') - | MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url') - | From e00ec2ac6aae2f67ac0addcd88949e3b631656f0 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 14:46:15 -0700 Subject: [PATCH 270/376] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 3f1a7afee..03cd87ddf 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -39,7 +39,7 @@ phases: # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct...) - | - MOST_RECENT_RUN_STUFF=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') + MOST_RECENT_RUN_STUFF=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true") - | echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_STUFF" - | From 2f23be33ef1494a6e15854684dbc7990f62947f9 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 14:49:15 -0700 Subject: [PATCH 271/376] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 03cd87ddf..85f9190db 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -35,6 +35,10 @@ phases: # Authenticate into the CI bot to allow session to download ESDK Dafny GHA artifact - git config --global user.name "aws-crypto-tools-ci-bot" - git config --global user.email "no-reply@noemail.local" + - echo $GH_TOKEN > token.txt + gh auth login --with-token < token.txt + rm token.txt + gh auth status # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct...) From 6a7d73250152a54efe9c0bc5d9c86f5971f1c843 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 14:50:06 -0700 Subject: [PATCH 272/376] debug --- codebuild/py37/generate_decrypt_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml index 517544815..f0d4d36a0 100644 --- a/codebuild/py37/generate_decrypt_vectors.yml +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -30,4 +30,4 @@ phases: - zip 37_master.zip 37_masterkey artifacts: files: - - 37_masterkey/**/* \ No newline at end of file + - 37_master.zip \ No newline at end of file From 8cbd3dcd128dfd56964463608b6031735ae38070 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 14:51:28 -0700 Subject: [PATCH 273/376] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 85f9190db..bce139705 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -36,9 +36,9 @@ phases: - git config --global user.name "aws-crypto-tools-ci-bot" - git config --global user.email "no-reply@noemail.local" - echo $GH_TOKEN > token.txt - gh auth login --with-token < token.txt - rm token.txt - gh auth status + - gh auth login --with-token < token.txt + - rm token.txt + - gh auth status # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct...) From 5cb4b13d6cbeb201f767e46c94d2ed4ac8f2407c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 14:55:27 -0700 Subject: [PATCH 274/376] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index bce139705..5f51f8f9a 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -36,6 +36,11 @@ phases: - git config --global user.name "aws-crypto-tools-ci-bot" - git config --global user.email "no-reply@noemail.local" - echo $GH_TOKEN > token.txt + + - type -p yum-config-manager >/dev/null || sudo yum install yum-utils + - sudo yum-config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo + - sudo yum install gh + - gh auth login --with-token < token.txt - rm token.txt - gh auth status From 9d89a742e449ec220ca274de0c4cae7ab45eb618 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 15:03:05 -0700 Subject: [PATCH 275/376] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 5f51f8f9a..e5f733da9 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -37,9 +37,7 @@ phases: - git config --global user.email "no-reply@noemail.local" - echo $GH_TOKEN > token.txt - - type -p yum-config-manager >/dev/null || sudo yum install yum-utils - - sudo yum-config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo - - sudo yum install gh + - sudo apt install gh -y - gh auth login --with-token < token.txt - rm token.txt From 51bf320208d14d3e3d260957b7d48daabc181f35 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 15:05:23 -0700 Subject: [PATCH 276/376] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index e5f733da9..0d608638b 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -37,7 +37,12 @@ phases: - git config --global user.email "no-reply@noemail.local" - echo $GH_TOKEN > token.txt - - sudo apt install gh -y + - | + sudo mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ + && sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ + && sudo apt update \ + && sudo apt install gh -y - gh auth login --with-token < token.txt - rm token.txt From 3be4969cc23f479877ae585254a55946b7bc1bff Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 15:08:07 -0700 Subject: [PATCH 277/376] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 0d608638b..1b3f9a6f6 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -44,7 +44,7 @@ phases: && sudo apt update \ && sudo apt install gh -y - - gh auth login --with-token < token.txt + - gh auth login - rm token.txt - gh auth status From 9e3358dd9cfa861bfe17b3b88ddc0a3cbb696a5a Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 15:10:39 -0700 Subject: [PATCH 278/376] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 1b3f9a6f6..5bd760579 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -44,20 +44,18 @@ phases: && sudo apt update \ && sudo apt install gh -y - - gh auth login - - rm token.txt - gh auth status # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct...) - | - MOST_RECENT_RUN_STUFF=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true") + MOST_RECENT_RUN_STUFF=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GH_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true") - | echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_STUFF" - | MOST_RECENT_RUN_ID=$(echo $MOST_RECENT_RUN_STUFF | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') - | - MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url') + MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GH_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url') - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" - | From 1b7a54b564542e51e86fb51da67cf26bf104ca84 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 15:21:26 -0700 Subject: [PATCH 279/376] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 5bd760579..75ed3f1a4 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -16,7 +16,7 @@ env: AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" git-credential-helper: yes secrets-manager: - GH_TOKEN: Github/aws-crypto-tools-ci-bot:personal access token + GH_TOKEN: Github/aws-crypto-tools-ci-bot:personal access token (new format) phases: install: From ce59f5777b0d3f37287c4cfd201a70a725b52fd8 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 15:22:26 -0700 Subject: [PATCH 280/376] debug --- codebuild/py37/generate_decrypt_vectors.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml index f0d4d36a0..0d5b18085 100644 --- a/codebuild/py37/generate_decrypt_vectors.yml +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -27,6 +27,8 @@ phases: --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ --output 37_masterkey - ls + - cd 37_masterkey + - ls - zip 37_master.zip 37_masterkey artifacts: files: From 6ef093b63b47785512056a340a269e38bb2799e0 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 15:24:12 -0700 Subject: [PATCH 281/376] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 75ed3f1a4..3e0726b66 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -16,7 +16,7 @@ env: AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" git-credential-helper: yes secrets-manager: - GH_TOKEN: Github/aws-crypto-tools-ci-bot:personal access token (new format) + GH_TOKEN: Github/aws-crypto-tools-ci-bot:personal access token (new token format) phases: install: From ce07e87f3d73d49a9e65ed017d8393505274eb0e Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 15:32:44 -0700 Subject: [PATCH 282/376] Debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 3e0726b66..53aa50c27 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -16,7 +16,7 @@ env: AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" git-credential-helper: yes secrets-manager: - GH_TOKEN: Github/aws-crypto-tools-ci-bot:personal access token (new token format) + GITHUB_TOKEN: Github/aws-crypto-tools-ci-bot:personal access token (new token format) phases: install: @@ -44,8 +44,6 @@ phases: && sudo apt update \ && sudo apt install gh -y - - gh auth status - # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct...) - | From 7225e51bdd76d19714bc192016596655d59161a1 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 15:34:16 -0700 Subject: [PATCH 283/376] debug --- codebuild/py37/generate_decrypt_vectors.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml index 0d5b18085..cdad0861d 100644 --- a/codebuild/py37/generate_decrypt_vectors.yml +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -27,9 +27,7 @@ phases: --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ --output 37_masterkey - ls - - cd 37_masterkey - - ls - - zip 37_master.zip 37_masterkey + - zip -r 37_master.zip 37_masterkey artifacts: files: - 37_master.zip \ No newline at end of file From 549fe07537ba69f36d8e9593c160bf0ef1839136 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 15:38:27 -0700 Subject: [PATCH 284/376] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 53aa50c27..3d5316873 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -35,7 +35,6 @@ phases: # Authenticate into the CI bot to allow session to download ESDK Dafny GHA artifact - git config --global user.name "aws-crypto-tools-ci-bot" - git config --global user.email "no-reply@noemail.local" - - echo $GH_TOKEN > token.txt - | sudo mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ @@ -47,17 +46,17 @@ phases: # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct...) - | - MOST_RECENT_RUN_STUFF=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GH_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true") + MOST_RECENT_RUN_STUFF=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true") - | echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_STUFF" - | MOST_RECENT_RUN_ID=$(echo $MOST_RECENT_RUN_STUFF | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') - | - MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GH_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url') + MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url') - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" - | - curl -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GH_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip $MOST_RECENT_RUN_DOWNLOAD_URL + curl -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip $MOST_RECENT_RUN_DOWNLOAD_URL - unzip ubuntu-latest_test_vector_artifact build: commands: From 0f9b66258b12429c0e3f8bedf227fd90ab3120c2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 15:41:36 -0700 Subject: [PATCH 285/376] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 3d5316873..f5dc3126f 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -46,17 +46,17 @@ phases: # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct...) - | - MOST_RECENT_RUN_STUFF=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true") + MOST_RECENT_RUN_STUFF=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: token $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true") - | echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_STUFF" - | MOST_RECENT_RUN_ID=$(echo $MOST_RECENT_RUN_STUFF | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') - | - MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url') + MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: token $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url') - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" - | - curl -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip $MOST_RECENT_RUN_DOWNLOAD_URL + curl -L -H "Accept: application/vnd.github+json" -H "Authorization: token $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip $MOST_RECENT_RUN_DOWNLOAD_URL - unzip ubuntu-latest_test_vector_artifact build: commands: From 8dd346883f3f58abfc9d59d844a7d6456198eee0 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 15:45:45 -0700 Subject: [PATCH 286/376] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index f5dc3126f..5aae54440 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -46,7 +46,7 @@ phases: # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct...) - | - MOST_RECENT_RUN_STUFF=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: token $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true") + MOST_RECENT_RUN_STUFF=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: token $(echo ${GITHUB_TOKEN})" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true") - | echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_STUFF" - | From 5378b6282b71c0c3598af44c9b1661279438eaea Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 15:47:59 -0700 Subject: [PATCH 287/376] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 5aae54440..0313c9702 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -36,17 +36,17 @@ phases: - git config --global user.name "aws-crypto-tools-ci-bot" - git config --global user.email "no-reply@noemail.local" - - | - sudo mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ - && sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ - && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ - && sudo apt update \ - && sudo apt install gh -y + # - | + # sudo mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ + # && sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ + # && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ + # && sudo apt update \ + # && sudo apt install gh -y # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct...) - | - MOST_RECENT_RUN_STUFF=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: token $(echo ${GITHUB_TOKEN})" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true") + MOST_RECENT_RUN_STUFF=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: token ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true") - | echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_STUFF" - | From f3db08a8c5454697558208dbad700baa9245c32e Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 16:20:41 -0700 Subject: [PATCH 288/376] debug --- codebuild/py37/generate_decrypt_vectors.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml index cdad0861d..b379377e9 100644 --- a/codebuild/py37/generate_decrypt_vectors.yml +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -30,4 +30,5 @@ phases: - zip -r 37_master.zip 37_masterkey artifacts: files: - - 37_master.zip \ No newline at end of file + - ./37_master.zip + name: 37_master.zip \ No newline at end of file From 6080556c38432f248f7b5b4fc26de1debfd5b9dc Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 16:47:43 -0700 Subject: [PATCH 289/376] debug --- codebuild/py37/generate_decrypt_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml index b379377e9..aac3d3f38 100644 --- a/codebuild/py37/generate_decrypt_vectors.yml +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -30,5 +30,5 @@ phases: - zip -r 37_master.zip 37_masterkey artifacts: files: - - ./37_master.zip + - test_vector_handlers/37_master.zip name: 37_master.zip \ No newline at end of file From 38623291b4277c6c2777de70511fd458c0fc0ea6 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 09:03:59 -0700 Subject: [PATCH 290/376] consume vectors --- codebuild/py37/decrypt_masterkey_with_js.yml | 6 +++++- codebuild/py37/decrypt_masterkey_with_masterkey.yml | 10 +++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/codebuild/py37/decrypt_masterkey_with_js.yml b/codebuild/py37/decrypt_masterkey_with_js.yml index 32db1083e..fe2f93535 100644 --- a/codebuild/py37/decrypt_masterkey_with_js.yml +++ b/codebuild/py37/decrypt_masterkey_with_js.yml @@ -33,7 +33,11 @@ phases: - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') - aws sts get-caller-identity - cd $CODEBUILD_SRC_DIR + + # Download generated vectors + # TODO rewrite URL + aws s3 cp s3://generated-vectors-artifacts-bucket/py37_generate_decrypt_vectors/test_vector_handlers/37_master.zip 37_master.zip build: commands: # Decrypt generated vectors with Javascript ESDK - - integration-node decrypt -v ../tmp/generated/37_masterkey \ No newline at end of file + - integration-node decrypt -v 37_master.zip \ No newline at end of file diff --git a/codebuild/py37/decrypt_masterkey_with_masterkey.yml b/codebuild/py37/decrypt_masterkey_with_masterkey.yml index 8e6f916f5..5a2347e8f 100644 --- a/codebuild/py37/decrypt_masterkey_with_masterkey.yml +++ b/codebuild/py37/decrypt_masterkey_with_masterkey.yml @@ -18,10 +18,18 @@ phases: install: runtime-versions: python: 3.7 + + + pre-build: + commands: + # Download generated vectors + # TODO rewrite URL + aws s3 cp s3://generated-vectors-artifacts-bucket/py37_generate_decrypt_vectors/test_vector_handlers/37_master.zip 37_masterkey.zip + unzip 37_master.zip build: commands: - pip install "tox < 4.0" - cd test_vector_handlers - | tox -- \ - --input tmp/generated/37_masterkey/manifest.json \ No newline at end of file + --input ../37_masterkey/manifest.json \ No newline at end of file From 30f71529f5a26a05247bd2ce305731d096e87a09 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 09:29:57 -0700 Subject: [PATCH 291/376] rerun ci --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 0313c9702..f7636d33a 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -32,7 +32,7 @@ phases: - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') - aws sts get-caller-identity - # Authenticate into the CI bot to allow session to download ESDK Dafny GHA artifact + # Authenticate into the CI bot to allow session to download ESDK Dafny GHA artifact. - git config --global user.name "aws-crypto-tools-ci-bot" - git config --global user.email "no-reply@noemail.local" From 876ed384f9894a9f65de4386fbee7938e8448d5f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 09:44:25 -0700 Subject: [PATCH 292/376] add missing files --- .../generate_decrypt_vectors_keyrings.yml | 29 +++++++++++++++++++ .../generate_decrypt_vectors_masterkey.yml | 28 ++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 codebuild/py311/generate_decrypt_vectors_keyrings.yml create mode 100644 codebuild/py311/generate_decrypt_vectors_masterkey.yml diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml new file mode 100644 index 000000000..081d944c5 --- /dev/null +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -0,0 +1,29 @@ +version: 0.2 + +env: + variables: + TOXENV: "py311-full_decrypt_generate-mpl" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + python: 3.11 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ + --output tmp/generated/37_masterkey + diff --git a/codebuild/py311/generate_decrypt_vectors_masterkey.yml b/codebuild/py311/generate_decrypt_vectors_masterkey.yml new file mode 100644 index 000000000..873aac2e6 --- /dev/null +++ b/codebuild/py311/generate_decrypt_vectors_masterkey.yml @@ -0,0 +1,28 @@ +version: 0.2 + +env: + variables: + TOXENV: "py37-full_decrypt_generate" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + python: 3.7 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ + --output tmp/generated/37_masterkey From 8a6bf33060d346519451a298409fb807e804866f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 09:46:35 -0700 Subject: [PATCH 293/376] 311 --- codebuild/py311/generate_decrypt_vectors_keyrings.yml | 9 +++++++-- codebuild/py311/generate_decrypt_vectors_masterkey.yml | 7 ++++++- codebuild/py37/generate_decrypt_vectors.yml | 1 - 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml index 081d944c5..1c9f3514a 100644 --- a/codebuild/py311/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -25,5 +25,10 @@ phases: - | tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ - --output tmp/generated/37_masterkey - + --output 311_keyring \ + --keyrings + - zip -r 311_keyring.zip 311_keyring + artifacts: + files: + - test_vector_handlers/311_keyring.zip + name: 311_keyring.zip \ No newline at end of file diff --git a/codebuild/py311/generate_decrypt_vectors_masterkey.yml b/codebuild/py311/generate_decrypt_vectors_masterkey.yml index 873aac2e6..69cbe418b 100644 --- a/codebuild/py311/generate_decrypt_vectors_masterkey.yml +++ b/codebuild/py311/generate_decrypt_vectors_masterkey.yml @@ -25,4 +25,9 @@ phases: - | tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ - --output tmp/generated/37_masterkey + --output 311_masterkey + - zip -r 311_masterkey.zip 311_masterkey + artifacts: + files: + - test_vector_handlers/311_masterkey.zip + name: 311_masterkey.zip \ No newline at end of file diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml index aac3d3f38..6c80a9a1c 100644 --- a/codebuild/py37/generate_decrypt_vectors.yml +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -26,7 +26,6 @@ phases: tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ --output 37_masterkey - - ls - zip -r 37_master.zip 37_masterkey artifacts: files: From 07f3b8f08dd8be8ed3eb4b1e1ca2147f410a81ad Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 09:58:50 -0700 Subject: [PATCH 294/376] cooking --- codebuild/py37/decrypt_masterkey_with_js.yml | 2 +- codebuild/py37/generate_decrypt_vectors.yml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/codebuild/py37/decrypt_masterkey_with_js.yml b/codebuild/py37/decrypt_masterkey_with_js.yml index fe2f93535..22ca730b7 100644 --- a/codebuild/py37/decrypt_masterkey_with_js.yml +++ b/codebuild/py37/decrypt_masterkey_with_js.yml @@ -36,7 +36,7 @@ phases: # Download generated vectors # TODO rewrite URL - aws s3 cp s3://generated-vectors-artifacts-bucket/py37_generate_decrypt_vectors/test_vector_handlers/37_master.zip 37_master.zip + aws s3 cp s3://generated-vectors-artifacts-bucket/GeneratedVectors/py37_generate_decrypt_vectors/test_vector_handlers/37_master.zip 37_master.zip build: commands: # Decrypt generated vectors with Javascript ESDK diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml index 6c80a9a1c..6e578e101 100644 --- a/codebuild/py37/generate_decrypt_vectors.yml +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -30,4 +30,5 @@ phases: artifacts: files: - test_vector_handlers/37_master.zip - name: 37_master.zip \ No newline at end of file + name: $CODEBUILD_BATCH_BUILD_IDENTIFIER/37_master.zip + discard-paths: true From a06684e6da126a991fb8c008fa1bc32624a71fe2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 10:06:16 -0700 Subject: [PATCH 295/376] cooking --- codebuild/py311/generate_decrypt_vectors_keyrings.yml | 9 +++++---- codebuild/py311/generate_decrypt_vectors_masterkey.yml | 9 +++++---- codebuild/py37/generate_decrypt_vectors.yml | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml index 1c9f3514a..f83ac733e 100644 --- a/codebuild/py311/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -28,7 +28,8 @@ phases: --output 311_keyring \ --keyrings - zip -r 311_keyring.zip 311_keyring - artifacts: - files: - - test_vector_handlers/311_keyring.zip - name: 311_keyring.zip \ No newline at end of file +artifacts: + files: + - test_vector_handlers/311_keyring.zip + name: $CODEBUILD_BATCH_BUILD_IDENTIFIER/311_keyring.zip + discard-paths: yes \ No newline at end of file diff --git a/codebuild/py311/generate_decrypt_vectors_masterkey.yml b/codebuild/py311/generate_decrypt_vectors_masterkey.yml index 69cbe418b..0975aa78c 100644 --- a/codebuild/py311/generate_decrypt_vectors_masterkey.yml +++ b/codebuild/py311/generate_decrypt_vectors_masterkey.yml @@ -27,7 +27,8 @@ phases: --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ --output 311_masterkey - zip -r 311_masterkey.zip 311_masterkey - artifacts: - files: - - test_vector_handlers/311_masterkey.zip - name: 311_masterkey.zip \ No newline at end of file +artifacts: + files: + - test_vector_handlers/311_masterkey.zip + name: $CODEBUILD_BATCH_BUILD_IDENTIFIER/311_masterkey.zip + discard-paths: yes \ No newline at end of file diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml index 6e578e101..7aad81e13 100644 --- a/codebuild/py37/generate_decrypt_vectors.yml +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -31,4 +31,4 @@ artifacts: files: - test_vector_handlers/37_master.zip name: $CODEBUILD_BATCH_BUILD_IDENTIFIER/37_master.zip - discard-paths: true + discard-paths: yes From 37fd225cd49a1c1da1dcf3e94e8dcac6905ad459 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 10:11:05 -0700 Subject: [PATCH 296/376] cooking --- codebuild/py311/generate_decrypt_vectors_masterkey.yml | 2 +- .../manifests/full_message/decrypt_generation.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/codebuild/py311/generate_decrypt_vectors_masterkey.yml b/codebuild/py311/generate_decrypt_vectors_masterkey.yml index 0975aa78c..848a19c92 100644 --- a/codebuild/py311/generate_decrypt_vectors_masterkey.yml +++ b/codebuild/py311/generate_decrypt_vectors_masterkey.yml @@ -17,7 +17,7 @@ env: phases: install: runtime-versions: - python: 3.7 + python: 3.11 build: commands: - pip install "tox < 4.0" diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index cef786335..782404704 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -50,7 +50,8 @@ ) _HAS_MPL = True -except ImportError: +except ImportError as e: + print(f"decrypt_generation ImportError: {e}") _HAS_MPL = False From e4590459b5f129a78f3579f9945ad7d0cb543366 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 10:15:33 -0700 Subject: [PATCH 297/376] cooking --- codebuild/py311/generate_decrypt_vectors_masterkey.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py311/generate_decrypt_vectors_masterkey.yml b/codebuild/py311/generate_decrypt_vectors_masterkey.yml index 848a19c92..655fb1985 100644 --- a/codebuild/py311/generate_decrypt_vectors_masterkey.yml +++ b/codebuild/py311/generate_decrypt_vectors_masterkey.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - TOXENV: "py37-full_decrypt_generate" + TOXENV: "py311-full_decrypt_generate" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- From 0e8c6c914495b0c4563de67190a9e0a4fd15be4c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 10:16:31 -0700 Subject: [PATCH 298/376] missing --- .../internal/tampering_mpl_materials.py | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 test_vector_handlers/src/awses_test_vectors/internal/tampering_mpl_materials.py diff --git a/test_vector_handlers/src/awses_test_vectors/internal/tampering_mpl_materials.py b/test_vector_handlers/src/awses_test_vectors/internal/tampering_mpl_materials.py new file mode 100644 index 000000000..4ad948424 --- /dev/null +++ b/test_vector_handlers/src/awses_test_vectors/internal/tampering_mpl_materials.py @@ -0,0 +1,108 @@ +"""Allows overriding the algorithm and signing_key for EncryptionMaterialsFromMPL. +This must ONLY be used in testing and NOT in production.. +This is used in message tampering testing. +""" +import attr +import six + +from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager + +# Ignore missing MPL for pylint, but the MPL is required for this class +# pylint: disable=import-error,no-name-in-module +from aws_encryption_sdk.materials_managers.mpl.materials import ( + EncryptionMaterialsFromMPL +) +from aws_encryption_sdk.materials_managers.mpl.cmm import ( + CryptoMaterialsManagerFromMPL +) + + +class HalfSigningEncryptionMaterialsFromMPL(EncryptionMaterialsFromMPL): + """Allows overriding the algorithm and signing_key for EncryptionMaterialsFromMPL. + This must ONLY be used in testing and NOT in production.. + This is used in testing malicious message modification (HalfSigningTampering). + """ + + _underlying_materials: EncryptionMaterialsFromMPL + + def __init__(self, underling_materials): + self._underlying_materials = underling_materials + + # pylint thinks EncryptionMaterialsFromMPL.algorithm is a method + # pylint: disable=invalid-overridden-method + @property + def algorithm(self): + """Return any previously-provided overriden algorithm; + if none was provided, returns underlying algorithm from encryption materials. + """ + if hasattr(self, "set_algorithm"): + return self.set_algorithm + return self._underlying_materials.algorithm + + @algorithm.setter + def algorithm(self, algorithm): + self.set_algorithm = algorithm + + # pylint thinks EncryptionMaterialsFromMPL.signing_key is a method + # pylint: disable=invalid-overridden-method + @property + def signing_key(self): + """Return any previously-provided overriden signing_key; + if none was provided, returns underlying signing_key from encryption materials. + """ + if hasattr(self, "set_signing_key"): + return self.set_signing_key + return self._underlying_materials.algorithm + + @signing_key.setter + def signing_key(self, signing_key): + self.set_signing_key = signing_key + + @property + def encryption_context(self): + return self._underlying_materials.encryption_context + + @property + def encrypted_data_keys(self): + return self._underlying_materials.encrypted_data_keys + + @property + def data_encryption_key(self): + return self._underlying_materials.data_encryption_key + + @property + def required_encryption_context_keys(self): + return self._underlying_materials.required_encryption_context_keys + + +class ProviderInfoChangingCryptoMaterialsManagerFromMPL(CryptoMaterialsManagerFromMPL): + """ + Custom CMM that modifies the provider info field on EDKS. + This extends CryptoMaterialsManagerFromMPL so ESDK-internal checks + follow MPL logic. + + THIS IS ONLY USED TO CREATE INVALID MESSAGES and should never be used in + production! + """ + + wrapped_cmm = attr.ib(validator=attr.validators.instance_of(CryptoMaterialsManager)) + new_provider_info = attr.ib(validator=attr.validators.instance_of(six.string_types)) + + def __init__(self, materials_manager, new_provider_info): + """Create a new CMM that wraps a the given CMM.""" + self.wrapped_cmm = materials_manager + self.new_provider_info = new_provider_info + + def get_encryption_materials(self, request): + """ + Request materials from the wrapped CMM, and then change the provider info + on each EDK. + """ + result = self.wrapped_cmm.get_encryption_materials(request) + for encrypted_data_key in result.encrypted_data_keys: + encrypted_data_key.key_provider.key_info = self.new_provider_info + return result + + def decrypt_materials(self, request): + """Thunks to the wrapped CMM""" + return self.wrapped_cmm.decrypt_materials(request) From 32446e2f2026751e1c1feb75a66dc785d91b7476 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 10:29:15 -0700 Subject: [PATCH 299/376] tampering mpl --- .../internal/tampering_mpl_materials.py | 64 ++++++++++++++++++- .../full_message/decrypt_generation.py | 31 ++++----- 2 files changed, 79 insertions(+), 16 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/internal/tampering_mpl_materials.py b/test_vector_handlers/src/awses_test_vectors/internal/tampering_mpl_materials.py index 4ad948424..1bb6705fc 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/tampering_mpl_materials.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/tampering_mpl_materials.py @@ -4,6 +4,8 @@ """ import attr import six +from copy import copy + from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager @@ -15,6 +17,66 @@ from aws_encryption_sdk.materials_managers.mpl.cmm import ( CryptoMaterialsManagerFromMPL ) +from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders +from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig +from aws_cryptographic_materialproviders.mpl.models import ( + CreateDefaultCryptographicMaterialsManagerInput, +) + +try: + from aws_encryption_sdk.identifiers import AlgorithmSuite +except ImportError: + from aws_encryption_sdk.identifiers import Algorithm as AlgorithmSuite + +class HalfSigningCryptoMaterialsManagerFromMPL(CryptoMaterialsManagerFromMPL): + """ + Custom CMM that modifies the provider info field on EDKs + This extends CryptoMaterialsManagerFromMPL so ESDK-internal checks + follow MPL logic. + + THIS IS ONLY USED TO CREATE INVALID MESSAGES and should never be used in + production! + """ + + wrapped_default_cmm = attr.ib(validator=attr.validators.instance_of(CryptoMaterialsManagerFromMPL)) + + def __init__(self, master_key_provider): + """Create a new CMM that wraps a the given CMM.""" + mpl = AwsCryptographicMaterialProviders(MaterialProvidersConfig()) + mpl_cmm = mpl.create_default_cryptographic_materials_manager( + CreateDefaultCryptographicMaterialsManagerInput( + keyring=master_key_provider + ) + ) + self.wrapped_default_cmm = CryptoMaterialsManagerFromMPL(mpl_cmm=mpl_cmm) + + def get_encryption_materials(self, request): + """ + Generate half-signing materials by requesting signing materials + from the wrapped default CMM, and then changing the algorithm suite + and removing the signing key from teh result. + """ + if request.algorithm == AlgorithmSuite.AES_256_GCM_HKDF_SHA512_COMMIT_KEY: + signing_request = copy(request) + signing_request.algorithm = AlgorithmSuite.AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384 + + result = HalfSigningEncryptionMaterialsFromMPL( + self.wrapped_default_cmm.get_encryption_materials(signing_request) + ) + + result.algorithm = request.algorithm + result.signing_key = None + + return result + + raise NotImplementedError( + "The half-sign tampering method is only supported on the " + "AES_256_GCM_HKDF_SHA512_COMMIT_KEY algorithm suite." + ) + + def decrypt_materials(self, request): + """Thunks to the wrapped default CMM""" + return self.wrapped_default_cmm.decrypt_materials(request) class HalfSigningEncryptionMaterialsFromMPL(EncryptionMaterialsFromMPL): @@ -77,7 +139,7 @@ def required_encryption_context_keys(self): class ProviderInfoChangingCryptoMaterialsManagerFromMPL(CryptoMaterialsManagerFromMPL): """ - Custom CMM that modifies the provider info field on EDKS. + Custom CMM that modifies the provider info field on EDKs. This extends CryptoMaterialsManagerFromMPL so ESDK-internal checks follow MPL logic. diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 782404704..e2fcc0f65 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -47,6 +47,7 @@ from awses_test_vectors.internal.tampering_mpl_materials import ( HalfSigningEncryptionMaterialsFromMPL, ProviderInfoChangingCryptoMaterialsManagerFromMPL, + HalfSigningCryptoMaterialsManagerFromMPL, ) _HAS_MPL = True @@ -319,9 +320,20 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, _p return: a list of (ciphertext, result) pairs. """ - tampering_materials_manager = HalfSigningCryptoMaterialsManager( - generation_scenario.encryption_scenario.master_key_provider_fn() - ) + if isinstance( + generation_scenario.encryption_scenario.master_key_provider_fn(), + MasterKeyProvider + ): + tampering_materials_manager = HalfSigningCryptoMaterialsManager( + generation_scenario.encryption_scenario.master_key_provider_fn() + ) + elif _HAS_MPL and isinstance( + generation_scenario.encryption_scenario.master_key_provider_fn(), + IKeyring + ): + tampering_materials_manager = HalfSigningCryptoMaterialsManagerFromMPL( + generation_scenario.encryption_scenario.master_key_provider_fn() + ) ciphertext_to_decrypt = generation_scenario.encryption_scenario.run(tampering_materials_manager) expected_result = MessageDecryptionTestResult.expect_error( "Unsigned message using a data key with a public key" @@ -349,18 +361,7 @@ def __init__(self, master_key_provider): Create a new CMM that wraps a new DefaultCryptoMaterialsManager based on the given master key provider. """ - if isinstance(master_key_provider, MasterKeyProvider): - self.wrapped_default_cmm = DefaultCryptoMaterialsManager(master_key_provider) - elif _HAS_MPL and isinstance(master_key_provider, IKeyring): - mpl = AwsCryptographicMaterialProviders(MaterialProvidersConfig()) - mpl_cmm = mpl.create_default_cryptographic_materials_manager( - CreateDefaultCryptographicMaterialsManagerInput( - keyring=master_key_provider - ) - ) - self.wrapped_default_cmm = CryptoMaterialsManagerFromMPL(mpl_cmm=mpl_cmm) - else: - raise TypeError(f"Unrecognized master_key_provider type: {master_key_provider}") + self.wrapped_default_cmm = DefaultCryptoMaterialsManager(master_key_provider) def get_encryption_materials(self, request): """ From f1cd456c4801bdc31300190736b836317f426169 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 10:53:49 -0700 Subject: [PATCH 300/376] more --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index f7636d33a..cc82757de 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -16,7 +16,7 @@ env: AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" git-credential-helper: yes secrets-manager: - GITHUB_TOKEN: Github/aws-crypto-tools-ci-bot:personal access token (new token format) + GITHUB_TOKEN: Github/lucasmcdonald3:actions:read fine-grained PAT phases: install: From a3267bc1d57df45a36d15a3b2a9a41f694dcc72b Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 10:56:43 -0700 Subject: [PATCH 301/376] more --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index cc82757de..62fa6b7e1 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -16,7 +16,7 @@ env: AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" git-credential-helper: yes secrets-manager: - GITHUB_TOKEN: Github/lucasmcdonald3:actions:read fine-grained PAT + GITHUB_TOKEN: Github/lucasmcdonald3:actions\:read fine-grained PAT phases: install: From d4db5ec6fcce95422cf0ac8a68fb44885e9d2d8d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 10:59:18 -0700 Subject: [PATCH 302/376] more --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 62fa6b7e1..939e42185 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -16,7 +16,7 @@ env: AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" git-credential-helper: yes secrets-manager: - GITHUB_TOKEN: Github/lucasmcdonald3:actions\:read fine-grained PAT + GITHUB_TOKEN: "Github/lucasmcdonald3:actions:read fine-grained PAT" phases: install: From bf78061292d3cf1594a22ef96c72011a561de17d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 11:10:06 -0700 Subject: [PATCH 303/376] more --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 939e42185..dc5732552 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -16,7 +16,7 @@ env: AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" git-credential-helper: yes secrets-manager: - GITHUB_TOKEN: "Github/lucasmcdonald3:actions:read fine-grained PAT" + GITHUB_TOKEN: Github/lucasmcdonald3-fgpat:actions read phases: install: From 008ae6ff5f8ebfa26c07192ad0a77ae6a5f01649 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 11:59:25 -0700 Subject: [PATCH 304/376] more --- src/aws_encryption_sdk/streaming_client.py | 1 + .../manifests/full_message/decrypt_generation.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index fb0935ff2..54ce76235 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -582,6 +582,7 @@ def _prep_message(self): else: # MPL verification key is PEM bytes, not DER bytes. # If the underlying CMM is from the MPL, load PEM bytes. + print(f"DEBUG: cmm is {self.config.materials_manager}") if (_HAS_MPL and isinstance(self.config.materials_manager, CryptoMaterialsManagerFromMPL)): self.signer = Signer.from_key_bytes( diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index e2fcc0f65..3fd40271a 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -173,6 +173,8 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, _p """ master_key_provider = generation_scenario.encryption_scenario.master_key_provider_fn() + print(f"DEBUG: mkp gen is {master_key_provider}") + # Use a caching CMM to avoid generating a new data key every time. if isinstance(master_key_provider, MasterKeyProvider): cache = LocalCryptoMaterialsCache(10) @@ -194,6 +196,8 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, _p else: raise TypeError(f"Unrecognized master_key_provider type: {master_key_provider}") + print(f"DEBUG: cmm gen is {cmm}") + return [ self.run_scenario_with_new_provider_info( ciphertext_writer, generation_scenario, cmm, new_provider_info @@ -204,6 +208,7 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, _p def run_scenario_with_new_provider_info( self, ciphertext_writer, generation_scenario, materials_manager, new_provider_info ): + print(f"DEBUG: materials_manager is {materials_manager}") """Run with tampering for a specific new provider info value""" if isinstance(materials_manager, CryptoMaterialsManager): tampering_materials_manager = ProviderInfoChangingCryptoMaterialsManager( From 19a9dad09b96ca773961dab3211861e09578bd27 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 12:49:01 -0700 Subject: [PATCH 305/376] debug --- codebuild/py37/generate_decrypt_vectors.yml | 1 - .../manifests/full_message/decrypt_generation.py | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml index 7aad81e13..7a1acca1b 100644 --- a/codebuild/py37/generate_decrypt_vectors.yml +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -31,4 +31,3 @@ artifacts: files: - test_vector_handlers/37_master.zip name: $CODEBUILD_BATCH_BUILD_IDENTIFIER/37_master.zip - discard-paths: yes diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 3fd40271a..a1fc8fa83 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -210,13 +210,13 @@ def run_scenario_with_new_provider_info( ): print(f"DEBUG: materials_manager is {materials_manager}") """Run with tampering for a specific new provider info value""" - if isinstance(materials_manager, CryptoMaterialsManager): - tampering_materials_manager = ProviderInfoChangingCryptoMaterialsManager( + if _HAS_MPL and isinstance(materials_manager, CryptoMaterialsManagerFromMPL): + tampering_materials_manager = ProviderInfoChangingCryptoMaterialsManagerFromMPL( materials_manager, new_provider_info ) - elif _HAS_MPL and isinstance(materials_manager, CryptoMaterialsManagerFromMPL): - tampering_materials_manager = ProviderInfoChangingCryptoMaterialsManagerFromMPL( + elif isinstance(materials_manager, CryptoMaterialsManager): + tampering_materials_manager = ProviderInfoChangingCryptoMaterialsManager( materials_manager, new_provider_info ) From a4aa0f9dd8e1b86de13d56cd9dda99b32a71bbf8 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 13:05:26 -0700 Subject: [PATCH 306/376] debug --- codebuild/py311/generate_decrypt_vectors_keyrings.yml | 2 +- codebuild/py311/generate_decrypt_vectors_masterkey.yml | 2 +- codebuild/py37/generate_decrypt_vectors.yml | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml index f83ac733e..e9c17c7b9 100644 --- a/codebuild/py311/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -31,5 +31,5 @@ phases: artifacts: files: - test_vector_handlers/311_keyring.zip - name: $CODEBUILD_BATCH_BUILD_IDENTIFIER/311_keyring.zip + name: $CODEBUILD_INITIATOR/311_keyring.zip discard-paths: yes \ No newline at end of file diff --git a/codebuild/py311/generate_decrypt_vectors_masterkey.yml b/codebuild/py311/generate_decrypt_vectors_masterkey.yml index 655fb1985..f4056832a 100644 --- a/codebuild/py311/generate_decrypt_vectors_masterkey.yml +++ b/codebuild/py311/generate_decrypt_vectors_masterkey.yml @@ -30,5 +30,5 @@ phases: artifacts: files: - test_vector_handlers/311_masterkey.zip - name: $CODEBUILD_BATCH_BUILD_IDENTIFIER/311_masterkey.zip + name: $CODEBUILD_INITIATOR/311_masterkey.zip discard-paths: yes \ No newline at end of file diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml index 7a1acca1b..784aaf44d 100644 --- a/codebuild/py37/generate_decrypt_vectors.yml +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -30,4 +30,5 @@ phases: artifacts: files: - test_vector_handlers/37_master.zip - name: $CODEBUILD_BATCH_BUILD_IDENTIFIER/37_master.zip + name: $CODEBUILD_INITIATOR/37_master.zip + discard-paths: yes From d53895de093854b94ff4bdf2c56152aab7b033c3 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 13:14:04 -0700 Subject: [PATCH 307/376] debug --- codebuild/py311/generate_decrypt_vectors_keyrings.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml index e9c17c7b9..1dbfc3b51 100644 --- a/codebuild/py311/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -31,5 +31,4 @@ phases: artifacts: files: - test_vector_handlers/311_keyring.zip - name: $CODEBUILD_INITIATOR/311_keyring.zip - discard-paths: yes \ No newline at end of file + name: $CODEBUILD_INITIATOR/311_keyring.zip \ No newline at end of file From 9c46200e47c1c4c3831ebce1d66868accd043190 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 13:14:30 -0700 Subject: [PATCH 308/376] debug --- codebuild/py311/generate_decrypt_vectors_keyrings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml index 1dbfc3b51..bbc09c89e 100644 --- a/codebuild/py311/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -31,4 +31,4 @@ phases: artifacts: files: - test_vector_handlers/311_keyring.zip - name: $CODEBUILD_INITIATOR/311_keyring.zip \ No newline at end of file + name: $CODEBUILD_INITIATOR \ No newline at end of file From 3af9f3266f5f3c9aa6304c8f92d79d64bfa99714 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 13:22:13 -0700 Subject: [PATCH 309/376] debug --- codebuild/py311/generate_decrypt_vectors_keyrings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml index bbc09c89e..4a178663a 100644 --- a/codebuild/py311/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -31,4 +31,4 @@ phases: artifacts: files: - test_vector_handlers/311_keyring.zip - name: $CODEBUILD_INITIATOR \ No newline at end of file + name: $CODEBUILD_RESOLVED_SOURCE_VERSION \ No newline at end of file From 9e255e487e6e86dbb67453293b8765f0b31ca9f3 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 13:28:54 -0700 Subject: [PATCH 310/376] debug --- codebuild/py311/generate_decrypt_vectors_keyrings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml index 4a178663a..c88001643 100644 --- a/codebuild/py311/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -31,4 +31,4 @@ phases: artifacts: files: - test_vector_handlers/311_keyring.zip - name: $CODEBUILD_RESOLVED_SOURCE_VERSION \ No newline at end of file + name: builds/$CODEBUILD_RESOLVED_SOURCE_VERSION/my-artifacts \ No newline at end of file From 262696b4373980cb727965f403c842431d11c9cf Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 13:36:10 -0700 Subject: [PATCH 311/376] debug --- codebuild/py311/generate_decrypt_vectors_keyrings.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml index c88001643..805179833 100644 --- a/codebuild/py311/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -22,13 +22,13 @@ phases: commands: - pip install "tox < 4.0" - cd test_vector_handlers + - mkdir $CODEBUILD_RESOLVED_SOURCE_VERSION - | tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ - --output 311_keyring \ + --output $CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring \ --keyrings - - zip -r 311_keyring.zip 311_keyring + - zip -r 311_keyring.zip $CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring artifacts: files: - - test_vector_handlers/311_keyring.zip - name: builds/$CODEBUILD_RESOLVED_SOURCE_VERSION/my-artifacts \ No newline at end of file + - test_vector_handlers/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip From fadea8c86ccf7851254b8112650c04c3f6d825df Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 13:42:40 -0700 Subject: [PATCH 312/376] debug --- codebuild/py311/generate_decrypt_vectors_keyrings.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml index 805179833..5f3c52240 100644 --- a/codebuild/py311/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -21,14 +21,14 @@ phases: build: commands: - pip install "tox < 4.0" - - cd test_vector_handlers - mkdir $CODEBUILD_RESOLVED_SOURCE_VERSION + - cd test_vector_handlers - | tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ - --output $CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring \ + --output ../$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring \ --keyrings - - zip -r 311_keyring.zip $CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring + - zip -r 311_keyring.zip ../$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring artifacts: files: - - test_vector_handlers/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip + - $CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip From b8dbb1f68ac1d57985be154543a7b30a8c6f4ae2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 13:45:54 -0700 Subject: [PATCH 313/376] debug --- codebuild/py311/generate_decrypt_vectors_keyrings.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml index 5f3c52240..db996a2fb 100644 --- a/codebuild/py311/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -21,14 +21,14 @@ phases: build: commands: - pip install "tox < 4.0" - - mkdir $CODEBUILD_RESOLVED_SOURCE_VERSION - cd test_vector_handlers + - mkdir $CODEBUILD_RESOLVED_SOURCE_VERSION - | tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ - --output ../$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring \ + --output 311_keyring \ --keyrings - - zip -r 311_keyring.zip ../$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring + - zip -r $CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip 311_keyring artifacts: files: - - $CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip + - test_vector_handlers/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip From f5d6cb7f4f3dcbd59e009f37f5d12030214ecd41 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 13:47:35 -0700 Subject: [PATCH 314/376] debug --- codebuild/py311/generate_decrypt_vectors_keyrings.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml index db996a2fb..9b4e71257 100644 --- a/codebuild/py311/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -22,13 +22,13 @@ phases: commands: - pip install "tox < 4.0" - cd test_vector_handlers - - mkdir $CODEBUILD_RESOLVED_SOURCE_VERSION + - mkdir $CODEBUILD_INITIATOR - | tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ --output 311_keyring \ --keyrings - - zip -r $CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip 311_keyring + - zip -r $CODEBUILD_INITIATOR/311_keyring.zip 311_keyring artifacts: files: - - test_vector_handlers/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip + - test_vector_handlers/$CODEBUILD_INITIATOR/311_keyring.zip From 774abf601694243eb1ede8c43408b946a06cc55f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 14:05:42 -0700 Subject: [PATCH 315/376] debug --- test_vector_handlers/test/mpl/__init__.py | 0 .../test/mpl/integration/__init__.py | 0 .../test/mpl/integration/commands/__init__.py | 0 .../commands/test_i_encrypt_keyrings.py | 64 +++++++++++++++++++ .../commands/test_i_esdk_dafny_keyrings.py | 0 .../commands/test_i_net_401_keyrings.py | 0 6 files changed, 64 insertions(+) create mode 100644 test_vector_handlers/test/mpl/__init__.py create mode 100644 test_vector_handlers/test/mpl/integration/__init__.py create mode 100644 test_vector_handlers/test/mpl/integration/commands/__init__.py create mode 100644 test_vector_handlers/test/mpl/integration/commands/test_i_encrypt_keyrings.py create mode 100644 test_vector_handlers/test/mpl/integration/commands/test_i_esdk_dafny_keyrings.py create mode 100644 test_vector_handlers/test/mpl/integration/commands/test_i_net_401_keyrings.py diff --git a/test_vector_handlers/test/mpl/__init__.py b/test_vector_handlers/test/mpl/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test_vector_handlers/test/mpl/integration/__init__.py b/test_vector_handlers/test/mpl/integration/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test_vector_handlers/test/mpl/integration/commands/__init__.py b/test_vector_handlers/test/mpl/integration/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test_vector_handlers/test/mpl/integration/commands/test_i_encrypt_keyrings.py b/test_vector_handlers/test/mpl/integration/commands/test_i_encrypt_keyrings.py new file mode 100644 index 000000000..077a36d63 --- /dev/null +++ b/test_vector_handlers/test/mpl/integration/commands/test_i_encrypt_keyrings.py @@ -0,0 +1,64 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +""" +Integration tests for `awses_test_vectors.commands` with keyrings. +""" +import pytest + +from awses_test_vectors.commands import full_message_decrypt, full_message_decrypt_generate, full_message_encrypt + +from ....integration.integration_test_utils import ( # noqa pylint: disable=unused-import + full_message_decrypt_generation_vectors, + full_message_encrypt_vectors, +) + + +pytestmark = [pytest.mark.integ] + + +def test_full_message_encrypt_canonical_full(full_message_encrypt_vectors): + full_message_encrypt.cli(["--input", full_message_encrypt_vectors, "--keyrings"]) + + +def test_full_message_cycle_canonical_full(tmpdir, full_message_decrypt_generation_vectors): + # Generate vectors using keyring interfaces + keyring_output_dir = tmpdir.join("output-keyrings") + full_message_decrypt_generate.cli([ + "--output", + str(keyring_output_dir), + "--input", + full_message_decrypt_generation_vectors, + "--keyrings" + ]) + + # Generate vectors using master key interfaces + master_key_output_dir = tmpdir.join("output-master-key") + full_message_decrypt_generate.cli([ + "--output", + str(master_key_output_dir), + "--input", + full_message_decrypt_generation_vectors + ]) + + # Validate that vectors generated using keyring interfaces + # can be decrypted by BOTH keyring and master key interfaces + keyring_decrypt_manifest_file = keyring_output_dir.join("manifest.json") + full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file), "--keyrings"]) + full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file)]) + + # Validate that vectors generated using master key interfaces + # can be decrypted by BOTH keyring and master key interfaces + master_key_decrypt_manifest_file = keyring_output_dir.join("manifest.json") + + full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file), "--keyrings"]) + full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file)]) diff --git a/test_vector_handlers/test/mpl/integration/commands/test_i_esdk_dafny_keyrings.py b/test_vector_handlers/test/mpl/integration/commands/test_i_esdk_dafny_keyrings.py new file mode 100644 index 000000000..e69de29bb diff --git a/test_vector_handlers/test/mpl/integration/commands/test_i_net_401_keyrings.py b/test_vector_handlers/test/mpl/integration/commands/test_i_net_401_keyrings.py new file mode 100644 index 000000000..e69de29bb From e16771ae0a211213d1db5ec3ef140dd16249500e Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 14:06:22 -0700 Subject: [PATCH 316/376] debug --- codebuild/py311/generate_decrypt_vectors_keyrings.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml index 9b4e71257..db996a2fb 100644 --- a/codebuild/py311/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -22,13 +22,13 @@ phases: commands: - pip install "tox < 4.0" - cd test_vector_handlers - - mkdir $CODEBUILD_INITIATOR + - mkdir $CODEBUILD_RESOLVED_SOURCE_VERSION - | tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ --output 311_keyring \ --keyrings - - zip -r $CODEBUILD_INITIATOR/311_keyring.zip 311_keyring + - zip -r $CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip 311_keyring artifacts: files: - - test_vector_handlers/$CODEBUILD_INITIATOR/311_keyring.zip + - test_vector_handlers/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip From 8752df72b0960952b10fffc7def6ab3445f8141d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 14:53:39 -0700 Subject: [PATCH 317/376] Debug --- buildspec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildspec.yml b/buildspec.yml index db168cc78..cc082e284 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -327,7 +327,7 @@ batch: # buildspec: codebuild/py312/decrypt_keyrings_with_js.yml # env: # image: aws/codebuild/standard:7.0 - + # # - identifier: code_coverage # buildspec: codebuild/coverage/coverage.yml # - identifier: code_coverage_mpl From 91d219c608ede8ee53c38609aa8d749f0a6ccb0e Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 15:02:40 -0700 Subject: [PATCH 318/376] Debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index dc5732552..a0e84a369 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -46,11 +46,9 @@ phases: # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct...) - | - MOST_RECENT_RUN_STUFF=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: token ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true") + MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: token ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') - | - echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_STUFF" - - | - MOST_RECENT_RUN_ID=$(echo $MOST_RECENT_RUN_STUFF | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') + echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" - | MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: token $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url') - | From 0178f7a9389292c16ef413fe9493d5dd246f7a77 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 15:12:45 -0700 Subject: [PATCH 319/376] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index a0e84a369..dce57d725 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -54,7 +54,7 @@ phases: - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" - | - curl -L -H "Accept: application/vnd.github+json" -H "Authorization: token $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip $MOST_RECENT_RUN_DOWNLOAD_URL + curl -L -H "Accept: application/vnd.github+json" -H "Authorization: token $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" $(echo MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip - unzip ubuntu-latest_test_vector_artifact build: commands: From 1060a3f42e01d14d3f818884c71be6123669835b Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 15:13:03 -0700 Subject: [PATCH 320/376] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index dce57d725..d3fb627af 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -54,7 +54,7 @@ phases: - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" - | - curl -L -H "Accept: application/vnd.github+json" -H "Authorization: token $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" $(echo MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip + curl -L -H "Accept: application/vnd.github+json" -H "Authorization: token $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip - unzip ubuntu-latest_test_vector_artifact build: commands: From 9806fba85cecba03789bda0e827893d1a1bd1ddd Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 15:14:41 -0700 Subject: [PATCH 321/376] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index d3fb627af..d3456636d 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -32,29 +32,29 @@ phases: - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') - aws sts get-caller-identity - # Authenticate into the CI bot to allow session to download ESDK Dafny GHA artifact. - - git config --global user.name "aws-crypto-tools-ci-bot" - - git config --global user.email "no-reply@noemail.local" - - # - | - # sudo mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ - # && sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ - # && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ - # && sudo apt update \ - # && sudo apt install gh -y - # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct...) - | - MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: token ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') + MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ + | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') - | echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" - | - MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: token $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url') + MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" \ + | jq '.artifacts[0].archive_download_url') - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" - | - curl -L -H "Accept: application/vnd.github+json" -H "Authorization: token $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip + curl -L -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip - unzip ubuntu-latest_test_vector_artifact build: commands: From 3625f98364949445d7f082f0dab7c21ca82a7996 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 15:20:18 -0700 Subject: [PATCH 322/376] debug --- .../py311/generate_decrypt_vectors_keyrings.yml | 12 +++++++----- codebuild/py37/generate_decrypt_vectors.yml | 13 +++++++------ 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml index db996a2fb..3e28dc113 100644 --- a/codebuild/py311/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -22,13 +22,15 @@ phases: commands: - pip install "tox < 4.0" - cd test_vector_handlers - - mkdir $CODEBUILD_RESOLVED_SOURCE_VERSION + # - mkdir $CODEBUILD_RESOLVED_SOURCE_VERSION - | tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ --output 311_keyring \ --keyrings - - zip -r $CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip 311_keyring -artifacts: - files: - - test_vector_handlers/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip + # - zip -r $CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip 311_keyring + - zip -r 311_keyring.zip 311_keyring + - aws s3 cp 311_keyring.zip s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip +# artifacts: +# files: +# - test_vector_handlers/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml index 784aaf44d..f5f12490e 100644 --- a/codebuild/py37/generate_decrypt_vectors.yml +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -26,9 +26,10 @@ phases: tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ --output 37_masterkey - - zip -r 37_master.zip 37_masterkey -artifacts: - files: - - test_vector_handlers/37_master.zip - name: $CODEBUILD_INITIATOR/37_master.zip - discard-paths: yes + - zip -r 37_masterkey.zip 37_masterkey + - aws s3 cp 37_masterkey.zip s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/37_masterkey.zip +# artifacts: +# files: +# - test_vector_handlers/37_master.zip +# name: $CODEBUILD_INITIATOR/37_master.zip +# discard-paths: yes From 77b9165380cfad5fe8b18809fe72b7ca7accfa1c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 15:22:22 -0700 Subject: [PATCH 323/376] Debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index d3456636d..aef28d67a 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -56,11 +56,12 @@ phases: -H "X-GitHub-Api-Version: 2022-11-28" \ $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip - unzip ubuntu-latest_test_vector_artifact + # This unzips to `net41/` build: commands: # NOTE: We need to pass the absolute path of the vectors - pip install "tox < 4.0" - - cd $CODEBUILD_SRC_DIR/test_vector_handlers + - cd /test_vector_handlers - | tox -- \ - --input $UNZIPPED_VECTORS_DIR/manifest.json + --input ../net41/manifest.json From cfc2681bef96d50a0928767d40b0185b6be9517c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 15:25:27 -0700 Subject: [PATCH 324/376] debug --- .../commands/test_i_encrypt_keyrings.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/test_vector_handlers/test/mpl/integration/commands/test_i_encrypt_keyrings.py b/test_vector_handlers/test/mpl/integration/commands/test_i_encrypt_keyrings.py index 077a36d63..85c94dd22 100644 --- a/test_vector_handlers/test/mpl/integration/commands/test_i_encrypt_keyrings.py +++ b/test_vector_handlers/test/mpl/integration/commands/test_i_encrypt_keyrings.py @@ -41,24 +41,5 @@ def test_full_message_cycle_canonical_full(tmpdir, full_message_decrypt_generati "--keyrings" ]) - # Generate vectors using master key interfaces - master_key_output_dir = tmpdir.join("output-master-key") - full_message_decrypt_generate.cli([ - "--output", - str(master_key_output_dir), - "--input", - full_message_decrypt_generation_vectors - ]) - - # Validate that vectors generated using keyring interfaces - # can be decrypted by BOTH keyring and master key interfaces keyring_decrypt_manifest_file = keyring_output_dir.join("manifest.json") full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file), "--keyrings"]) - full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file)]) - - # Validate that vectors generated using master key interfaces - # can be decrypted by BOTH keyring and master key interfaces - master_key_decrypt_manifest_file = keyring_output_dir.join("manifest.json") - - full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file), "--keyrings"]) - full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file)]) From 87bc057d17a6ea9b4e71276d6822bb9f44368c92 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 15:26:09 -0700 Subject: [PATCH 325/376] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index aef28d67a..f7213ac56 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -61,7 +61,7 @@ phases: commands: # NOTE: We need to pass the absolute path of the vectors - pip install "tox < 4.0" - - cd /test_vector_handlers + - cd test_vector_handlers - | tox -- \ --input ../net41/manifest.json From ca6f3a1b61d39af1f1d72f3983af6087680a712d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 15:29:52 -0700 Subject: [PATCH 326/376] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index f7213ac56..7a467a067 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -46,7 +46,7 @@ phases: MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" \ -H "Authorization: token $GITHUB_TOKEN" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/$MOST_RECENT_RUN_ID/artifacts?name=ubuntu-latest_vector_artifact" \ | jq '.artifacts[0].archive_download_url') - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" @@ -55,13 +55,16 @@ phases: -H "Authorization: token $GITHUB_TOKEN" \ -H "X-GitHub-Api-Version: 2022-11-28" \ $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip - - unzip ubuntu-latest_test_vector_artifact # This unzips to `net41/` + - unzip ubuntu-latest_test_vector_artifact + - ls build: commands: # NOTE: We need to pass the absolute path of the vectors - pip install "tox < 4.0" + - ls - cd test_vector_handlers + - ls - | tox -- \ --input ../net41/manifest.json From 3b01d387563790ad6e6b3fea627c7697814376e6 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 15:33:11 -0700 Subject: [PATCH 327/376] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 7a467a067..6c3b532ec 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -55,8 +55,10 @@ phases: -H "Authorization: token $GITHUB_TOKEN" \ -H "X-GitHub-Api-Version: 2022-11-28" \ $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip - # This unzips to `net41/` + # This unzips to `net41.zip` - unzip ubuntu-latest_test_vector_artifact + # This unzips to `net41/` + - unzip net41.zip - ls build: commands: From 68495e83e48192b9ef22b1ffe3b160ba350a2225 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 15:37:26 -0700 Subject: [PATCH 328/376] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 6c3b532ec..5e8fbcb1e 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -56,9 +56,9 @@ phases: -H "X-GitHub-Api-Version: 2022-11-28" \ $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip # This unzips to `net41.zip` - - unzip ubuntu-latest_test_vector_artifact + - unzip ubuntu-latest_test_vector_artifact.zip # This unzips to `net41/` - - unzip net41.zip + # - unzip net41.zip - ls build: commands: From 8a8103a94bd00758c860efd2eadc454dde7d19aa Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 15:41:25 -0700 Subject: [PATCH 329/376] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 5e8fbcb1e..3725ed526 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -56,9 +56,9 @@ phases: -H "X-GitHub-Api-Version: 2022-11-28" \ $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip # This unzips to `net41.zip` - - unzip ubuntu-latest_test_vector_artifact.zip + - unzip ubuntu-latest_test_vector_artifact # This unzips to `net41/` - # - unzip net41.zip + - unzip net41.zip -d net41 - ls build: commands: From 89c39ffea3758797d5fca8bcf36fd78bbfbceb79 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 15:50:45 -0700 Subject: [PATCH 330/376] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 3725ed526..e60084f94 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -59,14 +59,11 @@ phases: - unzip ubuntu-latest_test_vector_artifact # This unzips to `net41/` - unzip net41.zip -d net41 - - ls build: commands: # NOTE: We need to pass the absolute path of the vectors - pip install "tox < 4.0" - - ls - cd test_vector_handlers - - ls - | tox -- \ --input ../net41/manifest.json From fee0ccece2150a87e259d4a69cd05feaee8b68df Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 15:57:54 -0700 Subject: [PATCH 331/376] debug --- codebuild/py37/decrypt_masterkey_with_js.yml | 4 ++-- codebuild/py37/decrypt_masterkey_with_masterkey.yml | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/codebuild/py37/decrypt_masterkey_with_js.yml b/codebuild/py37/decrypt_masterkey_with_js.yml index 22ca730b7..6fefb12fd 100644 --- a/codebuild/py37/decrypt_masterkey_with_js.yml +++ b/codebuild/py37/decrypt_masterkey_with_js.yml @@ -36,8 +36,8 @@ phases: # Download generated vectors # TODO rewrite URL - aws s3 cp s3://generated-vectors-artifacts-bucket/GeneratedVectors/py37_generate_decrypt_vectors/test_vector_handlers/37_master.zip 37_master.zip + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/37_masterkey.zip 37_masterkey.zip build: commands: # Decrypt generated vectors with Javascript ESDK - - integration-node decrypt -v 37_master.zip \ No newline at end of file + - integration-node decrypt -v 37_masterkey.zip \ No newline at end of file diff --git a/codebuild/py37/decrypt_masterkey_with_masterkey.yml b/codebuild/py37/decrypt_masterkey_with_masterkey.yml index 5a2347e8f..d75c26a27 100644 --- a/codebuild/py37/decrypt_masterkey_with_masterkey.yml +++ b/codebuild/py37/decrypt_masterkey_with_masterkey.yml @@ -18,14 +18,12 @@ phases: install: runtime-versions: python: 3.7 - - pre-build: commands: # Download generated vectors # TODO rewrite URL - aws s3 cp s3://generated-vectors-artifacts-bucket/py37_generate_decrypt_vectors/test_vector_handlers/37_master.zip 37_masterkey.zip - unzip 37_master.zip + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/37_masterkey.zip 37_masterkey.zip + - unzip 37_masterkey.zip build: commands: - pip install "tox < 4.0" From c5ba2fd2b15f959a9e61ec1027a98084ed323594 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 16:00:40 -0700 Subject: [PATCH 332/376] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index e60084f94..bca8a3147 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -12,8 +12,6 @@ env: arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" git-credential-helper: yes secrets-manager: GITHUB_TOKEN: Github/lucasmcdonald3-fgpat:actions read @@ -24,14 +22,6 @@ phases: python: 3.7 pre_build: commands: - # Assume Role to access non-prod resources - - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") - - export TMP_ROLE - - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') - - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') - - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') - - aws sts get-caller-identity - # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct...) - | From 4875dbcbdc81cab69a0be83fb6b86c8ff3f8e380 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 16:09:53 -0700 Subject: [PATCH 333/376] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index bca8a3147..ee2d428c7 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -45,13 +45,12 @@ phases: -H "Authorization: token $GITHUB_TOKEN" \ -H "X-GitHub-Api-Version: 2022-11-28" \ $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip - # This unzips to `net41.zip` + # This unzips to `net41.zip`. - unzip ubuntu-latest_test_vector_artifact - # This unzips to `net41/` + # This unzips to `net41/`. - unzip net41.zip -d net41 build: commands: - # NOTE: We need to pass the absolute path of the vectors - pip install "tox < 4.0" - cd test_vector_handlers - | From 4ed99d371e449f97d167d1cdffe8a3ba72f2ba98 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 17:26:29 -0700 Subject: [PATCH 334/376] debug --- codebuild/py37/decrypt_masterkey_with_js.yml | 2 +- codebuild/py37/decrypt_masterkey_with_masterkey.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/codebuild/py37/decrypt_masterkey_with_js.yml b/codebuild/py37/decrypt_masterkey_with_js.yml index 6fefb12fd..3e5efe8b1 100644 --- a/codebuild/py37/decrypt_masterkey_with_js.yml +++ b/codebuild/py37/decrypt_masterkey_with_js.yml @@ -26,7 +26,7 @@ phases: pre_build: commands: # Assume Role to access non-prod resources - - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Java-Role-us-west-2" --role-session-name "CB-TestVectorResources") + - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") - export TMP_ROLE - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') diff --git a/codebuild/py37/decrypt_masterkey_with_masterkey.yml b/codebuild/py37/decrypt_masterkey_with_masterkey.yml index d75c26a27..c6117b9ba 100644 --- a/codebuild/py37/decrypt_masterkey_with_masterkey.yml +++ b/codebuild/py37/decrypt_masterkey_with_masterkey.yml @@ -18,7 +18,7 @@ phases: install: runtime-versions: python: 3.7 - pre-build: + pre_build: commands: # Download generated vectors # TODO rewrite URL From 9f76cbf0f12d66178bcd64eb0659bd5651fc89f3 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 17:42:25 -0700 Subject: [PATCH 335/376] debug --- buildspec.yml | 176 +++++++++++++++++++++++++------------------------- 1 file changed, 88 insertions(+), 88 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index cc082e284..1ff23ab8b 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -150,94 +150,94 @@ batch: # env: # image: aws/codebuild/standard:5.0 - - identifier: py311_integ - buildspec: codebuild/py311/integ.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_integ_mpl - buildspec: codebuild/py311/integ_mpl.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_examples - buildspec: codebuild/py311/examples.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_examples_mpl - buildspec: codebuild/py311/examples_mpl.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_awses_latest - buildspec: codebuild/py311/awses_local.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_awses_latest_mpl - buildspec: codebuild/py311/awses_local_mpl.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_mplawses_latest_mpl - buildspec: codebuild/py311/mplawses_local_mpl.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_dafny_esdk_vectors_masterkey - buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_dafny_esdk_vectors_keyrings - buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_net_401_vectors_masterkey - buildspec: codebuild/py311/decrypt_net_401_vectors_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_net_401_vectors_keyrings - buildspec: codebuild/py311/decrypt_net_401_vectors_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_generate_decrypt_vectors_masterkey - buildspec: codebuild/py311/generate_decrypt_vectors_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_masterkey_with_masterkey - depend-on: - - py311_generate_decrypt_vectors_masterkey - buildspec: codebuild/py311/decrypt_masterkey_with_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_masterkey_with_keyrings - depend-on: - - py311_generate_decrypt_vectors_masterkey - buildspec: codebuild/py311/decrypt_masterkey_with_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_masterkey_with_js - depend-on: - - py311_generate_decrypt_vectors_masterkey - buildspec: codebuild/py311/decrypt_masterkey_with_js.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_generate_decrypt_vectors_keyrings - buildspec: codebuild/py311/generate_decrypt_vectors_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_keyrings_with_masterkey - depend-on: - - py311_generate_decrypt_vectors_keyrings - buildspec: codebuild/py311/decrypt_keyrings_with_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_keyrings_with_keyrings - depend-on: - - py311_generate_decrypt_vectors_keyrings - buildspec: codebuild/py311/decrypt_keyrings_with_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_keyrings_with_js - depend-on: - - py311_generate_decrypt_vectors_keyrings - buildspec: codebuild/py311/decrypt_keyrings_with_js.yml - env: - image: aws/codebuild/standard:7.0 + # - identifier: py311_integ + # buildspec: codebuild/py311/integ.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_integ_mpl + # buildspec: codebuild/py311/integ_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_examples + # buildspec: codebuild/py311/examples.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_examples_mpl + # buildspec: codebuild/py311/examples_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_awses_latest + # buildspec: codebuild/py311/awses_local.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_awses_latest_mpl + # buildspec: codebuild/py311/awses_local_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_mplawses_latest_mpl + # buildspec: codebuild/py311/mplawses_local_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_dafny_esdk_vectors_masterkey + # buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_dafny_esdk_vectors_keyrings + # buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_net_401_vectors_masterkey + # buildspec: codebuild/py311/decrypt_net_401_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_net_401_vectors_keyrings + # buildspec: codebuild/py311/decrypt_net_401_vectors_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py311/generate_decrypt_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_masterkey_with_masterkey + # depend-on: + # - py311_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py311/decrypt_masterkey_with_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_masterkey_with_keyrings + # depend-on: + # - py311_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py311/decrypt_masterkey_with_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_masterkey_with_js + # depend-on: + # - py311_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py311/decrypt_masterkey_with_js.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py311/generate_decrypt_vectors_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_keyrings_with_masterkey + # depend-on: + # - py311_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py311/decrypt_keyrings_with_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_keyrings_with_keyrings + # depend-on: + # - py311_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py311/decrypt_keyrings_with_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_keyrings_with_js + # depend-on: + # - py311_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py311/decrypt_keyrings_with_js.yml + # env: + # image: aws/codebuild/standard:7.0 # - identifier: py312_integ # buildspec: codebuild/py312/integ.yml From 2f08f0d3a67d1ccb49691cd497cebfc04afadcf9 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 17:46:30 -0700 Subject: [PATCH 336/376] debug --- buildspec.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index 1ff23ab8b..6efc031c6 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -11,14 +11,14 @@ batch: buildspec: codebuild/py37/examples.yml env: image: aws/codebuild/standard:5.0 - - identifier: py37_awses_local - buildspec: codebuild/py37/awses_local.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py37_decrypt_dafny_esdk_vectors - buildspec: codebuild/py37/decrypt_dafny_esdk_vectors.yml - env: - image: aws/codebuild/standard:5.0 + # - identifier: py37_awses_local + # buildspec: codebuild/py37/awses_local.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py37_decrypt_dafny_esdk_vectors + # buildspec: codebuild/py37/decrypt_dafny_esdk_vectors.yml + # env: + # image: aws/codebuild/standard:5.0 - identifier: py37_decrypt_net_401_vectors buildspec: codebuild/py37/decrypt_net_401_vectors.yml env: From 594f2732b947bf79d111276dc19c34407d0f379b Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 09:11:11 -0700 Subject: [PATCH 337/376] debug --- codebuild/py37/decrypt_masterkey_with_js.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/codebuild/py37/decrypt_masterkey_with_js.yml b/codebuild/py37/decrypt_masterkey_with_js.yml index 3e5efe8b1..15dcb35ac 100644 --- a/codebuild/py37/decrypt_masterkey_with_js.yml +++ b/codebuild/py37/decrypt_masterkey_with_js.yml @@ -26,13 +26,13 @@ phases: pre_build: commands: # Assume Role to access non-prod resources - - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") - - export TMP_ROLE - - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') - - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') - - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') - - aws sts get-caller-identity - - cd $CODEBUILD_SRC_DIR + # - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") + # - export TMP_ROLE + # - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') + # - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') + # - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') + # - aws sts get-caller-identity + # - cd $CODEBUILD_SRC_DIR # Download generated vectors # TODO rewrite URL From 108cd03462be6d99b49502b50da33bee59c78b8a Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 09:26:11 -0700 Subject: [PATCH 338/376] refactor --- buildspec.yml | 20 +++++----- codebuild/py37/encrypt_masterkey.yml | 33 +++++++++++++++++ ...=> generate_decrypt_vectors_masterkey.yml} | 0 .../commands/test_i_full_message_encrypt.py | 37 ------------------- .../commands/test_i_esdk_dafny_keyrings.py | 0 .../commands/test_i_net_401_keyrings.py | 0 test_vector_handlers/tox.ini | 1 + 7 files changed, 44 insertions(+), 47 deletions(-) create mode 100644 codebuild/py37/encrypt_masterkey.yml rename codebuild/py37/{generate_decrypt_vectors.yml => generate_decrypt_vectors_masterkey.yml} (100%) delete mode 100644 test_vector_handlers/test/integration/commands/test_i_full_message_encrypt.py delete mode 100644 test_vector_handlers/test/mpl/integration/commands/test_i_esdk_dafny_keyrings.py delete mode 100644 test_vector_handlers/test/mpl/integration/commands/test_i_net_401_keyrings.py diff --git a/buildspec.yml b/buildspec.yml index 6efc031c6..2eb44ccff 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -11,20 +11,20 @@ batch: buildspec: codebuild/py37/examples.yml env: image: aws/codebuild/standard:5.0 - # - identifier: py37_awses_local - # buildspec: codebuild/py37/awses_local.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py37_decrypt_dafny_esdk_vectors - # buildspec: codebuild/py37/decrypt_dafny_esdk_vectors.yml - # env: - # image: aws/codebuild/standard:5.0 + - identifier: py37_decrypt_dafny_esdk_vectors + buildspec: codebuild/py37/decrypt_dafny_esdk_vectors.yml + env: + image: aws/codebuild/standard:5.0 - identifier: py37_decrypt_net_401_vectors buildspec: codebuild/py37/decrypt_net_401_vectors.yml env: image: aws/codebuild/standard:5.0 - - identifier: py37_generate_decrypt_vectors - buildspec: codebuild/py37/generate_decrypt_vectors.yml + - identifier: py37_encrypt_masterkey + buildspec: codebuild/py37/encrypt_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py37_generate_decrypt_vectors_masterkey + buildspec: codebuild/py37/generate_decrypt_vectors_masterkey.yml env: image: aws/codebuild/standard:5.0 - identifier: py37_decrypt_masterkey_with_masterkey diff --git a/codebuild/py37/encrypt_masterkey.yml b/codebuild/py37/encrypt_masterkey.yml new file mode 100644 index 000000000..c6117b9ba --- /dev/null +++ b/codebuild/py37/encrypt_masterkey.yml @@ -0,0 +1,33 @@ +version: 0.2 + +env: + variables: + TOXENV: "py37-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + python: 3.7 + pre_build: + commands: + # Download generated vectors + # TODO rewrite URL + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/37_masterkey.zip 37_masterkey.zip + - unzip 37_masterkey.zip + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../37_masterkey/manifest.json \ No newline at end of file diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors_masterkey.yml similarity index 100% rename from codebuild/py37/generate_decrypt_vectors.yml rename to codebuild/py37/generate_decrypt_vectors_masterkey.yml diff --git a/test_vector_handlers/test/integration/commands/test_i_full_message_encrypt.py b/test_vector_handlers/test/integration/commands/test_i_full_message_encrypt.py deleted file mode 100644 index 6305a15da..000000000 --- a/test_vector_handlers/test/integration/commands/test_i_full_message_encrypt.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. -""" -Integration tests for ``awses_test_vectors.commands``. -""" -import pytest - -from awses_test_vectors.commands import full_message_decrypt, full_message_decrypt_generate, full_message_encrypt - -from ..integration_test_utils import ( # noqa pylint: disable=unused-import - full_message_decrypt_generation_vectors, - full_message_encrypt_vectors, -) - -pytestmark = [pytest.mark.integ] - - -def test_full_message_encrypt_canonical_full(full_message_encrypt_vectors): - full_message_encrypt.cli(["--input", full_message_encrypt_vectors]) - - -def test_full_message_cycle_canonical_full(tmpdir, full_message_decrypt_generation_vectors): - output_dir = tmpdir.join("output") - full_message_decrypt_generate.cli(["--output", str(output_dir), "--input", full_message_decrypt_generation_vectors]) - - decrypt_manifest_file = output_dir.join("manifest.json") - full_message_decrypt.cli(["--input", str(decrypt_manifest_file)]) diff --git a/test_vector_handlers/test/mpl/integration/commands/test_i_esdk_dafny_keyrings.py b/test_vector_handlers/test/mpl/integration/commands/test_i_esdk_dafny_keyrings.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/test_vector_handlers/test/mpl/integration/commands/test_i_net_401_keyrings.py b/test_vector_handlers/test/mpl/integration/commands/test_i_net_401_keyrings.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index bf4b86724..c002323d3 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -58,6 +58,7 @@ commands = mplvectors: {[testenv:base-command]commands} test/mpl full_decrypt_generate: awses-full-message-decrypt-generate {posargs} full_decrypt: awses-full-message-decrypt {posargs} + full_encrypt: awses-full-message-encrypt {posargs} [testenv:full-encrypt] basepython = python3 From b116b0df644b78f7da0ba2eceea6b153e2d847eb Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 09:27:18 -0700 Subject: [PATCH 339/376] debug --- codebuild/py37/decrypt_masterkey_with_js.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_masterkey_with_js.yml b/codebuild/py37/decrypt_masterkey_with_js.yml index 15dcb35ac..0652b03bf 100644 --- a/codebuild/py37/decrypt_masterkey_with_js.yml +++ b/codebuild/py37/decrypt_masterkey_with_js.yml @@ -37,7 +37,8 @@ phases: # Download generated vectors # TODO rewrite URL - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/37_masterkey.zip 37_masterkey.zip + - unzip 37_masterkey.zip build: commands: # Decrypt generated vectors with Javascript ESDK - - integration-node decrypt -v 37_masterkey.zip \ No newline at end of file + - integration-node decrypt -v 37_masterkey \ No newline at end of file From 063989feea5db9c5029934a3966435f68a81ac97 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 09:31:02 -0700 Subject: [PATCH 340/376] debug --- buildspec.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index 2eb44ccff..9375821b0 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -29,13 +29,13 @@ batch: image: aws/codebuild/standard:5.0 - identifier: py37_decrypt_masterkey_with_masterkey depend-on: - - py37_generate_decrypt_vectors + - py37_generate_decrypt_vectors_masterkey buildspec: codebuild/py37/decrypt_masterkey_with_masterkey.yml env: image: aws/codebuild/standard:5.0 - identifier: py37_decrypt_masterkey_with_js depend-on: - - py37_generate_decrypt_vectors + - py37_generate_decrypt_vectors_masterkey buildspec: codebuild/py37/decrypt_masterkey_with_js.yml env: image: aws/codebuild/standard:5.0 From 101af711436ec6c1d73b16284597ddd5cbde4521 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 09:32:40 -0700 Subject: [PATCH 341/376] debug --- codebuild/py37/encrypt_masterkey.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/codebuild/py37/encrypt_masterkey.yml b/codebuild/py37/encrypt_masterkey.yml index c6117b9ba..384efc0bb 100644 --- a/codebuild/py37/encrypt_masterkey.yml +++ b/codebuild/py37/encrypt_masterkey.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - TOXENV: "py37-full_decrypt" + TOXENV: "py37-full_encrypt" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- @@ -18,16 +18,15 @@ phases: install: runtime-versions: python: 3.7 - pre_build: - commands: - # Download generated vectors - # TODO rewrite URL - - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/37_masterkey.zip 37_masterkey.zip - - unzip 37_masterkey.zip build: commands: - pip install "tox < 4.0" - cd test_vector_handlers - | tox -- \ - --input ../37_masterkey/manifest.json \ No newline at end of file + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0003-awses-message-encryption.v2.json \ +# artifacts: +# files: +# - test_vector_handlers/37_master.zip +# name: $CODEBUILD_INITIATOR/37_master.zip +# discard-paths: yes From 803ae4d7d3c42357c35041c8d819de46bc947b4e Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 09:49:26 -0700 Subject: [PATCH 342/376] debug --- codebuild/py37/decrypt_masterkey_with_js.yml | 3 +-- .../py37/decrypt_masterkey_with_masterkey.yml | 5 +---- codebuild/py37/decrypt_net_401_vectors.yml | 18 +++++++++--------- codebuild/py37/encrypt_masterkey.yml | 4 ++-- .../generate_decrypt_vectors_masterkey.yml | 4 ++-- 5 files changed, 15 insertions(+), 19 deletions(-) diff --git a/codebuild/py37/decrypt_masterkey_with_js.yml b/codebuild/py37/decrypt_masterkey_with_js.yml index 0652b03bf..e13f3d64a 100644 --- a/codebuild/py37/decrypt_masterkey_with_js.yml +++ b/codebuild/py37/decrypt_masterkey_with_js.yml @@ -34,8 +34,7 @@ phases: # - aws sts get-caller-identity # - cd $CODEBUILD_SRC_DIR - # Download generated vectors - # TODO rewrite URL + # Download previously generated vectors - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/37_masterkey.zip 37_masterkey.zip - unzip 37_masterkey.zip build: diff --git a/codebuild/py37/decrypt_masterkey_with_masterkey.yml b/codebuild/py37/decrypt_masterkey_with_masterkey.yml index c6117b9ba..1774b05f4 100644 --- a/codebuild/py37/decrypt_masterkey_with_masterkey.yml +++ b/codebuild/py37/decrypt_masterkey_with_masterkey.yml @@ -11,8 +11,6 @@ env: arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" phases: install: @@ -20,8 +18,7 @@ phases: python: 3.7 pre_build: commands: - # Download generated vectors - # TODO rewrite URL + # Download previously generated vectors - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/37_masterkey.zip 37_masterkey.zip - unzip 37_masterkey.zip build: diff --git a/codebuild/py37/decrypt_net_401_vectors.yml b/codebuild/py37/decrypt_net_401_vectors.yml index 5b3925890..943f0722a 100644 --- a/codebuild/py37/decrypt_net_401_vectors.yml +++ b/codebuild/py37/decrypt_net_401_vectors.yml @@ -12,8 +12,8 @@ env: arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + # AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + # AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" phases: install: @@ -21,13 +21,13 @@ phases: python: 3.7 pre_build: commands: - # Assume Role to access non-prod resources - - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") - - export TMP_ROLE - - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') - - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') - - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') - - aws sts get-caller-identity + # # Assume Role to access non-prod resources + # - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") + # - export TMP_ROLE + # - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') + # - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') + # - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') + # - aws sts get-caller-identity # Fetch ESDK .NET v4.0.1 Test Vectors - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip diff --git a/codebuild/py37/encrypt_masterkey.yml b/codebuild/py37/encrypt_masterkey.yml index 384efc0bb..70a36c4d0 100644 --- a/codebuild/py37/encrypt_masterkey.yml +++ b/codebuild/py37/encrypt_masterkey.yml @@ -11,8 +11,8 @@ env: arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + # AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + # AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" phases: install: diff --git a/codebuild/py37/generate_decrypt_vectors_masterkey.yml b/codebuild/py37/generate_decrypt_vectors_masterkey.yml index f5f12490e..8c5c9712f 100644 --- a/codebuild/py37/generate_decrypt_vectors_masterkey.yml +++ b/codebuild/py37/generate_decrypt_vectors_masterkey.yml @@ -11,8 +11,8 @@ env: arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + # AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + # AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" phases: install: From 58bd2714e041cca5d1779d2152a1d0b6cda5a072 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 10:24:55 -0700 Subject: [PATCH 343/376] debug --- buildspec.yml | 2 +- codebuild/py37/decrypt_masterkey_with_js.yml | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index 9375821b0..9d588c67b 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -35,7 +35,7 @@ batch: image: aws/codebuild/standard:5.0 - identifier: py37_decrypt_masterkey_with_js depend-on: - - py37_generate_decrypt_vectors_masterkey + # - py37_generate_decrypt_vectors_masterkey buildspec: codebuild/py37/decrypt_masterkey_with_js.yml env: image: aws/codebuild/standard:5.0 diff --git a/codebuild/py37/decrypt_masterkey_with_js.yml b/codebuild/py37/decrypt_masterkey_with_js.yml index e13f3d64a..c1a4196e6 100644 --- a/codebuild/py37/decrypt_masterkey_with_js.yml +++ b/codebuild/py37/decrypt_masterkey_with_js.yml @@ -35,9 +35,12 @@ phases: # - cd $CODEBUILD_SRC_DIR # Download previously generated vectors - - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/37_masterkey.zip 37_masterkey.zip + - aws s3 cp s3://generated-vectors-artifacts-bucket/77b9165380cfad5fe8b18809fe72b7ca7accfa1c/37_masterkey.zip 37_masterkey.zip + # Repackage zip in expected format - unzip 37_masterkey.zip + - cd 37_masterkey + - zip -r vectors.zip . build: commands: # Decrypt generated vectors with Javascript ESDK - - integration-node decrypt -v 37_masterkey \ No newline at end of file + - integration-node decrypt -v vectors.zip \ No newline at end of file From bccd1beb45d332940b9dfe8a7ce8a03f5159c792 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 10:29:53 -0700 Subject: [PATCH 344/376] clean37 --- codebuild/py37/decrypt_masterkey_with_js.yml | 14 +------------- codebuild/py37/decrypt_net_401_vectors.yml | 10 ---------- codebuild/py37/encrypt_masterkey.yml | 7 ------- .../py37/generate_decrypt_vectors_masterkey.yml | 7 ------- 4 files changed, 1 insertion(+), 37 deletions(-) diff --git a/codebuild/py37/decrypt_masterkey_with_js.yml b/codebuild/py37/decrypt_masterkey_with_js.yml index c1a4196e6..8a44e11e7 100644 --- a/codebuild/py37/decrypt_masterkey_with_js.yml +++ b/codebuild/py37/decrypt_masterkey_with_js.yml @@ -10,9 +10,6 @@ env: arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - GENERATE_OUTPUT_DIR: >- - $CODEBUILD_SRC_DIR/generated_vectors/ - phases: install: @@ -25,17 +22,8 @@ phases: pre_build: commands: - # Assume Role to access non-prod resources - # - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") - # - export TMP_ROLE - # - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') - # - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') - # - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') - # - aws sts get-caller-identity - # - cd $CODEBUILD_SRC_DIR - # Download previously generated vectors - - aws s3 cp s3://generated-vectors-artifacts-bucket/77b9165380cfad5fe8b18809fe72b7ca7accfa1c/37_masterkey.zip 37_masterkey.zip + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/37_masterkey.zip 37_masterkey.zip # Repackage zip in expected format - unzip 37_masterkey.zip - cd 37_masterkey diff --git a/codebuild/py37/decrypt_net_401_vectors.yml b/codebuild/py37/decrypt_net_401_vectors.yml index 943f0722a..1dfb48ce8 100644 --- a/codebuild/py37/decrypt_net_401_vectors.yml +++ b/codebuild/py37/decrypt_net_401_vectors.yml @@ -12,8 +12,6 @@ env: arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - # AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" - # AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" phases: install: @@ -21,14 +19,6 @@ phases: python: 3.7 pre_build: commands: - # # Assume Role to access non-prod resources - # - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") - # - export TMP_ROLE - # - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') - # - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') - # - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') - # - aws sts get-caller-identity - # Fetch ESDK .NET v4.0.1 Test Vectors - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip diff --git a/codebuild/py37/encrypt_masterkey.yml b/codebuild/py37/encrypt_masterkey.yml index 70a36c4d0..b5cb57dae 100644 --- a/codebuild/py37/encrypt_masterkey.yml +++ b/codebuild/py37/encrypt_masterkey.yml @@ -11,8 +11,6 @@ env: arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - # AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" - # AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" phases: install: @@ -25,8 +23,3 @@ phases: - | tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0003-awses-message-encryption.v2.json \ -# artifacts: -# files: -# - test_vector_handlers/37_master.zip -# name: $CODEBUILD_INITIATOR/37_master.zip -# discard-paths: yes diff --git a/codebuild/py37/generate_decrypt_vectors_masterkey.yml b/codebuild/py37/generate_decrypt_vectors_masterkey.yml index 8c5c9712f..55ec3e9e4 100644 --- a/codebuild/py37/generate_decrypt_vectors_masterkey.yml +++ b/codebuild/py37/generate_decrypt_vectors_masterkey.yml @@ -11,8 +11,6 @@ env: arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - # AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" - # AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" phases: install: @@ -28,8 +26,3 @@ phases: --output 37_masterkey - zip -r 37_masterkey.zip 37_masterkey - aws s3 cp 37_masterkey.zip s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/37_masterkey.zip -# artifacts: -# files: -# - test_vector_handlers/37_master.zip -# name: $CODEBUILD_INITIATOR/37_master.zip -# discard-paths: yes From 970ca3cc7bf5e86a50095f611370fd0014a733eb Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 10:30:23 -0700 Subject: [PATCH 345/376] clean37 --- buildspec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildspec.yml b/buildspec.yml index 9d588c67b..9375821b0 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -35,7 +35,7 @@ batch: image: aws/codebuild/standard:5.0 - identifier: py37_decrypt_masterkey_with_js depend-on: - # - py37_generate_decrypt_vectors_masterkey + - py37_generate_decrypt_vectors_masterkey buildspec: codebuild/py37/decrypt_masterkey_with_js.yml env: image: aws/codebuild/standard:5.0 From 9ccd01419b944f656b23623b00d84fa5cc2ae268 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 11:04:52 -0700 Subject: [PATCH 346/376] 311 --- buildspec.yml | 158 +++++++++--------- codebuild/py311/decrypt_keyrings_with_js.yml | 34 ++++ .../py311/decrypt_keyrings_with_keyrings.yml | 31 ++++ .../py311/decrypt_keyrings_with_masterkey.yml | 30 ++++ codebuild/py311/decrypt_masterkey_with_js.yml | 34 ++++ .../py311/decrypt_masterkey_with_keyrings.yml | 31 ++++ .../decrypt_masterkey_with_masterkey.yml | 30 ++++ .../decrypt_net_401_vectors_keyrings.yml | 10 -- .../generate_decrypt_vectors_keyrings.yml | 14 +- .../generate_decrypt_vectors_masterkey.yml | 8 +- 10 files changed, 273 insertions(+), 107 deletions(-) create mode 100644 codebuild/py311/decrypt_keyrings_with_js.yml create mode 100644 codebuild/py311/decrypt_keyrings_with_keyrings.yml create mode 100644 codebuild/py311/decrypt_keyrings_with_masterkey.yml create mode 100644 codebuild/py311/decrypt_masterkey_with_js.yml create mode 100644 codebuild/py311/decrypt_masterkey_with_keyrings.yml create mode 100644 codebuild/py311/decrypt_masterkey_with_masterkey.yml diff --git a/buildspec.yml b/buildspec.yml index 9375821b0..3c4b965de 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -150,22 +150,22 @@ batch: # env: # image: aws/codebuild/standard:5.0 - # - identifier: py311_integ - # buildspec: codebuild/py311/integ.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_integ_mpl - # buildspec: codebuild/py311/integ_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_examples - # buildspec: codebuild/py311/examples.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_examples_mpl - # buildspec: codebuild/py311/examples_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 + - identifier: py311_integ + buildspec: codebuild/py311/integ.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_integ_mpl + buildspec: codebuild/py311/integ_mpl.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_examples + buildspec: codebuild/py311/examples.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_examples_mpl + buildspec: codebuild/py311/examples_mpl.yml + env: + image: aws/codebuild/standard:7.0 # - identifier: py311_awses_latest # buildspec: codebuild/py311/awses_local.yml # env: @@ -175,69 +175,69 @@ batch: # env: # image: aws/codebuild/standard:7.0 # - identifier: py311_mplawses_latest_mpl - # buildspec: codebuild/py311/mplawses_local_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_dafny_esdk_vectors_masterkey - # buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_dafny_esdk_vectors_keyrings - # buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_net_401_vectors_masterkey - # buildspec: codebuild/py311/decrypt_net_401_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_net_401_vectors_keyrings - # buildspec: codebuild/py311/decrypt_net_401_vectors_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py311/generate_decrypt_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_masterkey_with_masterkey - # depend-on: - # - py311_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py311/decrypt_masterkey_with_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_masterkey_with_keyrings - # depend-on: - # - py311_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py311/decrypt_masterkey_with_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_masterkey_with_js - # depend-on: - # - py311_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py311/decrypt_masterkey_with_js.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py311/generate_decrypt_vectors_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_keyrings_with_masterkey - # depend-on: - # - py311_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py311/decrypt_keyrings_with_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_keyrings_with_keyrings - # depend-on: - # - py311_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py311/decrypt_keyrings_with_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_keyrings_with_js - # depend-on: - # - py311_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py311/decrypt_keyrings_with_js.yml - # env: - # image: aws/codebuild/standard:7.0 + # buildspec: codebuild/py311/mplawses_local_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_dafny_esdk_vectors_masterkey + buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_dafny_esdk_vectors_keyrings + buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_net_401_vectors_masterkey + buildspec: codebuild/py311/decrypt_net_401_vectors_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_net_401_vectors_keyrings + buildspec: codebuild/py311/decrypt_net_401_vectors_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_generate_decrypt_vectors_masterkey + buildspec: codebuild/py311/generate_decrypt_vectors_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_masterkey_with_masterkey + depend-on: + - py311_generate_decrypt_vectors_masterkey + buildspec: codebuild/py311/decrypt_masterkey_with_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_masterkey_with_keyrings + depend-on: + - py311_generate_decrypt_vectors_masterkey + buildspec: codebuild/py311/decrypt_masterkey_with_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_masterkey_with_js + depend-on: + - py311_generate_decrypt_vectors_masterkey + buildspec: codebuild/py311/decrypt_masterkey_with_js.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_generate_decrypt_vectors_keyrings + buildspec: codebuild/py311/generate_decrypt_vectors_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_keyrings_with_masterkey + depend-on: + - py311_generate_decrypt_vectors_keyrings + buildspec: codebuild/py311/decrypt_keyrings_with_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_keyrings_with_keyrings + depend-on: + - py311_generate_decrypt_vectors_keyrings + buildspec: codebuild/py311/decrypt_keyrings_with_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_keyrings_with_js + depend-on: + - py311_generate_decrypt_vectors_keyrings + buildspec: codebuild/py311/decrypt_keyrings_with_js.yml + env: + image: aws/codebuild/standard:7.0 # - identifier: py312_integ # buildspec: codebuild/py312/integ.yml diff --git a/codebuild/py311/decrypt_keyrings_with_js.yml b/codebuild/py311/decrypt_keyrings_with_js.yml new file mode 100644 index 000000000..578b83cab --- /dev/null +++ b/codebuild/py311/decrypt_keyrings_with_js.yml @@ -0,0 +1,34 @@ +version: 0.2 + +env: + variables: + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.11 + commands: + - n 16 + # Install the Javascript ESDK run test vectors + - npm install -g @aws-crypto/integration-node + + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyrings.zip 311_keyrings.zip + # Repackage zip in expected format + - unzip 311_keyrings.zip + - cd 311_keyrings + - zip -r vectors.zip . + build: + commands: + # Decrypt generated vectors with Javascript ESDK + - integration-node decrypt -v vectors.zip \ No newline at end of file diff --git a/codebuild/py311/decrypt_keyrings_with_keyrings.yml b/codebuild/py311/decrypt_keyrings_with_keyrings.yml new file mode 100644 index 000000000..5478fff38 --- /dev/null +++ b/codebuild/py311/decrypt_keyrings_with_keyrings.yml @@ -0,0 +1,31 @@ +version: 0.2 + +env: + variables: + TOXENV: "py311-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.11 + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyrings.zip 311_keyrings.zip + - unzip 311_keyrings.zip + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../311_keyrings/manifest.json \ + --keyrings \ No newline at end of file diff --git a/codebuild/py311/decrypt_keyrings_with_masterkey.yml b/codebuild/py311/decrypt_keyrings_with_masterkey.yml new file mode 100644 index 000000000..714882c54 --- /dev/null +++ b/codebuild/py311/decrypt_keyrings_with_masterkey.yml @@ -0,0 +1,30 @@ +version: 0.2 + +env: + variables: + TOXENV: "py311-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.11 + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyrings.zip 311_keyrings.zip + - unzip 311_keyrings.zip + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../311_keyrings/manifest.json \ No newline at end of file diff --git a/codebuild/py311/decrypt_masterkey_with_js.yml b/codebuild/py311/decrypt_masterkey_with_js.yml new file mode 100644 index 000000000..a73e93580 --- /dev/null +++ b/codebuild/py311/decrypt_masterkey_with_js.yml @@ -0,0 +1,34 @@ +version: 0.2 + +env: + variables: + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.11 + commands: + - n 16 + # Install the Javascript ESDK run test vectors + - npm install -g @aws-crypto/integration-node + + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_masterkey.zip 311_masterkey.zip + # Repackage zip in expected format + - unzip 311_masterkey.zip + - cd 311_masterkey + - zip -r vectors.zip . + build: + commands: + # Decrypt generated vectors with Javascript ESDK + - integration-node decrypt -v vectors.zip \ No newline at end of file diff --git a/codebuild/py311/decrypt_masterkey_with_keyrings.yml b/codebuild/py311/decrypt_masterkey_with_keyrings.yml new file mode 100644 index 000000000..5479ef16c --- /dev/null +++ b/codebuild/py311/decrypt_masterkey_with_keyrings.yml @@ -0,0 +1,31 @@ +version: 0.2 + +env: + variables: + TOXENV: "py311-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.11 + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_masterkey.zip 311_masterkey.zip + - unzip 311_masterkey.zip + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../311_masterkey/manifest.json \ + --keyrings \ No newline at end of file diff --git a/codebuild/py311/decrypt_masterkey_with_masterkey.yml b/codebuild/py311/decrypt_masterkey_with_masterkey.yml new file mode 100644 index 000000000..dd64d2dff --- /dev/null +++ b/codebuild/py311/decrypt_masterkey_with_masterkey.yml @@ -0,0 +1,30 @@ +version: 0.2 + +env: + variables: + TOXENV: "py311-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.11 + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_masterkey.zip 311_masterkey.zip + - unzip 311_masterkey.zip + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../311_masterkey/manifest.json \ No newline at end of file diff --git a/codebuild/py311/decrypt_net_401_vectors_keyrings.yml b/codebuild/py311/decrypt_net_401_vectors_keyrings.yml index 6634470c3..1a23f0917 100644 --- a/codebuild/py311/decrypt_net_401_vectors_keyrings.yml +++ b/codebuild/py311/decrypt_net_401_vectors_keyrings.yml @@ -12,8 +12,6 @@ env: arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" phases: install: @@ -21,14 +19,6 @@ phases: python: 3.11 pre_build: commands: - # Assume Role to access non-prod resource - - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") - - export TMP_ROLE - - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') - - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') - - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') - - aws sts get-caller-identity - # Fetch ESDK .NET v4.0.1 Test Vectors - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml index 3e28dc113..777a5703f 100644 --- a/codebuild/py311/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -11,8 +11,6 @@ env: arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" phases: install: @@ -22,15 +20,9 @@ phases: commands: - pip install "tox < 4.0" - cd test_vector_handlers - # - mkdir $CODEBUILD_RESOLVED_SOURCE_VERSION - | tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ - --output 311_keyring \ - --keyrings - # - zip -r $CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip 311_keyring - - zip -r 311_keyring.zip 311_keyring - - aws s3 cp 311_keyring.zip s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip -# artifacts: -# files: -# - test_vector_handlers/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip + --output 311_keyrings + - zip -r 311_keyrings.zip 311_keyrings + - aws s3 cp 311_keyrings.zip s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyrings.zip diff --git a/codebuild/py311/generate_decrypt_vectors_masterkey.yml b/codebuild/py311/generate_decrypt_vectors_masterkey.yml index f4056832a..84db3f176 100644 --- a/codebuild/py311/generate_decrypt_vectors_masterkey.yml +++ b/codebuild/py311/generate_decrypt_vectors_masterkey.yml @@ -11,8 +11,6 @@ env: arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" phases: install: @@ -27,8 +25,4 @@ phases: --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ --output 311_masterkey - zip -r 311_masterkey.zip 311_masterkey -artifacts: - files: - - test_vector_handlers/311_masterkey.zip - name: $CODEBUILD_INITIATOR/311_masterkey.zip - discard-paths: yes \ No newline at end of file + - aws s3 cp 311_masterkey.zip s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_masterkey.zip From 2c3b3800a83e2e6db1d294195aec8be1ffd656b8 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 11:08:51 -0700 Subject: [PATCH 347/376] 311 --- codebuild/py311/decrypt_keyrings_with_keyrings.yml | 2 +- codebuild/py311/decrypt_masterkey_with_keyrings.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/codebuild/py311/decrypt_keyrings_with_keyrings.yml b/codebuild/py311/decrypt_keyrings_with_keyrings.yml index 5478fff38..fec275d48 100644 --- a/codebuild/py311/decrypt_keyrings_with_keyrings.yml +++ b/codebuild/py311/decrypt_keyrings_with_keyrings.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - TOXENV: "py311-full_decrypt" + TOXENV: "py311-full_decrypt-mpl" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/codebuild/py311/decrypt_masterkey_with_keyrings.yml b/codebuild/py311/decrypt_masterkey_with_keyrings.yml index 5479ef16c..8543077bd 100644 --- a/codebuild/py311/decrypt_masterkey_with_keyrings.yml +++ b/codebuild/py311/decrypt_masterkey_with_keyrings.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - TOXENV: "py311-full_decrypt" + TOXENV: "py311-full_decrypt-mpl" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- From c8ca704f4c7b4a79affb9ea4e85652630a305363 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 11:09:01 -0700 Subject: [PATCH 348/376] 311 --- .../decrypt_dafny_esdk_vectors_keyrings.yml | 59 +++++++++++++++++++ .../decrypt_dafny_esdk_vectors_masterkey.yml | 58 ++++++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml create mode 100644 codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml diff --git a/codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml b/codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml new file mode 100644 index 000000000..d69ce9370 --- /dev/null +++ b/codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml @@ -0,0 +1,59 @@ +version: 0.2 +# Runs Only the ESDK-NET v4.0.1 Decryption Vectors, testing Required EC CMM + +env: + variables: + TOXENV: "py311-full_decrypt-mpl" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + git-credential-helper: yes + secrets-manager: + GITHUB_TOKEN: Github/lucasmcdonald3-fgpat:actions read + +phases: + install: + runtime-versions: + python: 3.11 + pre_build: + commands: + # Fetch test vectors from Dafny ESDK's most recent run + # (Assuming the first result is most recent; seems to be correct...) + - | + MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ + | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') + - | + echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" + - | + MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/$MOST_RECENT_RUN_ID/artifacts?name=ubuntu-latest_vector_artifact" \ + | jq '.artifacts[0].archive_download_url') + - | + echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" + - | + curl -L -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip + # This unzips to `net41.zip`. + - unzip ubuntu-latest_test_vector_artifact + # This unzips to `net41/`. + - unzip net41.zip -d net41 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../net41/manifest.json \ + --keyrings diff --git a/codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml b/codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml new file mode 100644 index 000000000..6106906b5 --- /dev/null +++ b/codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml @@ -0,0 +1,58 @@ +version: 0.2 +# Runs Only the ESDK-NET v4.0.1 Decryption Vectors, testing Required EC CMM + +env: + variables: + TOXENV: "py311-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + git-credential-helper: yes + secrets-manager: + GITHUB_TOKEN: Github/lucasmcdonald3-fgpat:actions read + +phases: + install: + runtime-versions: + python: 3.11 + pre_build: + commands: + # Fetch test vectors from Dafny ESDK's most recent run + # (Assuming the first result is most recent; seems to be correct...) + - | + MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ + | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') + - | + echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" + - | + MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/$MOST_RECENT_RUN_ID/artifacts?name=ubuntu-latest_vector_artifact" \ + | jq '.artifacts[0].archive_download_url') + - | + echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" + - | + curl -L -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip + # This unzips to `net41.zip`. + - unzip ubuntu-latest_test_vector_artifact + # This unzips to `net41/`. + - unzip net41.zip -d net41 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../net41/manifest.json From c8c5a4096e57cb361dc394dc58dd5fdcf5a01a3e Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 11:17:42 -0700 Subject: [PATCH 349/376] py311 --- codebuild/py311/encrypt_keyrings.yml | 26 ++++++++++++++++++++++++++ codebuild/py311/encrypt_masterkey.yml | 25 +++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 codebuild/py311/encrypt_keyrings.yml create mode 100644 codebuild/py311/encrypt_masterkey.yml diff --git a/codebuild/py311/encrypt_keyrings.yml b/codebuild/py311/encrypt_keyrings.yml new file mode 100644 index 000000000..8b7cb94b4 --- /dev/null +++ b/codebuild/py311/encrypt_keyrings.yml @@ -0,0 +1,26 @@ +version: 0.2 + +env: + variables: + TOXENV: "py311-full_encrypt-mpl" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.11 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0003-awses-message-encryption.v2.json \ + --keyrings \ No newline at end of file diff --git a/codebuild/py311/encrypt_masterkey.yml b/codebuild/py311/encrypt_masterkey.yml new file mode 100644 index 000000000..226e1586d --- /dev/null +++ b/codebuild/py311/encrypt_masterkey.yml @@ -0,0 +1,25 @@ +version: 0.2 + +env: + variables: + TOXENV: "py311-full_encrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.11 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0003-awses-message-encryption.v2.json From 8a0ddc4441390c8966e04d593e2b6e9830e8b911 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 11:34:23 -0700 Subject: [PATCH 350/376] 311 --- buildspec.yml | 8 ++++++++ codebuild/py37/encrypt_masterkey.yml | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/buildspec.yml b/buildspec.yml index 3c4b965de..80c04b2ff 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -194,6 +194,14 @@ batch: buildspec: codebuild/py311/decrypt_net_401_vectors_keyrings.yml env: image: aws/codebuild/standard:7.0 + - identifier: py311_encrypt_masterkey + buildspec: codebuild/py311/encrypt_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_encrypt_keyrings + buildspec: codebuild/py311/encrypt_keyrings.yml + env: + image: aws/codebuild/standard:5.0 - identifier: py311_generate_decrypt_vectors_masterkey buildspec: codebuild/py311/generate_decrypt_vectors_masterkey.yml env: diff --git a/codebuild/py37/encrypt_masterkey.yml b/codebuild/py37/encrypt_masterkey.yml index b5cb57dae..7cdc7848e 100644 --- a/codebuild/py37/encrypt_masterkey.yml +++ b/codebuild/py37/encrypt_masterkey.yml @@ -22,4 +22,4 @@ phases: - cd test_vector_handlers - | tox -- \ - --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0003-awses-message-encryption.v2.json \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0003-awses-message-encryption.v2.json From 20262d78e9260d8ade44bbf1a34e5f3a1cdc4eff Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 11:38:17 -0700 Subject: [PATCH 351/376] 311 --- buildspec.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index 80c04b2ff..57b86cb82 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -197,11 +197,11 @@ batch: - identifier: py311_encrypt_masterkey buildspec: codebuild/py311/encrypt_masterkey.yml env: - image: aws/codebuild/standard:5.0 + image: aws/codebuild/standard:7.0 - identifier: py311_encrypt_keyrings buildspec: codebuild/py311/encrypt_keyrings.yml env: - image: aws/codebuild/standard:5.0 + image: aws/codebuild/standard:7.0 - identifier: py311_generate_decrypt_vectors_masterkey buildspec: codebuild/py311/generate_decrypt_vectors_masterkey.yml env: From 89efb749c72b8ce49bcabaff3fddb084f4859c6d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 11:56:51 -0700 Subject: [PATCH 352/376] 312 --- .../py312/decrypt_keyrings_with_keyrings.yml | 31 +++++++++++++ .../py312/decrypt_keyrings_with_masterkey.yml | 30 +++++++++++++ codebuild/py312/decrypt_masterkey_with_js.yml | 34 ++++++++++++++ .../py312/decrypt_masterkey_with_keyrings.yml | 31 +++++++++++++ .../decrypt_masterkey_with_masterkey.yml | 30 +++++++++++++ .../decrypt_net_401_vectors_keyrings.yml | 36 +++++++++++++++ .../decrypt_net_401_vectors_masterkey.yml | 45 +++++++++++++++++++ .../{awses_local.yml => encrypt_keyrings.yml} | 16 +++---- codebuild/py312/encrypt_masterkey.yml | 25 +++++++++++ codebuild/py312/examples.yml | 7 +-- codebuild/py312/examples_mpl.yml | 13 ++---- .../generate_decrypt_vectors_keyrings.yml | 28 ++++++++++++ ...=> generate_decrypt_vectors_masterkey.yml} | 21 ++++----- codebuild/py312/integ.yml | 7 +-- codebuild/py312/integ_mpl.yml | 9 +--- codebuild/py312/mplawses_local_mpl.yml | 8 +--- 16 files changed, 311 insertions(+), 60 deletions(-) create mode 100644 codebuild/py312/decrypt_keyrings_with_keyrings.yml create mode 100644 codebuild/py312/decrypt_keyrings_with_masterkey.yml create mode 100644 codebuild/py312/decrypt_masterkey_with_js.yml create mode 100644 codebuild/py312/decrypt_masterkey_with_keyrings.yml create mode 100644 codebuild/py312/decrypt_masterkey_with_masterkey.yml create mode 100644 codebuild/py312/decrypt_net_401_vectors_keyrings.yml create mode 100644 codebuild/py312/decrypt_net_401_vectors_masterkey.yml rename codebuild/py312/{awses_local.yml => encrypt_keyrings.yml} (64%) create mode 100644 codebuild/py312/encrypt_masterkey.yml create mode 100644 codebuild/py312/generate_decrypt_vectors_keyrings.yml rename codebuild/py312/{awses_local_mpl.yml => generate_decrypt_vectors_masterkey.yml} (56%) diff --git a/codebuild/py312/decrypt_keyrings_with_keyrings.yml b/codebuild/py312/decrypt_keyrings_with_keyrings.yml new file mode 100644 index 000000000..3ab7058f9 --- /dev/null +++ b/codebuild/py312/decrypt_keyrings_with_keyrings.yml @@ -0,0 +1,31 @@ +version: 0.2 + +env: + variables: + TOXENV: "py312-full_decrypt-mpl" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.12 + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_keyrings.zip 312_keyrings.zip + - unzip 312_keyrings.zip + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../312_keyrings/manifest.json \ + --keyrings \ No newline at end of file diff --git a/codebuild/py312/decrypt_keyrings_with_masterkey.yml b/codebuild/py312/decrypt_keyrings_with_masterkey.yml new file mode 100644 index 000000000..bb06ba4a2 --- /dev/null +++ b/codebuild/py312/decrypt_keyrings_with_masterkey.yml @@ -0,0 +1,30 @@ +version: 0.2 + +env: + variables: + TOXENV: "py312-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.12 + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_keyrings.zip 312_keyrings.zip + - unzip 312_keyrings.zip + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../312_keyrings/manifest.json \ No newline at end of file diff --git a/codebuild/py312/decrypt_masterkey_with_js.yml b/codebuild/py312/decrypt_masterkey_with_js.yml new file mode 100644 index 000000000..7c57c3111 --- /dev/null +++ b/codebuild/py312/decrypt_masterkey_with_js.yml @@ -0,0 +1,34 @@ +version: 0.2 + +env: + variables: + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.12 + commands: + - n 16 + # Install the Javascript ESDK run test vectors + - npm install -g @aws-crypto/integration-node + + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_masterkey.zip 312_masterkey.zip + # Repackage zip in expected format + - unzip 312_masterkey.zip + - cd 312_masterkey + - zip -r vectors.zip . + build: + commands: + # Decrypt generated vectors with Javascript ESDK + - integration-node decrypt -v vectors.zip \ No newline at end of file diff --git a/codebuild/py312/decrypt_masterkey_with_keyrings.yml b/codebuild/py312/decrypt_masterkey_with_keyrings.yml new file mode 100644 index 000000000..21f646370 --- /dev/null +++ b/codebuild/py312/decrypt_masterkey_with_keyrings.yml @@ -0,0 +1,31 @@ +version: 0.2 + +env: + variables: + TOXENV: "py312-full_decrypt-mpl" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.12 + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_masterkey.zip 312_masterkey.zip + - unzip 312_masterkey.zip + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../312_masterkey/manifest.json \ + --keyrings \ No newline at end of file diff --git a/codebuild/py312/decrypt_masterkey_with_masterkey.yml b/codebuild/py312/decrypt_masterkey_with_masterkey.yml new file mode 100644 index 000000000..0529fd894 --- /dev/null +++ b/codebuild/py312/decrypt_masterkey_with_masterkey.yml @@ -0,0 +1,30 @@ +version: 0.2 + +env: + variables: + TOXENV: "py312-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.12 + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_masterkey.zip 312_masterkey.zip + - unzip 312_masterkey.zip + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../312_masterkey/manifest.json \ No newline at end of file diff --git a/codebuild/py312/decrypt_net_401_vectors_keyrings.yml b/codebuild/py312/decrypt_net_401_vectors_keyrings.yml new file mode 100644 index 000000000..aec3916e5 --- /dev/null +++ b/codebuild/py312/decrypt_net_401_vectors_keyrings.yml @@ -0,0 +1,36 @@ +version: 0.2 +# Runs Only the ESDK-NET v4.0.1 Decryption Vectors, testing Required EC CMM + +env: + variables: + TOXENV: "py312-full_decrypt-mpl" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.12 + pre_build: + commands: + # Fetch ESDK .NET v4.0.1 Test Vectors + - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip + - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip + - curl -s --output $VECTOR_ZIP --location $VECTORS_URL + - UNZIPPED_VECTORS_DIR=$CODEBUILD_SRC_DIR/test_vector_handlers/net_401_vectors + - unzip $VECTOR_ZIP -d $UNZIPPED_VECTORS_DIR + build: + commands: + # NOTE: We need to pass the absolute path of the vectors + - pip install "tox < 4.0" + - cd $CODEBUILD_SRC_DIR/test_vector_handlers + - | + tox -- \ + --input $UNZIPPED_VECTORS_DIR/manifest.json \ + --keyrings diff --git a/codebuild/py312/decrypt_net_401_vectors_masterkey.yml b/codebuild/py312/decrypt_net_401_vectors_masterkey.yml new file mode 100644 index 000000000..5d1ef9d94 --- /dev/null +++ b/codebuild/py312/decrypt_net_401_vectors_masterkey.yml @@ -0,0 +1,45 @@ +version: 0.2 +# Runs Only the ESDK-NET v4.0.1 Decryption Vectors, testing Required EC CMM + +env: + variables: + TOXENV: "py312-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + python: 3.12 + pre_build: + commands: + # Assume Role to access non-prod resources + - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") + - export TMP_ROLE + - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') + - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') + - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') + - aws sts get-caller-identity + + # Fetch ESDK .NET v4.0.1 Test Vectors + - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip + - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip + - curl -s --output $VECTOR_ZIP --location $VECTORS_URL + - UNZIPPED_VECTORS_DIR=$CODEBUILD_SRC_DIR/test_vector_handlers/net_401_vectors + - unzip $VECTOR_ZIP -d $UNZIPPED_VECTORS_DIR + build: + commands: + # NOTE: We need to pass the absolute path of the vectors + - pip install "tox < 4.0" + - cd $CODEBUILD_SRC_DIR/test_vector_handlers + - | + tox -- \ + --input $UNZIPPED_VECTORS_DIR/manifest.json \ No newline at end of file diff --git a/codebuild/py312/awses_local.yml b/codebuild/py312/encrypt_keyrings.yml similarity index 64% rename from codebuild/py312/awses_local.yml rename to codebuild/py312/encrypt_keyrings.yml index 0a81984ee..56a389e6f 100644 --- a/codebuild/py312/awses_local.yml +++ b/codebuild/py312/encrypt_keyrings.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - TOXENV: "py312-awses_local" + TOXENV: "py312-full_encrypt-mpl" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- @@ -11,20 +11,16 @@ env: arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" phases: install: runtime-versions: - python: latest + python: 3.12 build: commands: - - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - - - pyenv install --skip-existing 3.12.0 - - pyenv local 3.12.0 - - pip install --upgrade pip - - pip install setuptools - pip install "tox < 4.0" - cd test_vector_handlers - - tox + - | + tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0003-awses-message-encryption.v2.json \ + --keyrings \ No newline at end of file diff --git a/codebuild/py312/encrypt_masterkey.yml b/codebuild/py312/encrypt_masterkey.yml new file mode 100644 index 000000000..940f336a2 --- /dev/null +++ b/codebuild/py312/encrypt_masterkey.yml @@ -0,0 +1,25 @@ +version: 0.2 + +env: + variables: + TOXENV: "py312-full_encrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.12 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0003-awses-message-encryption.v2.json diff --git a/codebuild/py312/examples.yml b/codebuild/py312/examples.yml index 691ea0e60..855a8fcdb 100644 --- a/codebuild/py312/examples.yml +++ b/codebuild/py312/examples.yml @@ -15,13 +15,8 @@ env: phases: install: runtime-versions: - python: latest + python: 3.12 build: commands: - - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - - - pyenv install --skip-existing 3.12.0 - - pyenv local 3.12.0 - - pip install --upgrade pip - - pip install setuptools - pip install "tox < 4.0" - tox diff --git a/codebuild/py312/examples_mpl.yml b/codebuild/py312/examples_mpl.yml index 366222441..86774df0e 100644 --- a/codebuild/py312/examples_mpl.yml +++ b/codebuild/py312/examples_mpl.yml @@ -1,6 +1,3 @@ -# Runs the same tests as examples in an environment with the MPL installed -# to assert existing tests continue to pass with the MPL installed. -# Then, run MPL-specific tests. version: 0.2 env: @@ -19,19 +16,14 @@ env: phases: install: runtime-versions: - python: latest + python: 3.12 build: commands: - - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - - - pyenv install --skip-existing 3.12.0 - - pyenv local 3.12.0 - - pip install --upgrade pip - - pip install setuptools - pip install "tox < 4.0" # Run non-MPL-specific tests with the MPL installed - tox -e py312-examples-mpl # Assume special role to access keystore - - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-Py311ExamplesMpl") + - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-Py312ExamplesMpl") - export TMP_ROLE - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') @@ -39,3 +31,4 @@ phases: - aws sts get-caller-identity # Run MPL-specific tests with special role - tox -e py312-mplexamples-mpl + diff --git a/codebuild/py312/generate_decrypt_vectors_keyrings.yml b/codebuild/py312/generate_decrypt_vectors_keyrings.yml new file mode 100644 index 000000000..51a1415ee --- /dev/null +++ b/codebuild/py312/generate_decrypt_vectors_keyrings.yml @@ -0,0 +1,28 @@ +version: 0.2 + +env: + variables: + TOXENV: "py312-full_decrypt_generate-mpl" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.12 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ + --output 312_keyrings + - zip -r 312_keyrings.zip 312_keyrings + - aws s3 cp 312_keyrings.zip s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_keyrings.zip diff --git a/codebuild/py312/awses_local_mpl.yml b/codebuild/py312/generate_decrypt_vectors_masterkey.yml similarity index 56% rename from codebuild/py312/awses_local_mpl.yml rename to codebuild/py312/generate_decrypt_vectors_masterkey.yml index 96ca5bc28..1fadba985 100644 --- a/codebuild/py312/awses_local_mpl.yml +++ b/codebuild/py312/generate_decrypt_vectors_masterkey.yml @@ -1,11 +1,8 @@ -# Runs test vectors using native constructs in an environment with the MPL installed. -# This asserts that installing the MPL does not change existing behavior. version: 0.2 env: variables: - TOXENV: "py312-awses_local-mpl" - REGION: "us-west-2" + TOXENV: "py312-full_decrypt_generate" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- @@ -14,20 +11,18 @@ env: arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" phases: install: runtime-versions: - python: latest + python: 3.12 build: commands: - - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - - - pyenv install --skip-existing 3.12.0 - - pyenv local 3.12.0 - - pip install --upgrade pip - - pip install setuptools - pip install "tox < 4.0" - cd test_vector_handlers - - tox + - | + tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ + --output 312_masterkey + - zip -r 312_masterkey.zip 312_masterkey + - aws s3 cp 312_masterkey.zip s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_masterkey.zip diff --git a/codebuild/py312/integ.yml b/codebuild/py312/integ.yml index 10899f1df..2ccad8913 100644 --- a/codebuild/py312/integ.yml +++ b/codebuild/py312/integ.yml @@ -15,13 +15,8 @@ env: phases: install: runtime-versions: - python: latest + python: 3.12 build: commands: - - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - - - pyenv install --skip-existing 3.12.0 - - pyenv local 3.12.0 - - pip install --upgrade pip - - pip install setuptools - pip install "tox < 4.0" - tox diff --git a/codebuild/py312/integ_mpl.yml b/codebuild/py312/integ_mpl.yml index e292acc57..28bbaa422 100644 --- a/codebuild/py312/integ_mpl.yml +++ b/codebuild/py312/integ_mpl.yml @@ -1,5 +1,3 @@ -# Runs the same tests as integ in an environment with the MPL installed. -# This asserts existing tests continue to pass with the MPL installed. version: 0.2 env: @@ -18,13 +16,8 @@ env: phases: install: runtime-versions: - python: latest + python: 3.12 build: commands: - - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - - - pyenv install --skip-existing 3.12.0 - - pyenv local 3.12.0 - - pip install --upgrade pip - - pip install setuptools - pip install "tox < 4.0" - tox diff --git a/codebuild/py312/mplawses_local_mpl.yml b/codebuild/py312/mplawses_local_mpl.yml index e11f7523b..8a7d5f5c6 100644 --- a/codebuild/py312/mplawses_local_mpl.yml +++ b/codebuild/py312/mplawses_local_mpl.yml @@ -1,4 +1,3 @@ -# Runs test vectors using MPL constructs. version: 0.2 env: @@ -19,14 +18,9 @@ env: phases: install: runtime-versions: - python: latest + python: 3.12 build: commands: - - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - - - pyenv install --skip-existing 3.12.0 - - pyenv local 3.12.0 - - pip install --upgrade pip - - pip install setuptools - pip install "tox < 4.0" - cd test_vector_handlers - tox From e1700b9910e42d342e8c24c05a774af0d020d01f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 12:01:49 -0700 Subject: [PATCH 353/376] ALL --- buildspec.yml | 370 +++++++++++++++++++------------------------------- 1 file changed, 143 insertions(+), 227 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index 57b86cb82..e303d9b35 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -3,152 +3,82 @@ version: 0.2 batch: fast-fail: false build-graph: - - identifier: py37_integ - buildspec: codebuild/py37/integ.yml + + # 3.8 + - identifier: py38_integ + buildspec: codebuild/py38/integ.yml env: image: aws/codebuild/standard:5.0 - - identifier: py37_examples - buildspec: codebuild/py37/examples.yml + - identifier: py38_examples + buildspec: codebuild/py38/examples.yml env: image: aws/codebuild/standard:5.0 - - identifier: py37_decrypt_dafny_esdk_vectors - buildspec: codebuild/py37/decrypt_dafny_esdk_vectors.yml + - identifier: py38_decrypt_dafny_esdk_vectors + buildspec: codebuild/py38/decrypt_dafny_esdk_vectors.yml env: image: aws/codebuild/standard:5.0 - - identifier: py37_decrypt_net_401_vectors - buildspec: codebuild/py37/decrypt_net_401_vectors.yml + - identifier: py38_decrypt_net_401_vectors + buildspec: codebuild/py38/decrypt_net_401_vectors.yml env: image: aws/codebuild/standard:5.0 - - identifier: py37_encrypt_masterkey - buildspec: codebuild/py37/encrypt_masterkey.yml + - identifier: py38_encrypt_masterkey + buildspec: codebuild/py38/encrypt_masterkey.yml env: image: aws/codebuild/standard:5.0 - - identifier: py37_generate_decrypt_vectors_masterkey - buildspec: codebuild/py37/generate_decrypt_vectors_masterkey.yml + - identifier: py38_generate_decrypt_vectors_masterkey + buildspec: codebuild/py38/generate_decrypt_vectors_masterkey.yml env: image: aws/codebuild/standard:5.0 - - identifier: py37_decrypt_masterkey_with_masterkey + - identifier: py38_decrypt_masterkey_with_masterkey depend-on: - - py37_generate_decrypt_vectors_masterkey - buildspec: codebuild/py37/decrypt_masterkey_with_masterkey.yml + - py38_generate_decrypt_vectors_masterkey + buildspec: codebuild/py38/decrypt_masterkey_with_masterkey.yml env: image: aws/codebuild/standard:5.0 - - identifier: py37_decrypt_masterkey_with_js + - identifier: py38_decrypt_masterkey_with_js depend-on: - - py37_generate_decrypt_vectors_masterkey - buildspec: codebuild/py37/decrypt_masterkey_with_js.yml + - py38_generate_decrypt_vectors_masterkey + buildspec: codebuild/py38/decrypt_masterkey_with_js.yml env: image: aws/codebuild/standard:5.0 - # - identifier: py38_integ - # buildspec: codebuild/py38/integ.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py38_examples - # buildspec: codebuild/py38/examples.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py38_awses_local - # buildspec: codebuild/py38/awses_local.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py38_decrypt_dafny_esdk_vectors - # buildspec: codebuild/py38/decrypt_dafny_esdk_vectors.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py38_decrypt_net_401_vectors - # buildspec: codebuild/py38/decrypt_net_401_vectors.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py38_generate_decrypt_vectors - # buildspec: codebuild/py38/generate_decrypt_vectors.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py38_decrypt_masterkey_with_masterkey - # depend-on: - # - py38_generate_decrypt_vectors - # buildspec: codebuild/py38/decrypt_masterkey_with_masterkey.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py38_decrypt_masterkey_with_js - # depend-on: - # - py38_generate_decrypt_vectors - # buildspec: codebuild/py38/decrypt_generated_with_js.yml - # env: - # image: aws/codebuild/standard:5.0 - - # - identifier: py39_integ - # buildspec: codebuild/py39/integ.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py39_examples - # buildspec: codebuild/py39/examples.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py39_awses_latest - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py39_decrypt_dafny_esdk_vectors - # buildspec: codebuild/py39/decrypt_dafny_esdk_vectors.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py39_decrypt_net_401_vectors - # buildspec: codebuild/py39/decrypt_net_401_vectors.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py39_generate_decrypt_vectors - # buildspec: codebuild/py39/generate_decrypt_vectors.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py39_decrypt_masterkey_with_masterkey - # depend-on: - # - py39_generate_decrypt_vectors - # buildspec: codebuild/py39/decrypt_masterkey_with_masterkey.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py39_decrypt_masterkey_with_js - # depend-on: - # - py39_generate_decrypt_vectors - # buildspec: codebuild/py39/decrypt_generated_with_js.yml - # env: - # image: aws/codebuild/standard:5.0 - - # - identifier: py310_integ - # buildspec: codebuild/py310/integ.yml - # env: - # image: aws/codebuild/standard:6.0 - # - identifier: py310_examples - # buildspec: codebuild/py310/examples.yml - # env: - # image: aws/codebuild/standard:6.0 - # - identifier: py310_awses_latest - # buildspec: codebuild/py310/awses_local.yml - # env: - # image: aws/codebuild/standard:6.0 - # - identifier: py310_decrypt_dafny_esdk_vectors - # buildspec: codebuild/py310/decrypt_dafny_esdk_vectors.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py310_decrypt_net_401_vectors - # buildspec: codebuild/py310/decrypt_net_401_vectors.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py310_generate_decrypt_vectors - # buildspec: codebuild/py310/generate_decrypt_vectors.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py310_decrypt_masterkey_with_masterkey - # depend-on: - # - py310_generate_decrypt_vectors - # buildspec: codebuild/py310/decrypt_masterkey_with_masterkey.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py310_decrypt_masterkey_with_js - # depend-on: - # - py310_generate_decrypt_vectors - # buildspec: codebuild/py310/decrypt_generated_with_js.yml - # env: - # image: aws/codebuild/standard:5.0 + # 3.9 + - identifier: py39_integ + buildspec: codebuild/py39/integ.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_examples + buildspec: codebuild/py39/examples.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_decrypt_dafny_esdk_vectors + buildspec: codebuild/py39/decrypt_dafny_esdk_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_decrypt_net_401_vectors + buildspec: codebuild/py39/decrypt_net_401_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_encrypt_masterkey + buildspec: codebuild/py39/encrypt_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_generate_decrypt_vectors_masterkey + buildspec: codebuild/py39/generate_decrypt_vectors_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_decrypt_masterkey_with_masterkey + depend-on: + - py39_generate_decrypt_vectors_masterkey + buildspec: codebuild/py39/decrypt_masterkey_with_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_decrypt_masterkey_with_js + depend-on: + - py39_generate_decrypt_vectors_masterkey + buildspec: codebuild/py39/decrypt_masterkey_with_js.yml + env: + image: aws/codebuild/standard:5.0 - identifier: py311_integ buildspec: codebuild/py311/integ.yml @@ -166,18 +96,6 @@ batch: buildspec: codebuild/py311/examples_mpl.yml env: image: aws/codebuild/standard:7.0 - # - identifier: py311_awses_latest - # buildspec: codebuild/py311/awses_local.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_awses_latest_mpl - # buildspec: codebuild/py311/awses_local_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_mplawses_latest_mpl - # buildspec: codebuild/py311/mplawses_local_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 - identifier: py311_decrypt_dafny_esdk_vectors_masterkey buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml env: @@ -246,95 +164,93 @@ batch: buildspec: codebuild/py311/decrypt_keyrings_with_js.yml env: image: aws/codebuild/standard:7.0 + + + - identifier: py312_integ + buildspec: codebuild/py312/integ.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_integ_mpl + buildspec: codebuild/py312/integ_mpl.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_examples + buildspec: codebuild/py312/examples.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_examples_mpl + buildspec: codebuild/py312/examples_mpl.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_dafny_esdk_vectors_masterkey + buildspec: codebuild/py312/decrypt_dafny_esdk_vectors_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_dafny_esdk_vectors_keyrings + buildspec: codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_net_401_vectors_masterkey + buildspec: codebuild/py312/decrypt_net_401_vectors_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_net_401_vectors_keyrings + buildspec: codebuild/py312/decrypt_net_401_vectors_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_encrypt_masterkey + buildspec: codebuild/py312/encrypt_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_encrypt_keyrings + buildspec: codebuild/py312/encrypt_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_generate_decrypt_vectors_masterkey + buildspec: codebuild/py312/generate_decrypt_vectors_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_masterkey_with_masterkey + depend-on: + - py312_generate_decrypt_vectors_masterkey + buildspec: codebuild/py312/decrypt_masterkey_with_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_masterkey_with_keyrings + depend-on: + - py312_generate_decrypt_vectors_masterkey + buildspec: codebuild/py312/decrypt_masterkey_with_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_masterkey_with_js + depend-on: + - py312_generate_decrypt_vectors_masterkey + buildspec: codebuild/py312/decrypt_masterkey_with_js.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_generate_decrypt_vectors_keyrings + buildspec: codebuild/py312/generate_decrypt_vectors_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_keyrings_with_masterkey + depend-on: + - py312_generate_decrypt_vectors_keyrings + buildspec: codebuild/py312/decrypt_keyrings_with_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_keyrings_with_keyrings + depend-on: + - py312_generate_decrypt_vectors_keyrings + buildspec: codebuild/py312/decrypt_keyrings_with_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_keyrings_with_js + depend-on: + - py312_generate_decrypt_vectors_keyrings + buildspec: codebuild/py312/decrypt_keyrings_with_js.yml + env: + image: aws/codebuild/standard:7.0 - # - identifier: py312_integ - # buildspec: codebuild/py312/integ.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_integ_mpl - # buildspec: codebuild/py312/integ_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_examples - # buildspec: codebuild/py312/examples.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_examples_mpl - # buildspec: codebuild/py312/examples_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_awses_latest - # buildspec: codebuild/py312/awses_local.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_awses_latest_mpl - # buildspec: codebuild/py312/awses_local_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_mplawses_latest_mpl - # buildspec: codebuild/py312/mplawses_local_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_dafny_esdk_vectors_masterkey - # buildspec: codebuild/py312/decrypt_dafny_esdk_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_dafny_esdk_vectors_keyrings - # buildspec: codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_net_401_vectors_masterkey - # buildspec: codebuild/py312/decrypt_net_401_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_net_401_vectors_keyrings - # buildspec: codebuild/py312/decrypt_net_401_vectors_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py312/generate_decrypt_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_masterkey_with_masterkey - # depend-on: - # - py312_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py312/decrypt_masterkey_with_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_masterkey_with_keyrings - # depend-on: - # - py312_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py312/decrypt_masterkey_with_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_masterkey_with_js - # depend-on: - # - py312_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py312/decrypt_masterkey_with_js.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py312/generate_decrypt_vectors_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_keyrings_with_masterkey - # depend-on: - # - py312_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py312/decrypt_keyrings_with_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_keyrings_with_keyrings - # depend-on: - # - py312_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py312/decrypt_keyrings_with_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_keyrings_with_js - # depend-on: - # - py312_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py312/decrypt_keyrings_with_js.yml - # env: - # image: aws/codebuild/standard:7.0 # # - identifier: code_coverage # buildspec: codebuild/coverage/coverage.yml From 1d7fcaee7dad7f31165108066c898b06a6bbe5c1 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 12:01:57 -0700 Subject: [PATCH 354/376] ALL --- .../py310/decrypt_dafny_esdk_vectors.yml | 58 ++++++++++++++++++ codebuild/py310/decrypt_masterkey_with_js.yml | 34 +++++++++++ .../decrypt_masterkey_with_masterkey.yml | 30 ++++++++++ codebuild/py310/decrypt_net_401_vectors.yml | 35 +++++++++++ codebuild/py310/encrypt_masterkey.yml | 25 ++++++++ codebuild/py310/examples copy.yml | 22 +++++++ .../generate_decrypt_vectors_masterkey.yml | 28 +++++++++ codebuild/py310/integ copy.yml | 22 +++++++ codebuild/py312/awses_local.yml | 25 ++++++++ codebuild/py312/awses_local_mpl.yml | 26 ++++++++ .../decrypt_dafny_esdk_vectors_keyrings.yml | 59 +++++++++++++++++++ .../decrypt_dafny_esdk_vectors_masterkey.yml | 58 ++++++++++++++++++ codebuild/py312/decrypt_keyrings_with_js.yml | 34 +++++++++++ codebuild/py38/decrypt_dafny_esdk_vectors.yml | 58 ++++++++++++++++++ codebuild/py38/decrypt_masterkey_with_js.yml | 34 +++++++++++ .../py38/decrypt_masterkey_with_masterkey.yml | 30 ++++++++++ codebuild/py38/decrypt_net_401_vectors.yml | 35 +++++++++++ codebuild/py38/encrypt_masterkey.yml | 25 ++++++++ .../generate_decrypt_vectors_masterkey.yml | 28 +++++++++ codebuild/py39/awses_local.yml | 25 ++++++++ codebuild/py39/decrypt_dafny_esdk_vectors.yml | 58 ++++++++++++++++++ codebuild/py39/decrypt_masterkey_with_js.yml | 34 +++++++++++ .../py39/decrypt_masterkey_with_masterkey.yml | 30 ++++++++++ codebuild/py39/decrypt_net_401_vectors.yml | 35 +++++++++++ codebuild/py39/encrypt_masterkey.yml | 25 ++++++++ codebuild/py39/examples copy.yml | 22 +++++++ .../generate_decrypt_vectors_masterkey.yml | 28 +++++++++ codebuild/py39/integ copy.yml | 22 +++++++ 28 files changed, 945 insertions(+) create mode 100644 codebuild/py310/decrypt_dafny_esdk_vectors.yml create mode 100644 codebuild/py310/decrypt_masterkey_with_js.yml create mode 100644 codebuild/py310/decrypt_masterkey_with_masterkey.yml create mode 100644 codebuild/py310/decrypt_net_401_vectors.yml create mode 100644 codebuild/py310/encrypt_masterkey.yml create mode 100644 codebuild/py310/examples copy.yml create mode 100644 codebuild/py310/generate_decrypt_vectors_masterkey.yml create mode 100644 codebuild/py310/integ copy.yml create mode 100644 codebuild/py312/awses_local.yml create mode 100644 codebuild/py312/awses_local_mpl.yml create mode 100644 codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml create mode 100644 codebuild/py312/decrypt_dafny_esdk_vectors_masterkey.yml create mode 100644 codebuild/py312/decrypt_keyrings_with_js.yml create mode 100644 codebuild/py38/decrypt_dafny_esdk_vectors.yml create mode 100644 codebuild/py38/decrypt_masterkey_with_js.yml create mode 100644 codebuild/py38/decrypt_masterkey_with_masterkey.yml create mode 100644 codebuild/py38/decrypt_net_401_vectors.yml create mode 100644 codebuild/py38/encrypt_masterkey.yml create mode 100644 codebuild/py38/generate_decrypt_vectors_masterkey.yml create mode 100644 codebuild/py39/awses_local.yml create mode 100644 codebuild/py39/decrypt_dafny_esdk_vectors.yml create mode 100644 codebuild/py39/decrypt_masterkey_with_js.yml create mode 100644 codebuild/py39/decrypt_masterkey_with_masterkey.yml create mode 100644 codebuild/py39/decrypt_net_401_vectors.yml create mode 100644 codebuild/py39/encrypt_masterkey.yml create mode 100644 codebuild/py39/examples copy.yml create mode 100644 codebuild/py39/generate_decrypt_vectors_masterkey.yml create mode 100644 codebuild/py39/integ copy.yml diff --git a/codebuild/py310/decrypt_dafny_esdk_vectors.yml b/codebuild/py310/decrypt_dafny_esdk_vectors.yml new file mode 100644 index 000000000..505f3157c --- /dev/null +++ b/codebuild/py310/decrypt_dafny_esdk_vectors.yml @@ -0,0 +1,58 @@ +version: 0.2 +# Runs Only the ESDK-NET v4.0.1 Decryption Vectors, testing Required EC CMM + +env: + variables: + TOXENV: "py310-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + git-credential-helper: yes + secrets-manager: + GITHUB_TOKEN: Github/lucasmcdonald3-fgpat:actions read + +phases: + install: + runtime-versions: + python: 3.10 + pre_build: + commands: + # Fetch test vectors from Dafny ESDK's most recent run + # (Assuming the first result is most recent; seems to be correct...) + - | + MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ + | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') + - | + echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" + - | + MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/$MOST_RECENT_RUN_ID/artifacts?name=ubuntu-latest_vector_artifact" \ + | jq '.artifacts[0].archive_download_url') + - | + echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" + - | + curl -L -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip + # This unzips to `net41.zip`. + - unzip ubuntu-latest_test_vector_artifact + # This unzips to `net41/`. + - unzip net41.zip -d net41 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../net41/manifest.json diff --git a/codebuild/py310/decrypt_masterkey_with_js.yml b/codebuild/py310/decrypt_masterkey_with_js.yml new file mode 100644 index 000000000..fdfb2363c --- /dev/null +++ b/codebuild/py310/decrypt_masterkey_with_js.yml @@ -0,0 +1,34 @@ +version: 0.2 + +env: + variables: + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.10 + commands: + - n 16 + # Install the Javascript ESDK run test vectors + - npm install -g @aws-crypto/integration-node + + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/310_masterkey.zip 310_masterkey.zip + # Repackage zip in expected format + - unzip 310_masterkey.zip + - cd 310_masterkey + - zip -r vectors.zip . + build: + commands: + # Decrypt generated vectors with Javascript ESDK + - integration-node decrypt -v vectors.zip \ No newline at end of file diff --git a/codebuild/py310/decrypt_masterkey_with_masterkey.yml b/codebuild/py310/decrypt_masterkey_with_masterkey.yml new file mode 100644 index 000000000..577e81b9a --- /dev/null +++ b/codebuild/py310/decrypt_masterkey_with_masterkey.yml @@ -0,0 +1,30 @@ +version: 0.2 + +env: + variables: + TOXENV: "py310-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.10 + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/310_masterkey.zip 310_masterkey.zip + - unzip 310_masterkey.zip + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../310_masterkey/manifest.json \ No newline at end of file diff --git a/codebuild/py310/decrypt_net_401_vectors.yml b/codebuild/py310/decrypt_net_401_vectors.yml new file mode 100644 index 000000000..82ac642d9 --- /dev/null +++ b/codebuild/py310/decrypt_net_401_vectors.yml @@ -0,0 +1,35 @@ +version: 0.2 +# Runs Only the ESDK-NET v4.0.1 Decryption Vectors, testing Required EC CMM + +env: + variables: + TOXENV: "py310-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.10 + pre_build: + commands: + # Fetch ESDK .NET v4.0.1 Test Vectors + - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip + - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip + - curl -s --output $VECTOR_ZIP --location $VECTORS_URL + - UNZIPPED_VECTORS_DIR=$CODEBUILD_SRC_DIR/test_vector_handlers/net_401_vectors + - unzip $VECTOR_ZIP -d $UNZIPPED_VECTORS_DIR + build: + commands: + # NOTE: We need to pass the absolute path of the vectors + - pip install "tox < 4.0" + - cd $CODEBUILD_SRC_DIR/test_vector_handlers + - | + tox -- \ + --input $UNZIPPED_VECTORS_DIR/manifest.json diff --git a/codebuild/py310/encrypt_masterkey.yml b/codebuild/py310/encrypt_masterkey.yml new file mode 100644 index 000000000..9cd89fb8f --- /dev/null +++ b/codebuild/py310/encrypt_masterkey.yml @@ -0,0 +1,25 @@ +version: 0.2 + +env: + variables: + TOXENV: "py310-full_encrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.10 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0003-awses-message-encryption.v2.json diff --git a/codebuild/py310/examples copy.yml b/codebuild/py310/examples copy.yml new file mode 100644 index 000000000..b495a327c --- /dev/null +++ b/codebuild/py310/examples copy.yml @@ -0,0 +1,22 @@ +version: 0.2 + +env: + variables: + TOXENV: "py310-examples" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.10 + build: + commands: + - pip install "tox < 4.0" + - tox diff --git a/codebuild/py310/generate_decrypt_vectors_masterkey.yml b/codebuild/py310/generate_decrypt_vectors_masterkey.yml new file mode 100644 index 000000000..640fb72d6 --- /dev/null +++ b/codebuild/py310/generate_decrypt_vectors_masterkey.yml @@ -0,0 +1,28 @@ +version: 0.2 + +env: + variables: + TOXENV: "py310-full_decrypt_generate" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.10 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ + --output 310_masterkey + - zip -r 310_masterkey.zip 310_masterkey + - aws s3 cp 310_masterkey.zip s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/310_masterkey.zip diff --git a/codebuild/py310/integ copy.yml b/codebuild/py310/integ copy.yml new file mode 100644 index 000000000..6b557e709 --- /dev/null +++ b/codebuild/py310/integ copy.yml @@ -0,0 +1,22 @@ +version: 0.2 + +env: + variables: + TOXENV: "py310-integ" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.10 + build: + commands: + - pip install "tox < 4.0" + - tox diff --git a/codebuild/py312/awses_local.yml b/codebuild/py312/awses_local.yml new file mode 100644 index 000000000..844cc7993 --- /dev/null +++ b/codebuild/py312/awses_local.yml @@ -0,0 +1,25 @@ +version: 0.2 + +env: + variables: + TOXENV: "py312-awses_local" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + python: 3.12 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - tox diff --git a/codebuild/py312/awses_local_mpl.yml b/codebuild/py312/awses_local_mpl.yml new file mode 100644 index 000000000..11f995c16 --- /dev/null +++ b/codebuild/py312/awses_local_mpl.yml @@ -0,0 +1,26 @@ +version: 0.2 + +env: + variables: + TOXENV: "py312-awses_local-mpl" + REGION: "us-west-2" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + python: 3.12 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - tox diff --git a/codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml b/codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml new file mode 100644 index 000000000..810d16b74 --- /dev/null +++ b/codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml @@ -0,0 +1,59 @@ +version: 0.2 +# Runs Only the ESDK-NET v4.0.1 Decryption Vectors, testing Required EC CMM + +env: + variables: + TOXENV: "py312-full_decrypt-mpl" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + git-credential-helper: yes + secrets-manager: + GITHUB_TOKEN: Github/lucasmcdonald3-fgpat:actions read + +phases: + install: + runtime-versions: + python: 3.12 + pre_build: + commands: + # Fetch test vectors from Dafny ESDK's most recent run + # (Assuming the first result is most recent; seems to be correct...) + - | + MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ + | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') + - | + echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" + - | + MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/$MOST_RECENT_RUN_ID/artifacts?name=ubuntu-latest_vector_artifact" \ + | jq '.artifacts[0].archive_download_url') + - | + echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" + - | + curl -L -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip + # This unzips to `net41.zip`. + - unzip ubuntu-latest_test_vector_artifact + # This unzips to `net41/`. + - unzip net41.zip -d net41 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../net41/manifest.json \ + --keyrings diff --git a/codebuild/py312/decrypt_dafny_esdk_vectors_masterkey.yml b/codebuild/py312/decrypt_dafny_esdk_vectors_masterkey.yml new file mode 100644 index 000000000..b375651c5 --- /dev/null +++ b/codebuild/py312/decrypt_dafny_esdk_vectors_masterkey.yml @@ -0,0 +1,58 @@ +version: 0.2 +# Runs Only the ESDK-NET v4.0.1 Decryption Vectors, testing Required EC CMM + +env: + variables: + TOXENV: "py312-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + git-credential-helper: yes + secrets-manager: + GITHUB_TOKEN: Github/lucasmcdonald3-fgpat:actions read + +phases: + install: + runtime-versions: + python: 3.12 + pre_build: + commands: + # Fetch test vectors from Dafny ESDK's most recent run + # (Assuming the first result is most recent; seems to be correct...) + - | + MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ + | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') + - | + echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" + - | + MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/$MOST_RECENT_RUN_ID/artifacts?name=ubuntu-latest_vector_artifact" \ + | jq '.artifacts[0].archive_download_url') + - | + echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" + - | + curl -L -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip + # This unzips to `net41.zip`. + - unzip ubuntu-latest_test_vector_artifact + # This unzips to `net41/`. + - unzip net41.zip -d net41 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../net41/manifest.json diff --git a/codebuild/py312/decrypt_keyrings_with_js.yml b/codebuild/py312/decrypt_keyrings_with_js.yml new file mode 100644 index 000000000..9b1ebc270 --- /dev/null +++ b/codebuild/py312/decrypt_keyrings_with_js.yml @@ -0,0 +1,34 @@ +version: 0.2 + +env: + variables: + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.12 + commands: + - n 16 + # Install the Javascript ESDK run test vectors + - npm install -g @aws-crypto/integration-node + + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_keyrings.zip 312_keyrings.zip + # Repackage zip in expected format + - unzip 312_keyrings.zip + - cd 312_keyrings + - zip -r vectors.zip . + build: + commands: + # Decrypt generated vectors with Javascript ESDK + - integration-node decrypt -v vectors.zip \ No newline at end of file diff --git a/codebuild/py38/decrypt_dafny_esdk_vectors.yml b/codebuild/py38/decrypt_dafny_esdk_vectors.yml new file mode 100644 index 000000000..968a74690 --- /dev/null +++ b/codebuild/py38/decrypt_dafny_esdk_vectors.yml @@ -0,0 +1,58 @@ +version: 0.2 +# Runs Only the ESDK-NET v4.0.1 Decryption Vectors, testing Required EC CMM + +env: + variables: + TOXENV: "py38-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + git-credential-helper: yes + secrets-manager: + GITHUB_TOKEN: Github/lucasmcdonald3-fgpat:actions read + +phases: + install: + runtime-versions: + python: 3.8 + pre_build: + commands: + # Fetch test vectors from Dafny ESDK's most recent run + # (Assuming the first result is most recent; seems to be correct...) + - | + MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ + | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') + - | + echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" + - | + MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/$MOST_RECENT_RUN_ID/artifacts?name=ubuntu-latest_vector_artifact" \ + | jq '.artifacts[0].archive_download_url') + - | + echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" + - | + curl -L -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip + # This unzips to `net41.zip`. + - unzip ubuntu-latest_test_vector_artifact + # This unzips to `net41/`. + - unzip net41.zip -d net41 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../net41/manifest.json diff --git a/codebuild/py38/decrypt_masterkey_with_js.yml b/codebuild/py38/decrypt_masterkey_with_js.yml new file mode 100644 index 000000000..953e8818a --- /dev/null +++ b/codebuild/py38/decrypt_masterkey_with_js.yml @@ -0,0 +1,34 @@ +version: 0.2 + +env: + variables: + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.8 + commands: + - n 16 + # Install the Javascript ESDK run test vectors + - npm install -g @aws-crypto/integration-node + + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/38_masterkey.zip 38_masterkey.zip + # Repackage zip in expected format + - unzip 38_masterkey.zip + - cd 38_masterkey + - zip -r vectors.zip . + build: + commands: + # Decrypt generated vectors with Javascript ESDK + - integration-node decrypt -v vectors.zip \ No newline at end of file diff --git a/codebuild/py38/decrypt_masterkey_with_masterkey.yml b/codebuild/py38/decrypt_masterkey_with_masterkey.yml new file mode 100644 index 000000000..6b32dcf15 --- /dev/null +++ b/codebuild/py38/decrypt_masterkey_with_masterkey.yml @@ -0,0 +1,30 @@ +version: 0.2 + +env: + variables: + TOXENV: "py38-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.8 + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/38_masterkey.zip 38_masterkey.zip + - unzip 38_masterkey.zip + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../38_masterkey/manifest.json \ No newline at end of file diff --git a/codebuild/py38/decrypt_net_401_vectors.yml b/codebuild/py38/decrypt_net_401_vectors.yml new file mode 100644 index 000000000..298711975 --- /dev/null +++ b/codebuild/py38/decrypt_net_401_vectors.yml @@ -0,0 +1,35 @@ +version: 0.2 +# Runs Only the ESDK-NET v4.0.1 Decryption Vectors, testing Required EC CMM + +env: + variables: + TOXENV: "py38-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.8 + pre_build: + commands: + # Fetch ESDK .NET v4.0.1 Test Vectors + - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip + - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip + - curl -s --output $VECTOR_ZIP --location $VECTORS_URL + - UNZIPPED_VECTORS_DIR=$CODEBUILD_SRC_DIR/test_vector_handlers/net_401_vectors + - unzip $VECTOR_ZIP -d $UNZIPPED_VECTORS_DIR + build: + commands: + # NOTE: We need to pass the absolute path of the vectors + - pip install "tox < 4.0" + - cd $CODEBUILD_SRC_DIR/test_vector_handlers + - | + tox -- \ + --input $UNZIPPED_VECTORS_DIR/manifest.json diff --git a/codebuild/py38/encrypt_masterkey.yml b/codebuild/py38/encrypt_masterkey.yml new file mode 100644 index 000000000..b05396cc2 --- /dev/null +++ b/codebuild/py38/encrypt_masterkey.yml @@ -0,0 +1,25 @@ +version: 0.2 + +env: + variables: + TOXENV: "py38-full_encrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.8 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0003-awses-message-encryption.v2.json diff --git a/codebuild/py38/generate_decrypt_vectors_masterkey.yml b/codebuild/py38/generate_decrypt_vectors_masterkey.yml new file mode 100644 index 000000000..8705ef57c --- /dev/null +++ b/codebuild/py38/generate_decrypt_vectors_masterkey.yml @@ -0,0 +1,28 @@ +version: 0.2 + +env: + variables: + TOXENV: "py38-full_decrypt_generate" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.8 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ + --output 38_masterkey + - zip -r 38_masterkey.zip 38_masterkey + - aws s3 cp 38_masterkey.zip s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/38_masterkey.zip diff --git a/codebuild/py39/awses_local.yml b/codebuild/py39/awses_local.yml new file mode 100644 index 000000000..e56a9ff45 --- /dev/null +++ b/codebuild/py39/awses_local.yml @@ -0,0 +1,25 @@ +version: 0.2 + +env: + variables: + TOXENV: "py39-awses_local" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + python: 3.9 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - tox diff --git a/codebuild/py39/decrypt_dafny_esdk_vectors.yml b/codebuild/py39/decrypt_dafny_esdk_vectors.yml new file mode 100644 index 000000000..ddb50db1c --- /dev/null +++ b/codebuild/py39/decrypt_dafny_esdk_vectors.yml @@ -0,0 +1,58 @@ +version: 0.2 +# Runs Only the ESDK-NET v4.0.1 Decryption Vectors, testing Required EC CMM + +env: + variables: + TOXENV: "py39-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + git-credential-helper: yes + secrets-manager: + GITHUB_TOKEN: Github/lucasmcdonald3-fgpat:actions read + +phases: + install: + runtime-versions: + python: 3.9 + pre_build: + commands: + # Fetch test vectors from Dafny ESDK's most recent run + # (Assuming the first result is most recent; seems to be correct...) + - | + MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ + | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') + - | + echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" + - | + MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/$MOST_RECENT_RUN_ID/artifacts?name=ubuntu-latest_vector_artifact" \ + | jq '.artifacts[0].archive_download_url') + - | + echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" + - | + curl -L -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip + # This unzips to `net41.zip`. + - unzip ubuntu-latest_test_vector_artifact + # This unzips to `net41/`. + - unzip net41.zip -d net41 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../net41/manifest.json diff --git a/codebuild/py39/decrypt_masterkey_with_js.yml b/codebuild/py39/decrypt_masterkey_with_js.yml new file mode 100644 index 000000000..53f6433f8 --- /dev/null +++ b/codebuild/py39/decrypt_masterkey_with_js.yml @@ -0,0 +1,34 @@ +version: 0.2 + +env: + variables: + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.9 + commands: + - n 16 + # Install the Javascript ESDK run test vectors + - npm install -g @aws-crypto/integration-node + + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/39_masterkey.zip 39_masterkey.zip + # Repackage zip in expected format + - unzip 39_masterkey.zip + - cd 39_masterkey + - zip -r vectors.zip . + build: + commands: + # Decrypt generated vectors with Javascript ESDK + - integration-node decrypt -v vectors.zip \ No newline at end of file diff --git a/codebuild/py39/decrypt_masterkey_with_masterkey.yml b/codebuild/py39/decrypt_masterkey_with_masterkey.yml new file mode 100644 index 000000000..fcd9d3220 --- /dev/null +++ b/codebuild/py39/decrypt_masterkey_with_masterkey.yml @@ -0,0 +1,30 @@ +version: 0.2 + +env: + variables: + TOXENV: "py39-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.9 + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/39_masterkey.zip 39_masterkey.zip + - unzip 39_masterkey.zip + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../39_masterkey/manifest.json \ No newline at end of file diff --git a/codebuild/py39/decrypt_net_401_vectors.yml b/codebuild/py39/decrypt_net_401_vectors.yml new file mode 100644 index 000000000..635abc95b --- /dev/null +++ b/codebuild/py39/decrypt_net_401_vectors.yml @@ -0,0 +1,35 @@ +version: 0.2 +# Runs Only the ESDK-NET v4.0.1 Decryption Vectors, testing Required EC CMM + +env: + variables: + TOXENV: "py39-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.9 + pre_build: + commands: + # Fetch ESDK .NET v4.0.1 Test Vectors + - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip + - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip + - curl -s --output $VECTOR_ZIP --location $VECTORS_URL + - UNZIPPED_VECTORS_DIR=$CODEBUILD_SRC_DIR/test_vector_handlers/net_401_vectors + - unzip $VECTOR_ZIP -d $UNZIPPED_VECTORS_DIR + build: + commands: + # NOTE: We need to pass the absolute path of the vectors + - pip install "tox < 4.0" + - cd $CODEBUILD_SRC_DIR/test_vector_handlers + - | + tox -- \ + --input $UNZIPPED_VECTORS_DIR/manifest.json diff --git a/codebuild/py39/encrypt_masterkey.yml b/codebuild/py39/encrypt_masterkey.yml new file mode 100644 index 000000000..3bf18fbde --- /dev/null +++ b/codebuild/py39/encrypt_masterkey.yml @@ -0,0 +1,25 @@ +version: 0.2 + +env: + variables: + TOXENV: "py39-full_encrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.9 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0003-awses-message-encryption.v2.json diff --git a/codebuild/py39/examples copy.yml b/codebuild/py39/examples copy.yml new file mode 100644 index 000000000..3d1399251 --- /dev/null +++ b/codebuild/py39/examples copy.yml @@ -0,0 +1,22 @@ +version: 0.2 + +env: + variables: + TOXENV: "py39-examples" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.9 + build: + commands: + - pip install "tox < 4.0" + - tox diff --git a/codebuild/py39/generate_decrypt_vectors_masterkey.yml b/codebuild/py39/generate_decrypt_vectors_masterkey.yml new file mode 100644 index 000000000..eb57d915a --- /dev/null +++ b/codebuild/py39/generate_decrypt_vectors_masterkey.yml @@ -0,0 +1,28 @@ +version: 0.2 + +env: + variables: + TOXENV: "py39-full_decrypt_generate" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.9 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ + --output 39_masterkey + - zip -r 39_masterkey.zip 39_masterkey + - aws s3 cp 39_masterkey.zip s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/39_masterkey.zip diff --git a/codebuild/py39/integ copy.yml b/codebuild/py39/integ copy.yml new file mode 100644 index 000000000..6dec85b07 --- /dev/null +++ b/codebuild/py39/integ copy.yml @@ -0,0 +1,22 @@ +version: 0.2 + +env: + variables: + TOXENV: "py39-integ" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.9 + build: + commands: + - pip install "tox < 4.0" + - tox From bdacdeb455ce1fbe5d44e550054bb9689a8171ad Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 12:06:23 -0700 Subject: [PATCH 355/376] ALL --- buildspec.yml | 94 ++++++++++++++++++++++++++++--- codebuild/py310/examples copy.yml | 22 -------- codebuild/py310/integ copy.yml | 22 -------- codebuild/py39/examples copy.yml | 22 -------- codebuild/py39/integ copy.yml | 22 -------- 5 files changed, 85 insertions(+), 97 deletions(-) delete mode 100644 codebuild/py310/examples copy.yml delete mode 100644 codebuild/py310/integ copy.yml delete mode 100644 codebuild/py39/examples copy.yml delete mode 100644 codebuild/py39/integ copy.yml diff --git a/buildspec.yml b/buildspec.yml index e303d9b35..873e5941e 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -4,6 +4,44 @@ batch: fast-fail: false build-graph: + # 3.7 + - identifier: py37_integ + buildspec: codebuild/py37/integ.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py37_examples + buildspec: codebuild/py37/examples.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py37_decrypt_dafny_esdk_vectors + buildspec: codebuild/py37/decrypt_dafny_esdk_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py37_decrypt_net_401_vectors + buildspec: codebuild/py37/decrypt_net_401_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py37_encrypt_masterkey + buildspec: codebuild/py37/encrypt_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py37_generate_decrypt_vectors_masterkey + buildspec: codebuild/py37/generate_decrypt_vectors_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py37_decrypt_masterkey_with_masterkey + depend-on: + - py37_generate_decrypt_vectors_masterkey + buildspec: codebuild/py37/decrypt_masterkey_with_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py37_decrypt_masterkey_with_js + depend-on: + - py37_generate_decrypt_vectors_masterkey + buildspec: codebuild/py37/decrypt_masterkey_with_js.yml + env: + image: aws/codebuild/standard:5.0 + # 3.8 - identifier: py38_integ buildspec: codebuild/py38/integ.yml @@ -80,6 +118,44 @@ batch: env: image: aws/codebuild/standard:5.0 + # 3.10 + - identifier: py310_integ + buildspec: codebuild/py310/integ.yml + env: + image: aws/codebuild/standard:6.0 + - identifier: py310_examples + buildspec: codebuild/py310/examples.yml + env: + image: aws/codebuild/standard:6.0 + - identifier: py310_decrypt_dafny_esdk_vectors + buildspec: codebuild/py310/decrypt_dafny_esdk_vectors.yml + env: + image: aws/codebuild/standard:6.0 + - identifier: py310_decrypt_net_401_vectors + buildspec: codebuild/py310/decrypt_net_401_vectors.yml + env: + image: aws/codebuild/standard:6.0 + - identifier: py310_encrypt_masterkey + buildspec: codebuild/py310/encrypt_masterkey.yml + env: + image: aws/codebuild/standard:6.0 + - identifier: py310_generate_decrypt_vectors_masterkey + buildspec: codebuild/py310/generate_decrypt_vectors_masterkey.yml + env: + image: aws/codebuild/standard:6.0 + - identifier: py310_decrypt_masterkey_with_masterkey + depend-on: + - py310_generate_decrypt_vectors_masterkey + buildspec: codebuild/py310/decrypt_masterkey_with_masterkey.yml + env: + image: aws/codebuild/standard:6.0 + - identifier: py310_decrypt_masterkey_with_js + depend-on: + - py310_generate_decrypt_vectors_masterkey + buildspec: codebuild/py310/decrypt_masterkey_with_js.yml + env: + image: aws/codebuild/standard:6.0 + - identifier: py311_integ buildspec: codebuild/py311/integ.yml env: @@ -251,13 +327,13 @@ batch: env: image: aws/codebuild/standard:7.0 - # - # - identifier: code_coverage - # buildspec: codebuild/coverage/coverage.yml - # - identifier: code_coverage_mpl - # buildspec: codebuild/coverage/coverage_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 + + - identifier: code_coverage + buildspec: codebuild/coverage/coverage.yml + - identifier: code_coverage_mpl + buildspec: codebuild/coverage/coverage_mpl.yml + env: + image: aws/codebuild/standard:7.0 - # - identifier: compliance - # buildspec: codebuild/compliance/compliance.yml + - identifier: compliance + buildspec: codebuild/compliance/compliance.yml diff --git a/codebuild/py310/examples copy.yml b/codebuild/py310/examples copy.yml deleted file mode 100644 index b495a327c..000000000 --- a/codebuild/py310/examples copy.yml +++ /dev/null @@ -1,22 +0,0 @@ -version: 0.2 - -env: - variables: - TOXENV: "py310-examples" - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- - arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- - arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- - arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- - arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - -phases: - install: - runtime-versions: - python: 3.10 - build: - commands: - - pip install "tox < 4.0" - - tox diff --git a/codebuild/py310/integ copy.yml b/codebuild/py310/integ copy.yml deleted file mode 100644 index 6b557e709..000000000 --- a/codebuild/py310/integ copy.yml +++ /dev/null @@ -1,22 +0,0 @@ -version: 0.2 - -env: - variables: - TOXENV: "py310-integ" - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- - arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- - arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- - arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- - arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - -phases: - install: - runtime-versions: - python: 3.10 - build: - commands: - - pip install "tox < 4.0" - - tox diff --git a/codebuild/py39/examples copy.yml b/codebuild/py39/examples copy.yml deleted file mode 100644 index 3d1399251..000000000 --- a/codebuild/py39/examples copy.yml +++ /dev/null @@ -1,22 +0,0 @@ -version: 0.2 - -env: - variables: - TOXENV: "py39-examples" - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- - arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- - arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- - arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- - arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - -phases: - install: - runtime-versions: - python: 3.9 - build: - commands: - - pip install "tox < 4.0" - - tox diff --git a/codebuild/py39/integ copy.yml b/codebuild/py39/integ copy.yml deleted file mode 100644 index 6dec85b07..000000000 --- a/codebuild/py39/integ copy.yml +++ /dev/null @@ -1,22 +0,0 @@ -version: 0.2 - -env: - variables: - TOXENV: "py39-integ" - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- - arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- - arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- - arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- - arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - -phases: - install: - runtime-versions: - python: 3.9 - build: - commands: - - pip install "tox < 4.0" - - tox From 27838089d0ebe32b78366dd74f55b650e76c02f2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 12:18:47 -0700 Subject: [PATCH 356/376] gen decrypt keyrings --- codebuild/py311/generate_decrypt_vectors_keyrings.yml | 3 ++- codebuild/py312/generate_decrypt_vectors_keyrings.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml index 777a5703f..179ec0f12 100644 --- a/codebuild/py311/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -23,6 +23,7 @@ phases: - | tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ - --output 311_keyrings + --output 311_keyrings \ + --keyrings - zip -r 311_keyrings.zip 311_keyrings - aws s3 cp 311_keyrings.zip s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyrings.zip diff --git a/codebuild/py312/generate_decrypt_vectors_keyrings.yml b/codebuild/py312/generate_decrypt_vectors_keyrings.yml index 51a1415ee..ae79b86ce 100644 --- a/codebuild/py312/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py312/generate_decrypt_vectors_keyrings.yml @@ -23,6 +23,7 @@ phases: - | tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ - --output 312_keyrings + --output 312_keyrings \ + --keyrings - zip -r 312_keyrings.zip 312_keyrings - aws s3 cp 312_keyrings.zip s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_keyrings.zip From f690cf7dfca9a18301711aa9ab6e40ac727f07e9 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 15:11:30 -0700 Subject: [PATCH 357/376] cleanup --- buildspec.yml | 574 +++++++++--------- .../materials_managers/mpl/cmm.py | 16 - .../materials_managers/mpl/materials.py | 2 - src/aws_encryption_sdk/streaming_client.py | 1 - .../internal/tampering_mpl_materials.py | 9 +- .../manifests/full_message/decrypt.py | 21 +- .../full_message/decrypt_generation.py | 10 +- .../manifests/full_message/encrypt.py | 139 ----- .../commands/test_i_full_message_encrypt.py} | 24 +- .../integration/integration_test_utils.py | 35 +- test_vector_handlers/tox.ini | 5 + 11 files changed, 322 insertions(+), 514 deletions(-) rename test_vector_handlers/test/{mpl/integration/commands/test_i_encrypt_keyrings.py => integration/commands/test_i_full_message_encrypt.py} (62%) diff --git a/buildspec.yml b/buildspec.yml index 873e5941e..90b5dbfd2 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -42,298 +42,298 @@ batch: env: image: aws/codebuild/standard:5.0 - # 3.8 - - identifier: py38_integ - buildspec: codebuild/py38/integ.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py38_examples - buildspec: codebuild/py38/examples.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py38_decrypt_dafny_esdk_vectors - buildspec: codebuild/py38/decrypt_dafny_esdk_vectors.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py38_decrypt_net_401_vectors - buildspec: codebuild/py38/decrypt_net_401_vectors.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py38_encrypt_masterkey - buildspec: codebuild/py38/encrypt_masterkey.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py38_generate_decrypt_vectors_masterkey - buildspec: codebuild/py38/generate_decrypt_vectors_masterkey.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py38_decrypt_masterkey_with_masterkey - depend-on: - - py38_generate_decrypt_vectors_masterkey - buildspec: codebuild/py38/decrypt_masterkey_with_masterkey.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py38_decrypt_masterkey_with_js - depend-on: - - py38_generate_decrypt_vectors_masterkey - buildspec: codebuild/py38/decrypt_masterkey_with_js.yml - env: - image: aws/codebuild/standard:5.0 + # # 3.8 + # - identifier: py38_integ + # buildspec: codebuild/py38/integ.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py38_examples + # buildspec: codebuild/py38/examples.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py38_decrypt_dafny_esdk_vectors + # buildspec: codebuild/py38/decrypt_dafny_esdk_vectors.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py38_decrypt_net_401_vectors + # buildspec: codebuild/py38/decrypt_net_401_vectors.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py38_encrypt_masterkey + # buildspec: codebuild/py38/encrypt_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py38_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py38/generate_decrypt_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py38_decrypt_masterkey_with_masterkey + # depend-on: + # - py38_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py38/decrypt_masterkey_with_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py38_decrypt_masterkey_with_js + # depend-on: + # - py38_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py38/decrypt_masterkey_with_js.yml + # env: + # image: aws/codebuild/standard:5.0 - # 3.9 - - identifier: py39_integ - buildspec: codebuild/py39/integ.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py39_examples - buildspec: codebuild/py39/examples.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py39_decrypt_dafny_esdk_vectors - buildspec: codebuild/py39/decrypt_dafny_esdk_vectors.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py39_decrypt_net_401_vectors - buildspec: codebuild/py39/decrypt_net_401_vectors.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py39_encrypt_masterkey - buildspec: codebuild/py39/encrypt_masterkey.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py39_generate_decrypt_vectors_masterkey - buildspec: codebuild/py39/generate_decrypt_vectors_masterkey.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py39_decrypt_masterkey_with_masterkey - depend-on: - - py39_generate_decrypt_vectors_masterkey - buildspec: codebuild/py39/decrypt_masterkey_with_masterkey.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py39_decrypt_masterkey_with_js - depend-on: - - py39_generate_decrypt_vectors_masterkey - buildspec: codebuild/py39/decrypt_masterkey_with_js.yml - env: - image: aws/codebuild/standard:5.0 + # # 3.9 + # - identifier: py39_integ + # buildspec: codebuild/py39/integ.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py39_examples + # buildspec: codebuild/py39/examples.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py39_decrypt_dafny_esdk_vectors + # buildspec: codebuild/py39/decrypt_dafny_esdk_vectors.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py39_decrypt_net_401_vectors + # buildspec: codebuild/py39/decrypt_net_401_vectors.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py39_encrypt_masterkey + # buildspec: codebuild/py39/encrypt_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py39_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py39/generate_decrypt_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py39_decrypt_masterkey_with_masterkey + # depend-on: + # - py39_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py39/decrypt_masterkey_with_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py39_decrypt_masterkey_with_js + # depend-on: + # - py39_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py39/decrypt_masterkey_with_js.yml + # env: + # image: aws/codebuild/standard:5.0 - # 3.10 - - identifier: py310_integ - buildspec: codebuild/py310/integ.yml - env: - image: aws/codebuild/standard:6.0 - - identifier: py310_examples - buildspec: codebuild/py310/examples.yml - env: - image: aws/codebuild/standard:6.0 - - identifier: py310_decrypt_dafny_esdk_vectors - buildspec: codebuild/py310/decrypt_dafny_esdk_vectors.yml - env: - image: aws/codebuild/standard:6.0 - - identifier: py310_decrypt_net_401_vectors - buildspec: codebuild/py310/decrypt_net_401_vectors.yml - env: - image: aws/codebuild/standard:6.0 - - identifier: py310_encrypt_masterkey - buildspec: codebuild/py310/encrypt_masterkey.yml - env: - image: aws/codebuild/standard:6.0 - - identifier: py310_generate_decrypt_vectors_masterkey - buildspec: codebuild/py310/generate_decrypt_vectors_masterkey.yml - env: - image: aws/codebuild/standard:6.0 - - identifier: py310_decrypt_masterkey_with_masterkey - depend-on: - - py310_generate_decrypt_vectors_masterkey - buildspec: codebuild/py310/decrypt_masterkey_with_masterkey.yml - env: - image: aws/codebuild/standard:6.0 - - identifier: py310_decrypt_masterkey_with_js - depend-on: - - py310_generate_decrypt_vectors_masterkey - buildspec: codebuild/py310/decrypt_masterkey_with_js.yml - env: - image: aws/codebuild/standard:6.0 + # # 3.10 + # - identifier: py310_integ + # buildspec: codebuild/py310/integ.yml + # env: + # image: aws/codebuild/standard:6.0 + # - identifier: py310_examples + # buildspec: codebuild/py310/examples.yml + # env: + # image: aws/codebuild/standard:6.0 + # - identifier: py310_decrypt_dafny_esdk_vectors + # buildspec: codebuild/py310/decrypt_dafny_esdk_vectors.yml + # env: + # image: aws/codebuild/standard:6.0 + # - identifier: py310_decrypt_net_401_vectors + # buildspec: codebuild/py310/decrypt_net_401_vectors.yml + # env: + # image: aws/codebuild/standard:6.0 + # - identifier: py310_encrypt_masterkey + # buildspec: codebuild/py310/encrypt_masterkey.yml + # env: + # image: aws/codebuild/standard:6.0 + # - identifier: py310_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py310/generate_decrypt_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:6.0 + # - identifier: py310_decrypt_masterkey_with_masterkey + # depend-on: + # - py310_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py310/decrypt_masterkey_with_masterkey.yml + # env: + # image: aws/codebuild/standard:6.0 + # - identifier: py310_decrypt_masterkey_with_js + # depend-on: + # - py310_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py310/decrypt_masterkey_with_js.yml + # env: + # image: aws/codebuild/standard:6.0 - - identifier: py311_integ - buildspec: codebuild/py311/integ.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_integ_mpl - buildspec: codebuild/py311/integ_mpl.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_examples - buildspec: codebuild/py311/examples.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_examples_mpl - buildspec: codebuild/py311/examples_mpl.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_dafny_esdk_vectors_masterkey - buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_dafny_esdk_vectors_keyrings - buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_net_401_vectors_masterkey - buildspec: codebuild/py311/decrypt_net_401_vectors_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_net_401_vectors_keyrings - buildspec: codebuild/py311/decrypt_net_401_vectors_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_encrypt_masterkey - buildspec: codebuild/py311/encrypt_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_encrypt_keyrings - buildspec: codebuild/py311/encrypt_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_generate_decrypt_vectors_masterkey - buildspec: codebuild/py311/generate_decrypt_vectors_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_masterkey_with_masterkey - depend-on: - - py311_generate_decrypt_vectors_masterkey - buildspec: codebuild/py311/decrypt_masterkey_with_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_masterkey_with_keyrings - depend-on: - - py311_generate_decrypt_vectors_masterkey - buildspec: codebuild/py311/decrypt_masterkey_with_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_masterkey_with_js - depend-on: - - py311_generate_decrypt_vectors_masterkey - buildspec: codebuild/py311/decrypt_masterkey_with_js.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_generate_decrypt_vectors_keyrings - buildspec: codebuild/py311/generate_decrypt_vectors_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_keyrings_with_masterkey - depend-on: - - py311_generate_decrypt_vectors_keyrings - buildspec: codebuild/py311/decrypt_keyrings_with_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_keyrings_with_keyrings - depend-on: - - py311_generate_decrypt_vectors_keyrings - buildspec: codebuild/py311/decrypt_keyrings_with_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_keyrings_with_js - depend-on: - - py311_generate_decrypt_vectors_keyrings - buildspec: codebuild/py311/decrypt_keyrings_with_js.yml - env: - image: aws/codebuild/standard:7.0 + # - identifier: py311_integ + # buildspec: codebuild/py311/integ.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_integ_mpl + # buildspec: codebuild/py311/integ_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_examples + # buildspec: codebuild/py311/examples.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_examples_mpl + # buildspec: codebuild/py311/examples_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_dafny_esdk_vectors_masterkey + # buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_dafny_esdk_vectors_keyrings + # buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_net_401_vectors_masterkey + # buildspec: codebuild/py311/decrypt_net_401_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_net_401_vectors_keyrings + # buildspec: codebuild/py311/decrypt_net_401_vectors_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_encrypt_masterkey + # buildspec: codebuild/py311/encrypt_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_encrypt_keyrings + # buildspec: codebuild/py311/encrypt_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py311/generate_decrypt_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_masterkey_with_masterkey + # depend-on: + # - py311_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py311/decrypt_masterkey_with_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_masterkey_with_keyrings + # depend-on: + # - py311_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py311/decrypt_masterkey_with_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_masterkey_with_js + # depend-on: + # - py311_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py311/decrypt_masterkey_with_js.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py311/generate_decrypt_vectors_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_keyrings_with_masterkey + # depend-on: + # - py311_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py311/decrypt_keyrings_with_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_keyrings_with_keyrings + # depend-on: + # - py311_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py311/decrypt_keyrings_with_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_keyrings_with_js + # depend-on: + # - py311_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py311/decrypt_keyrings_with_js.yml + # env: + # image: aws/codebuild/standard:7.0 - - identifier: py312_integ - buildspec: codebuild/py312/integ.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_integ_mpl - buildspec: codebuild/py312/integ_mpl.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_examples - buildspec: codebuild/py312/examples.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_examples_mpl - buildspec: codebuild/py312/examples_mpl.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_decrypt_dafny_esdk_vectors_masterkey - buildspec: codebuild/py312/decrypt_dafny_esdk_vectors_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_decrypt_dafny_esdk_vectors_keyrings - buildspec: codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_decrypt_net_401_vectors_masterkey - buildspec: codebuild/py312/decrypt_net_401_vectors_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_decrypt_net_401_vectors_keyrings - buildspec: codebuild/py312/decrypt_net_401_vectors_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_encrypt_masterkey - buildspec: codebuild/py312/encrypt_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_encrypt_keyrings - buildspec: codebuild/py312/encrypt_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_generate_decrypt_vectors_masterkey - buildspec: codebuild/py312/generate_decrypt_vectors_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_decrypt_masterkey_with_masterkey - depend-on: - - py312_generate_decrypt_vectors_masterkey - buildspec: codebuild/py312/decrypt_masterkey_with_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_decrypt_masterkey_with_keyrings - depend-on: - - py312_generate_decrypt_vectors_masterkey - buildspec: codebuild/py312/decrypt_masterkey_with_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_decrypt_masterkey_with_js - depend-on: - - py312_generate_decrypt_vectors_masterkey - buildspec: codebuild/py312/decrypt_masterkey_with_js.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_generate_decrypt_vectors_keyrings - buildspec: codebuild/py312/generate_decrypt_vectors_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_decrypt_keyrings_with_masterkey - depend-on: - - py312_generate_decrypt_vectors_keyrings - buildspec: codebuild/py312/decrypt_keyrings_with_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_decrypt_keyrings_with_keyrings - depend-on: - - py312_generate_decrypt_vectors_keyrings - buildspec: codebuild/py312/decrypt_keyrings_with_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_decrypt_keyrings_with_js - depend-on: - - py312_generate_decrypt_vectors_keyrings - buildspec: codebuild/py312/decrypt_keyrings_with_js.yml - env: - image: aws/codebuild/standard:7.0 + # - identifier: py312_integ + # buildspec: codebuild/py312/integ.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_integ_mpl + # buildspec: codebuild/py312/integ_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_examples + # buildspec: codebuild/py312/examples.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_examples_mpl + # buildspec: codebuild/py312/examples_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_decrypt_dafny_esdk_vectors_masterkey + # buildspec: codebuild/py312/decrypt_dafny_esdk_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_decrypt_dafny_esdk_vectors_keyrings + # buildspec: codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_decrypt_net_401_vectors_masterkey + # buildspec: codebuild/py312/decrypt_net_401_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_decrypt_net_401_vectors_keyrings + # buildspec: codebuild/py312/decrypt_net_401_vectors_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_encrypt_masterkey + # buildspec: codebuild/py312/encrypt_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_encrypt_keyrings + # buildspec: codebuild/py312/encrypt_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py312/generate_decrypt_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_decrypt_masterkey_with_masterkey + # depend-on: + # - py312_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py312/decrypt_masterkey_with_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_decrypt_masterkey_with_keyrings + # depend-on: + # - py312_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py312/decrypt_masterkey_with_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_decrypt_masterkey_with_js + # depend-on: + # - py312_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py312/decrypt_masterkey_with_js.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py312/generate_decrypt_vectors_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_decrypt_keyrings_with_masterkey + # depend-on: + # - py312_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py312/decrypt_keyrings_with_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_decrypt_keyrings_with_keyrings + # depend-on: + # - py312_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py312/decrypt_keyrings_with_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_decrypt_keyrings_with_js + # depend-on: + # - py312_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py312/decrypt_keyrings_with_js.yml + # env: + # image: aws/codebuild/standard:7.0 - - identifier: code_coverage - buildspec: codebuild/coverage/coverage.yml - - identifier: code_coverage_mpl - buildspec: codebuild/coverage/coverage_mpl.yml - env: - image: aws/codebuild/standard:7.0 + # - identifier: code_coverage + # buildspec: codebuild/coverage/coverage.yml + # - identifier: code_coverage_mpl + # buildspec: codebuild/coverage/coverage_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 - - identifier: compliance - buildspec: codebuild/compliance/compliance.yml + # - identifier: compliance + # buildspec: codebuild/compliance/compliance.yml diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index c398904a9..71e9adf8b 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -69,22 +69,6 @@ def get_encryption_materials( ) mpl_output: MPL_GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) - - # ???????????????????????????? - # kpis = set() - # for edk in mpl_output.encryption_materials.encrypted_data_keys: - # kpis.add(edk.key_provider_info) - - # print(kpis) - # input - - # if len(kpis) == 1: - # for edk in mpl_output.encryption_materials.encrypted_data_keys: - # if edk.key_provider_info == b"rsa-4096-public": - # edk.key_provider_info = b"rsa-4096-private" - - # mpl_output.encryption_materials.encrypted_data_keys[0].key_provider_info = b"rsa-4096-private" - return EncryptionMaterialsFromMPL(mpl_output.encryption_materials) except AwsCryptographicMaterialProvidersException as mpl_exception: # Wrap MPL error into the ESDK error type diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index b70e48efe..54ea21b39 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -75,8 +75,6 @@ def encrypted_data_keys(self) -> List[Native_EncryptedDataKey]: ), encrypted_data_key=mpl_edk.ciphertext, ) for mpl_edk in mpl_edk_list} - # print(f"{key_blob_list=}") - # input() return key_blob_list @property diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 54ce76235..fb0935ff2 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -582,7 +582,6 @@ def _prep_message(self): else: # MPL verification key is PEM bytes, not DER bytes. # If the underlying CMM is from the MPL, load PEM bytes. - print(f"DEBUG: cmm is {self.config.materials_manager}") if (_HAS_MPL and isinstance(self.config.materials_manager, CryptoMaterialsManagerFromMPL)): self.signer = Signer.from_key_bytes( diff --git a/test_vector_handlers/src/awses_test_vectors/internal/tampering_mpl_materials.py b/test_vector_handlers/src/awses_test_vectors/internal/tampering_mpl_materials.py index 1bb6705fc..4f7bc658e 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/tampering_mpl_materials.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/tampering_mpl_materials.py @@ -1,6 +1,5 @@ -"""Allows overriding the algorithm and signing_key for EncryptionMaterialsFromMPL. -This must ONLY be used in testing and NOT in production.. -This is used in message tampering testing. +"""Allows using ESDK-MPL interfaces with the tampering tests. +These must ONLY be used in testing and NOT in production. """ import attr import six @@ -30,7 +29,7 @@ class HalfSigningCryptoMaterialsManagerFromMPL(CryptoMaterialsManagerFromMPL): """ - Custom CMM that modifies the provider info field on EDKs + Custom CMM that uses HalfSigningEncryptionMaterialsFromMPL. This extends CryptoMaterialsManagerFromMPL so ESDK-internal checks follow MPL logic. @@ -81,7 +80,7 @@ def decrypt_materials(self, request): class HalfSigningEncryptionMaterialsFromMPL(EncryptionMaterialsFromMPL): """Allows overriding the algorithm and signing_key for EncryptionMaterialsFromMPL. - This must ONLY be used in testing and NOT in production.. + This must ONLY be used in testing and NOT in production. This is used in testing malicious message modification (HalfSigningTampering). """ diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index 4432502c5..2aaaf1bca 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -313,7 +313,8 @@ def master_key_provider_fn(): cmm_type = scenario["cmm"] elif scenario["cmm"] == "RequiredEncryptionContext": # Skip RequiredEncryptionContext CMM for master keys; - # This is unsupported for master keys + # RequiredEncryptionContext is unsupported for master keys. + # Caller logic should expect `None` to mean "no scenario". if keyrings: cmm_type = scenario["cmm"] else: @@ -384,9 +385,9 @@ def _one_shot_decrypt(self): required_ec_cmm: ICryptographicMaterialsManager = \ mpl.create_required_encryption_context_cmm( CreateRequiredEncryptionContextCMMInput( - # Currently, the test vector manifest requires that - # if using the required encryption context CMM, - # both and only "key1" and "key2" are required. + # Currently, the test vector manifest assumes these + # are the only required encryption context keys for any message. + # If this assumption changes, this logic must be augmented. required_encryption_context_keys=["key1", "key2"], underlying_cmm=underlying_cmm, ) @@ -436,9 +437,9 @@ def _streaming_decrypt(self): required_ec_cmm: ICryptographicMaterialsManager = \ mpl.create_required_encryption_context_cmm( CreateRequiredEncryptionContextCMMInput( - # Currently, the test vector manifest requires that - # if using the required encryption context CMM, - # both and only "key1" and "key2" are required. + # Currently, the test vector manifest assumes these + # are the only required encryption context keys for any message. + # If this assumption changes, this logic must be augmented. required_encryption_context_keys=["key1", "key2"], underlying_cmm=underlying_cmm, ) @@ -490,9 +491,9 @@ def _streaming_decrypt_unsigned(self): required_ec_cmm: ICryptographicMaterialsManager = \ mpl.create_required_encryption_context_cmm( CreateRequiredEncryptionContextCMMInput( - # Currently, the test vector manifest requires that - # if using the required encryption context CMM, - # both and only "key1" and "key2" are required. + # Currently, the test vector manifest assumes these + # are the only required encryption context keys for any message. + # If this assumption changes, this logic must be augmented. required_encryption_context_keys=["key1", "key2"], underlying_cmm=underlying_cmm, ) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index a1fc8fa83..f94facf13 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -52,7 +52,6 @@ _HAS_MPL = True except ImportError as e: - print(f"decrypt_generation ImportError: {e}") _HAS_MPL = False @@ -173,8 +172,6 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, _p """ master_key_provider = generation_scenario.encryption_scenario.master_key_provider_fn() - print(f"DEBUG: mkp gen is {master_key_provider}") - # Use a caching CMM to avoid generating a new data key every time. if isinstance(master_key_provider, MasterKeyProvider): cache = LocalCryptoMaterialsCache(10) @@ -185,6 +182,8 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, _p max_messages_encrypted=100, ) cmm = caching_cmm + # No caching CMM in MPL :( + # Use default CMM elif _HAS_MPL and isinstance(master_key_provider, IKeyring): mpl = AwsCryptographicMaterialProviders(MaterialProvidersConfig()) mpl_cmm = mpl.create_default_cryptographic_materials_manager( @@ -196,8 +195,6 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, _p else: raise TypeError(f"Unrecognized master_key_provider type: {master_key_provider}") - print(f"DEBUG: cmm gen is {cmm}") - return [ self.run_scenario_with_new_provider_info( ciphertext_writer, generation_scenario, cmm, new_provider_info @@ -208,7 +205,6 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, _p def run_scenario_with_new_provider_info( self, ciphertext_writer, generation_scenario, materials_manager, new_provider_info ): - print(f"DEBUG: materials_manager is {materials_manager}") """Run with tampering for a specific new provider info value""" if _HAS_MPL and isinstance(materials_manager, CryptoMaterialsManagerFromMPL): tampering_materials_manager = ProviderInfoChangingCryptoMaterialsManagerFromMPL( @@ -543,7 +539,7 @@ def _generate_plaintexts(plaintexts_specs): :return: Mapping of plaintext name to randomly generated bytes :rtype: dict """ - return {name: b"a" * size for name, size in plaintexts_specs.items()} + return {name: os.urandom(size) for name, size in plaintexts_specs.items()} @classmethod def from_file(cls, input_file, keyrings): diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index c1ffcdaa0..57de8504c 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -168,145 +168,6 @@ def run(self, materials_manager=None): return ciphertext -# @attr.s -# class MessageEncryptionWithMasterKeysTestScenario(MessageEncryptionTestScenario): -# # pylint: disable=too-many-instance-attributes -# """Data class for a single full message decrypt test scenario that uses master keys. - -# :param master_key_specs: Iterable of loaded master key specifications -# :type master_key_specs: iterable of :class:`MasterKeySpec` -# :param Callable master_key_provider_fn: -# """ - -# master_key_specs = attr.ib(validator=iterable_validator(list, MasterKeySpec)) -# master_key_provider_fn = attr.ib(validator=attr.validators.is_callable()) - -# @classmethod -# def from_scenario(cls, scenario, keys, plaintexts): -# # type: (ENCRYPT_SCENARIO_SPEC, KeysManifest, Dict[str, bytes]) -> MessageEncryptionTestScenario -# """Load from a scenario specification. - -# :param dict scenario: Scenario specification JSON -# :param KeysManifest keys: Loaded keys -# :param dict plaintexts: Mapping of plaintext names to plaintext values -# :return: Loaded test scenario -# :rtype: MessageEncryptionTestScenario -# """ -# algorithm = algorithm_suite_from_string_id(scenario["algorithm"]) -# master_key_specs = [MasterKeySpec.from_scenario(spec) for spec in scenario["master-keys"]] - -# def master_key_provider_fn(): -# return master_key_provider_from_master_key_specs(keys, master_key_specs) - -# return cls( -# plaintext_name=scenario["plaintext"], -# plaintext=plaintexts[scenario["plaintext"]], -# algorithm=algorithm, -# frame_size=scenario["frame-size"], -# encryption_context=scenario["encryption-context"], -# master_key=True, -# master_key_specs=master_key_specs, -# master_key_provider_fn=master_key_provider_fn, -# ) - -# def run(self, materials_manager=None): -# """Run this scenario, writing the resulting ciphertext with ``ciphertext_writer`` and returning -# a :class:`MessageDecryptionTestScenario` that describes the matching decrypt scenario. - -# :param callable ciphertext_writer: Callable that will write the requested named ciphertext and -# return a URI locating the written data -# :param str plaintext_uri: URI locating the written plaintext data for this scenario -# :return: Decrypt test scenario that describes the generated scenario -# :rtype: MessageDecryptionTestScenario -# """ -# commitment_policy = CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT -# if self.algorithm.is_committing(): -# commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT - -# client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=commitment_policy) -# encrypt_kwargs = dict( -# source=self.plaintext, -# algorithm=self.algorithm, -# frame_length=self.frame_size, -# encryption_context=self.encryption_context, -# ) -# if materials_manager: -# encrypt_kwargs["materials_manager"] = materials_manager -# else: -# encrypt_kwargs["key_provider"] = self.master_key_provider_fn() -# ciphertext, _header = client.encrypt(**encrypt_kwargs) -# return ciphertext - -# @attr.s -# class MessageEncryptionWithKeyringsTestScenario(MessageEncryptionTestScenario): -# # pylint: disable=too-many-instance-attributes -# """Data class for a single full message decrypt test scenario that uses keyrings. - -# :param master_key_specs: Iterable of loaded master key specifications -# :type master_key_specs: iterable of :class:`MasterKeySpec` -# :param Callable master_key_provider_fn: -# """ - -# master_key_specs = attr.ib(validator=iterable_validator(list, MasterKeySpec)) -# master_key_provider_fn = attr.ib(validator=attr.validators.is_callable()) - -# @classmethod -# def from_scenario(cls, scenario, keys_uri, plaintexts): -# # type: (ENCRYPT_SCENARIO_SPEC, KeysManifest, Dict[str, bytes]) -> MessageEncryptionTestScenario -# """Load from a scenario specification. - -# :param dict scenario: Scenario specification JSON -# :param KeysManifest keys: Loaded keys -# :param dict plaintexts: Mapping of plaintext names to plaintext values -# :return: Loaded test scenario -# :rtype: MessageEncryptionTestScenario -# """ -# algorithm = algorithm_suite_from_string_id(scenario["algorithm"]) -# # manifest still keys these as `master-keys` even though these are keyrings -# master_key_specs = [KeyringSpec.from_scenario(spec) for spec in scenario["master-keys"]] - -# def keyring_provider_fn(): -# return keyring_from_master_key_specs(keys_uri, master_key_specs) - -# return cls( -# plaintext_name=scenario["plaintext"], -# plaintext=plaintexts[scenario["plaintext"]], -# algorithm=algorithm, -# frame_size=scenario["frame-size"], -# encryption_context=scenario["encryption-context"], -# master_key=True, -# master_key_specs=master_key_specs, -# master_key_provider_fn=keyring_provider_fn, -# ) - -# def run(self, materials_manager=None): -# """Run this scenario, writing the resulting ciphertext with ``ciphertext_writer`` and returning -# a :class:`MessageDecryptionTestScenario` that describes the matching decrypt scenario. - -# :param callable ciphertext_writer: Callable that will write the requested named ciphertext and -# return a URI locating the written data -# :param str plaintext_uri: URI locating the written plaintext data for this scenario -# :return: Decrypt test scenario that describes the generated scenario -# :rtype: MessageDecryptionTestScenario -# """ -# commitment_policy = CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT -# if self.algorithm.is_committing(): -# commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT - -# client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=commitment_policy) -# encrypt_kwargs = dict( -# source=self.plaintext, -# algorithm=self.algorithm, -# frame_length=self.frame_size, -# encryption_context=self.encryption_context, -# ) -# if materials_manager: -# encrypt_kwargs["materials_manager"] = materials_manager -# else: -# encrypt_kwargs["keyring"] = self.keyring_provider_fn() -# ciphertext, _header = client.encrypt(**encrypt_kwargs) -# return ciphertext - @attr.s class MessageEncryptionManifest(object): """AWS Encryption SDK Encrypt Message manifest handler. diff --git a/test_vector_handlers/test/mpl/integration/commands/test_i_encrypt_keyrings.py b/test_vector_handlers/test/integration/commands/test_i_full_message_encrypt.py similarity index 62% rename from test_vector_handlers/test/mpl/integration/commands/test_i_encrypt_keyrings.py rename to test_vector_handlers/test/integration/commands/test_i_full_message_encrypt.py index 85c94dd22..6305a15da 100644 --- a/test_vector_handlers/test/mpl/integration/commands/test_i_encrypt_keyrings.py +++ b/test_vector_handlers/test/integration/commands/test_i_full_message_encrypt.py @@ -11,35 +11,27 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. """ -Integration tests for `awses_test_vectors.commands` with keyrings. +Integration tests for ``awses_test_vectors.commands``. """ import pytest from awses_test_vectors.commands import full_message_decrypt, full_message_decrypt_generate, full_message_encrypt -from ....integration.integration_test_utils import ( # noqa pylint: disable=unused-import +from ..integration_test_utils import ( # noqa pylint: disable=unused-import full_message_decrypt_generation_vectors, full_message_encrypt_vectors, ) - pytestmark = [pytest.mark.integ] def test_full_message_encrypt_canonical_full(full_message_encrypt_vectors): - full_message_encrypt.cli(["--input", full_message_encrypt_vectors, "--keyrings"]) + full_message_encrypt.cli(["--input", full_message_encrypt_vectors]) def test_full_message_cycle_canonical_full(tmpdir, full_message_decrypt_generation_vectors): - # Generate vectors using keyring interfaces - keyring_output_dir = tmpdir.join("output-keyrings") - full_message_decrypt_generate.cli([ - "--output", - str(keyring_output_dir), - "--input", - full_message_decrypt_generation_vectors, - "--keyrings" - ]) - - keyring_decrypt_manifest_file = keyring_output_dir.join("manifest.json") - full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file), "--keyrings"]) + output_dir = tmpdir.join("output") + full_message_decrypt_generate.cli(["--output", str(output_dir), "--input", full_message_decrypt_generation_vectors]) + + decrypt_manifest_file = output_dir.join("manifest.json") + full_message_decrypt.cli(["--input", str(decrypt_manifest_file)]) diff --git a/test_vector_handlers/test/integration/integration_test_utils.py b/test_vector_handlers/test/integration/integration_test_utils.py index b8c8beb56..fbe6cf7b7 100644 --- a/test_vector_handlers/test/integration/integration_test_utils.py +++ b/test_vector_handlers/test/integration/integration_test_utils.py @@ -18,47 +18,20 @@ import pytest -here = os.path.abspath(os.path.dirname(__file__)) - - -def legacy_vectors_dir(): +def vectors_dir(): + here = os.path.abspath(os.path.dirname(__file__)) return os.path.abspath(os.path.join(here, "..", "aws-crypto-tools-test-vector-framework")) -def mpl_vectors_dir(): - return os.path.abspath(os.path.join(here, "..", "golden-manifest-TODORENAMEANDGETFROMGHA")) - - -def required_ec_vectors_dir(): - return os.path.abspath(os.path.join(here, "..", "required-ec-TODORENAMEANDGETFROMGHA")) - - @pytest.fixture def full_message_encrypt_vectors(): return os.path.join( - legacy_vectors_dir(), "features", "CANONICAL-GENERATED-MANIFESTS", "0003-awses-message-encryption.v2.json" + vectors_dir(), "features", "CANONICAL-GENERATED-MANIFESTS", "0003-awses-message-encryption.v2.json" ) @pytest.fixture def full_message_decrypt_generation_vectors(): return os.path.join( - legacy_vectors_dir(), - "features", - "CANONICAL-GENERATED-MANIFESTS", - "0006-awses-message-decryption-generation.v2.json" - ) - - -@pytest.fixture -def mpl_decrypt_vectors(): - return os.path.join( - mpl_vectors_dir(), "manifest.json" - ) - - -@pytest.fixture -def required_encryption_context_cmm_decrypt_vectors(): - return os.path.join( - required_ec_vectors_dir(), "manifest.json" + vectors_dir(), "features", "CANONICAL-GENERATED-MANIFESTS", "0006-awses-message-decryption-generation.v2.json" ) diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index c002323d3..654e72189 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -56,6 +56,11 @@ deps = commands = awses_local: {[testenv:base-command]commands} test/integration mplvectors: {[testenv:base-command]commands} test/mpl + + ; full_decrypt_generate: {[testenv:base-command]commands} test/integration/commands/test_i_generate_decrypt_vectors.py + ; full_decrypt: {[testenv:base-command]commands} test/integration/commands/test_i_decrypt_generated_vectors.py + ; full_encrypt: {[testenv:base-command]commands} test/integration/commands/test_i_encrypt_vectors.py + full_decrypt_generate: awses-full-message-decrypt-generate {posargs} full_decrypt: awses-full-message-decrypt {posargs} full_encrypt: awses-full-message-encrypt {posargs} From 446eaa4c93f7c89f9d321d939192ca7dd9df482c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 15:16:42 -0700 Subject: [PATCH 358/376] cleanup --- buildspec.yml | 574 +++++++++++++++++------------------ test_vector_handlers/tox.ini | 8 - 2 files changed, 287 insertions(+), 295 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index 90b5dbfd2..873e5941e 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -42,298 +42,298 @@ batch: env: image: aws/codebuild/standard:5.0 - # # 3.8 - # - identifier: py38_integ - # buildspec: codebuild/py38/integ.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py38_examples - # buildspec: codebuild/py38/examples.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py38_decrypt_dafny_esdk_vectors - # buildspec: codebuild/py38/decrypt_dafny_esdk_vectors.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py38_decrypt_net_401_vectors - # buildspec: codebuild/py38/decrypt_net_401_vectors.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py38_encrypt_masterkey - # buildspec: codebuild/py38/encrypt_masterkey.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py38_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py38/generate_decrypt_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py38_decrypt_masterkey_with_masterkey - # depend-on: - # - py38_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py38/decrypt_masterkey_with_masterkey.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py38_decrypt_masterkey_with_js - # depend-on: - # - py38_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py38/decrypt_masterkey_with_js.yml - # env: - # image: aws/codebuild/standard:5.0 + # 3.8 + - identifier: py38_integ + buildspec: codebuild/py38/integ.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py38_examples + buildspec: codebuild/py38/examples.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py38_decrypt_dafny_esdk_vectors + buildspec: codebuild/py38/decrypt_dafny_esdk_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py38_decrypt_net_401_vectors + buildspec: codebuild/py38/decrypt_net_401_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py38_encrypt_masterkey + buildspec: codebuild/py38/encrypt_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py38_generate_decrypt_vectors_masterkey + buildspec: codebuild/py38/generate_decrypt_vectors_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py38_decrypt_masterkey_with_masterkey + depend-on: + - py38_generate_decrypt_vectors_masterkey + buildspec: codebuild/py38/decrypt_masterkey_with_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py38_decrypt_masterkey_with_js + depend-on: + - py38_generate_decrypt_vectors_masterkey + buildspec: codebuild/py38/decrypt_masterkey_with_js.yml + env: + image: aws/codebuild/standard:5.0 - # # 3.9 - # - identifier: py39_integ - # buildspec: codebuild/py39/integ.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py39_examples - # buildspec: codebuild/py39/examples.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py39_decrypt_dafny_esdk_vectors - # buildspec: codebuild/py39/decrypt_dafny_esdk_vectors.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py39_decrypt_net_401_vectors - # buildspec: codebuild/py39/decrypt_net_401_vectors.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py39_encrypt_masterkey - # buildspec: codebuild/py39/encrypt_masterkey.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py39_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py39/generate_decrypt_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py39_decrypt_masterkey_with_masterkey - # depend-on: - # - py39_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py39/decrypt_masterkey_with_masterkey.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py39_decrypt_masterkey_with_js - # depend-on: - # - py39_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py39/decrypt_masterkey_with_js.yml - # env: - # image: aws/codebuild/standard:5.0 + # 3.9 + - identifier: py39_integ + buildspec: codebuild/py39/integ.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_examples + buildspec: codebuild/py39/examples.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_decrypt_dafny_esdk_vectors + buildspec: codebuild/py39/decrypt_dafny_esdk_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_decrypt_net_401_vectors + buildspec: codebuild/py39/decrypt_net_401_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_encrypt_masterkey + buildspec: codebuild/py39/encrypt_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_generate_decrypt_vectors_masterkey + buildspec: codebuild/py39/generate_decrypt_vectors_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_decrypt_masterkey_with_masterkey + depend-on: + - py39_generate_decrypt_vectors_masterkey + buildspec: codebuild/py39/decrypt_masterkey_with_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_decrypt_masterkey_with_js + depend-on: + - py39_generate_decrypt_vectors_masterkey + buildspec: codebuild/py39/decrypt_masterkey_with_js.yml + env: + image: aws/codebuild/standard:5.0 - # # 3.10 - # - identifier: py310_integ - # buildspec: codebuild/py310/integ.yml - # env: - # image: aws/codebuild/standard:6.0 - # - identifier: py310_examples - # buildspec: codebuild/py310/examples.yml - # env: - # image: aws/codebuild/standard:6.0 - # - identifier: py310_decrypt_dafny_esdk_vectors - # buildspec: codebuild/py310/decrypt_dafny_esdk_vectors.yml - # env: - # image: aws/codebuild/standard:6.0 - # - identifier: py310_decrypt_net_401_vectors - # buildspec: codebuild/py310/decrypt_net_401_vectors.yml - # env: - # image: aws/codebuild/standard:6.0 - # - identifier: py310_encrypt_masterkey - # buildspec: codebuild/py310/encrypt_masterkey.yml - # env: - # image: aws/codebuild/standard:6.0 - # - identifier: py310_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py310/generate_decrypt_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:6.0 - # - identifier: py310_decrypt_masterkey_with_masterkey - # depend-on: - # - py310_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py310/decrypt_masterkey_with_masterkey.yml - # env: - # image: aws/codebuild/standard:6.0 - # - identifier: py310_decrypt_masterkey_with_js - # depend-on: - # - py310_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py310/decrypt_masterkey_with_js.yml - # env: - # image: aws/codebuild/standard:6.0 + # 3.10 + - identifier: py310_integ + buildspec: codebuild/py310/integ.yml + env: + image: aws/codebuild/standard:6.0 + - identifier: py310_examples + buildspec: codebuild/py310/examples.yml + env: + image: aws/codebuild/standard:6.0 + - identifier: py310_decrypt_dafny_esdk_vectors + buildspec: codebuild/py310/decrypt_dafny_esdk_vectors.yml + env: + image: aws/codebuild/standard:6.0 + - identifier: py310_decrypt_net_401_vectors + buildspec: codebuild/py310/decrypt_net_401_vectors.yml + env: + image: aws/codebuild/standard:6.0 + - identifier: py310_encrypt_masterkey + buildspec: codebuild/py310/encrypt_masterkey.yml + env: + image: aws/codebuild/standard:6.0 + - identifier: py310_generate_decrypt_vectors_masterkey + buildspec: codebuild/py310/generate_decrypt_vectors_masterkey.yml + env: + image: aws/codebuild/standard:6.0 + - identifier: py310_decrypt_masterkey_with_masterkey + depend-on: + - py310_generate_decrypt_vectors_masterkey + buildspec: codebuild/py310/decrypt_masterkey_with_masterkey.yml + env: + image: aws/codebuild/standard:6.0 + - identifier: py310_decrypt_masterkey_with_js + depend-on: + - py310_generate_decrypt_vectors_masterkey + buildspec: codebuild/py310/decrypt_masterkey_with_js.yml + env: + image: aws/codebuild/standard:6.0 - # - identifier: py311_integ - # buildspec: codebuild/py311/integ.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_integ_mpl - # buildspec: codebuild/py311/integ_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_examples - # buildspec: codebuild/py311/examples.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_examples_mpl - # buildspec: codebuild/py311/examples_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_dafny_esdk_vectors_masterkey - # buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_dafny_esdk_vectors_keyrings - # buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_net_401_vectors_masterkey - # buildspec: codebuild/py311/decrypt_net_401_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_net_401_vectors_keyrings - # buildspec: codebuild/py311/decrypt_net_401_vectors_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_encrypt_masterkey - # buildspec: codebuild/py311/encrypt_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_encrypt_keyrings - # buildspec: codebuild/py311/encrypt_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py311/generate_decrypt_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_masterkey_with_masterkey - # depend-on: - # - py311_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py311/decrypt_masterkey_with_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_masterkey_with_keyrings - # depend-on: - # - py311_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py311/decrypt_masterkey_with_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_masterkey_with_js - # depend-on: - # - py311_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py311/decrypt_masterkey_with_js.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py311/generate_decrypt_vectors_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_keyrings_with_masterkey - # depend-on: - # - py311_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py311/decrypt_keyrings_with_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_keyrings_with_keyrings - # depend-on: - # - py311_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py311/decrypt_keyrings_with_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_keyrings_with_js - # depend-on: - # - py311_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py311/decrypt_keyrings_with_js.yml - # env: - # image: aws/codebuild/standard:7.0 + - identifier: py311_integ + buildspec: codebuild/py311/integ.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_integ_mpl + buildspec: codebuild/py311/integ_mpl.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_examples + buildspec: codebuild/py311/examples.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_examples_mpl + buildspec: codebuild/py311/examples_mpl.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_dafny_esdk_vectors_masterkey + buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_dafny_esdk_vectors_keyrings + buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_net_401_vectors_masterkey + buildspec: codebuild/py311/decrypt_net_401_vectors_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_net_401_vectors_keyrings + buildspec: codebuild/py311/decrypt_net_401_vectors_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_encrypt_masterkey + buildspec: codebuild/py311/encrypt_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_encrypt_keyrings + buildspec: codebuild/py311/encrypt_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_generate_decrypt_vectors_masterkey + buildspec: codebuild/py311/generate_decrypt_vectors_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_masterkey_with_masterkey + depend-on: + - py311_generate_decrypt_vectors_masterkey + buildspec: codebuild/py311/decrypt_masterkey_with_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_masterkey_with_keyrings + depend-on: + - py311_generate_decrypt_vectors_masterkey + buildspec: codebuild/py311/decrypt_masterkey_with_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_masterkey_with_js + depend-on: + - py311_generate_decrypt_vectors_masterkey + buildspec: codebuild/py311/decrypt_masterkey_with_js.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_generate_decrypt_vectors_keyrings + buildspec: codebuild/py311/generate_decrypt_vectors_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_keyrings_with_masterkey + depend-on: + - py311_generate_decrypt_vectors_keyrings + buildspec: codebuild/py311/decrypt_keyrings_with_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_keyrings_with_keyrings + depend-on: + - py311_generate_decrypt_vectors_keyrings + buildspec: codebuild/py311/decrypt_keyrings_with_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_keyrings_with_js + depend-on: + - py311_generate_decrypt_vectors_keyrings + buildspec: codebuild/py311/decrypt_keyrings_with_js.yml + env: + image: aws/codebuild/standard:7.0 - # - identifier: py312_integ - # buildspec: codebuild/py312/integ.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_integ_mpl - # buildspec: codebuild/py312/integ_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_examples - # buildspec: codebuild/py312/examples.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_examples_mpl - # buildspec: codebuild/py312/examples_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_dafny_esdk_vectors_masterkey - # buildspec: codebuild/py312/decrypt_dafny_esdk_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_dafny_esdk_vectors_keyrings - # buildspec: codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_net_401_vectors_masterkey - # buildspec: codebuild/py312/decrypt_net_401_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_net_401_vectors_keyrings - # buildspec: codebuild/py312/decrypt_net_401_vectors_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_encrypt_masterkey - # buildspec: codebuild/py312/encrypt_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_encrypt_keyrings - # buildspec: codebuild/py312/encrypt_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py312/generate_decrypt_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_masterkey_with_masterkey - # depend-on: - # - py312_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py312/decrypt_masterkey_with_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_masterkey_with_keyrings - # depend-on: - # - py312_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py312/decrypt_masterkey_with_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_masterkey_with_js - # depend-on: - # - py312_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py312/decrypt_masterkey_with_js.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py312/generate_decrypt_vectors_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_keyrings_with_masterkey - # depend-on: - # - py312_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py312/decrypt_keyrings_with_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_keyrings_with_keyrings - # depend-on: - # - py312_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py312/decrypt_keyrings_with_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_keyrings_with_js - # depend-on: - # - py312_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py312/decrypt_keyrings_with_js.yml - # env: - # image: aws/codebuild/standard:7.0 + - identifier: py312_integ + buildspec: codebuild/py312/integ.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_integ_mpl + buildspec: codebuild/py312/integ_mpl.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_examples + buildspec: codebuild/py312/examples.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_examples_mpl + buildspec: codebuild/py312/examples_mpl.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_dafny_esdk_vectors_masterkey + buildspec: codebuild/py312/decrypt_dafny_esdk_vectors_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_dafny_esdk_vectors_keyrings + buildspec: codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_net_401_vectors_masterkey + buildspec: codebuild/py312/decrypt_net_401_vectors_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_net_401_vectors_keyrings + buildspec: codebuild/py312/decrypt_net_401_vectors_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_encrypt_masterkey + buildspec: codebuild/py312/encrypt_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_encrypt_keyrings + buildspec: codebuild/py312/encrypt_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_generate_decrypt_vectors_masterkey + buildspec: codebuild/py312/generate_decrypt_vectors_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_masterkey_with_masterkey + depend-on: + - py312_generate_decrypt_vectors_masterkey + buildspec: codebuild/py312/decrypt_masterkey_with_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_masterkey_with_keyrings + depend-on: + - py312_generate_decrypt_vectors_masterkey + buildspec: codebuild/py312/decrypt_masterkey_with_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_masterkey_with_js + depend-on: + - py312_generate_decrypt_vectors_masterkey + buildspec: codebuild/py312/decrypt_masterkey_with_js.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_generate_decrypt_vectors_keyrings + buildspec: codebuild/py312/generate_decrypt_vectors_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_keyrings_with_masterkey + depend-on: + - py312_generate_decrypt_vectors_keyrings + buildspec: codebuild/py312/decrypt_keyrings_with_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_keyrings_with_keyrings + depend-on: + - py312_generate_decrypt_vectors_keyrings + buildspec: codebuild/py312/decrypt_keyrings_with_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_keyrings_with_js + depend-on: + - py312_generate_decrypt_vectors_keyrings + buildspec: codebuild/py312/decrypt_keyrings_with_js.yml + env: + image: aws/codebuild/standard:7.0 - # - identifier: code_coverage - # buildspec: codebuild/coverage/coverage.yml - # - identifier: code_coverage_mpl - # buildspec: codebuild/coverage/coverage_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 + - identifier: code_coverage + buildspec: codebuild/coverage/coverage.yml + - identifier: code_coverage_mpl + buildspec: codebuild/coverage/coverage_mpl.yml + env: + image: aws/codebuild/standard:7.0 - # - identifier: compliance - # buildspec: codebuild/compliance/compliance.yml + - identifier: compliance + buildspec: codebuild/compliance/compliance.yml diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index 654e72189..70819dd5f 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -4,7 +4,6 @@ envlist = # so until release we can only effectively test the local version of the ESDK. py{37,38,39,310}-awses_local py{311,312}-awses_local{,-mpl} - py{311,312}-mplvectors-mpl # 1.2.0 and 1.2.max are being difficult because of attrs bandit, doc8, readme, {flake8,pylint}{,-tests}, @@ -54,13 +53,6 @@ deps = mpl: -rrequirements_mpl.txt .. commands = - awses_local: {[testenv:base-command]commands} test/integration - mplvectors: {[testenv:base-command]commands} test/mpl - - ; full_decrypt_generate: {[testenv:base-command]commands} test/integration/commands/test_i_generate_decrypt_vectors.py - ; full_decrypt: {[testenv:base-command]commands} test/integration/commands/test_i_decrypt_generated_vectors.py - ; full_encrypt: {[testenv:base-command]commands} test/integration/commands/test_i_encrypt_vectors.py - full_decrypt_generate: awses-full-message-decrypt-generate {posargs} full_decrypt: awses-full-message-decrypt {posargs} full_encrypt: awses-full-message-encrypt {posargs} From 70b68f9bffe71db82a1b9addaee2d2dfe80bd883 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 15:54:24 -0700 Subject: [PATCH 359/376] cleanup --- .../test/integration/__init__.py | 2 + .../test/integration/commands/__init__.py | 2 + test_vector_handlers/test/keys.json | 214 ------------------ test_vector_handlers/test/mpl/__init__.py | 0 .../test/mpl/integration/__init__.py | 0 .../test/mpl/integration/commands/__init__.py | 0 test_vector_handlers/tox.ini | 3 +- 7 files changed, 6 insertions(+), 215 deletions(-) delete mode 100644 test_vector_handlers/test/keys.json delete mode 100644 test_vector_handlers/test/mpl/__init__.py delete mode 100644 test_vector_handlers/test/mpl/integration/__init__.py delete mode 100644 test_vector_handlers/test/mpl/integration/commands/__init__.py diff --git a/test_vector_handlers/test/integration/__init__.py b/test_vector_handlers/test/integration/__init__.py index e69de29bb..76a5b798a 100644 --- a/test_vector_handlers/test/integration/__init__.py +++ b/test_vector_handlers/test/integration/__init__.py @@ -0,0 +1,2 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 \ No newline at end of file diff --git a/test_vector_handlers/test/integration/commands/__init__.py b/test_vector_handlers/test/integration/commands/__init__.py index e69de29bb..76a5b798a 100644 --- a/test_vector_handlers/test/integration/commands/__init__.py +++ b/test_vector_handlers/test/integration/commands/__init__.py @@ -0,0 +1,2 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 \ No newline at end of file diff --git a/test_vector_handlers/test/keys.json b/test_vector_handlers/test/keys.json deleted file mode 100644 index 304dae5f7..000000000 --- a/test_vector_handlers/test/keys.json +++ /dev/null @@ -1,214 +0,0 @@ -{ - "manifest": { - "type": "keys", - "version": 3 - }, - "keys": { - "aes-128": { - "encrypt": true, - "decrypt": true, - "algorithm": "aes", - "type": "symmetric", - "bits": 128, - "encoding": "base64", - "material": "AAECAwQFBgcICRAREhMUFQ==", - "key-id": "aes-128" - }, - "aes-192": { - "encrypt": true, - "decrypt": true, - "algorithm": "aes", - "type": "symmetric", - "bits": 192, - "encoding": "base64", - "material": "AAECAwQFBgcICRAREhMUFRYXGBkgISIj", - "key-id": "aes-192" - }, - "aes-256": { - "encrypt": true, - "decrypt": true, - "algorithm": "aes", - "type": "symmetric", - "bits": 256, - "encoding": "base64", - "material": "AAECAwQFBgcICRAREhMUFRYXGBkgISIjJCUmJygpMDE=", - "key-id": "aes-256" - }, - "rsa-4096-private": { - "encrypt": true, - "decrypt": true, - "algorithm": "rsa", - "type": "private", - "bits": 4096, - "encoding": "pem", - "material": "-----BEGIN PRIVATE KEY-----\nMIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCztGg1gQ8AjCzz\n1VX6StqtW//jBt2ZQBoApaBa7FmLmdr0YlKaeEKSrItGbvA9tBjgsKhrn8gxTGQc\nuxgM92651jRCbQZyjE6W8kodijhGMXsfKJLfgPp2/I7gZ3dqrSZkejFIYLFb/uF/\nTfAQzNyJUldYdeFojSUPqevMgSAusTgv7dXYt4BCO9mxMp35tgyp5k4vazKJVUgB\nTw87AAYZUGugmi94Wb9JSnqUKI3QzaRN7JADZrHdBO1lIBryfCsjtTnZc7NWZ0yJ\nwmzLY+C5b3y17cy44N0rbjI2QciRhqZ4/9SZ/9ImyFQlB3lr9NSndcT4eE5YC6bH\nba0gOUK9lLXVy6TZ+nRZ4dSddoLX03mpYp+8cQpK6DO3L/PeUY/si0WGsXZfWokd\n4ACwvXWSOjotzjwqwTW8q9udbhUvIHfB02JW+ZQ07b209fBpHRDkZuveOTedTN2Q\nQei4dZDjWW5s4cIIE3dXXeaH8yC02ERIeN+aY6eHngSsP2xoDV3sKNN/yDbCqaMS\nq8ZJbo2rvOFxZHa2nWiV+VLugfO6Xj8jeGeR8vopvbEBZZpAq+Dea2xjY4+XMUQ/\nS1HlRwc9+nkJ5LVfODuE3q9EgJbqbiXe7YckWV3ZqQMybW+dLPxEJs9buOntgHFS\nRYmbKky0bti/ZoZlcZtS0zyjVxlqsQIDAQABAoICAEr3m/GWIXgNAkPGX9PGnmtr\n0dgX6SIhh7d1YOwNZV3DlYAV9HfUa5Fcwc1kQny7QRWbHOepBI7sW2dQ9buTDXIh\nVjPP37yxo6d89EZWfxtpUP+yoXL0D4jL257qCvtJuJZ6E00qaVMDhXbiQKABlo8C\n9sVEiABhwXBDZsctpwtTiykTgv6hrrPy2+H8R8MAm0/VcBCAG9kG5r8FCEmIvQKa\ndgvNxrfiWNZuZ6yfLmpJH54SbhG9Kb4WbCKfvh4ihqyi0btRdSM6fMeLgG9o/zrc\ns54B0kHeLOYNVo0j7FQpZBFeSIbmHfln4RKBh7ntrTke/Ejbh3NbiPvxWSP0P067\nSYWPkQpip2q0ION81wSQZ1haP2GewFFu4IEjG3DlqqpKKGLqXrmjMufnildVFpBx\nir+MgvgQfEBoGEx0aElyO7QuRYaEiXeb/BhMZeC5O65YhJrWSuTVizh3xgJWjgfV\naYwYgxN8SBXBhXLIVvnPhadTqsW1C/aevLOk110eSFWcHf+FCK781ykIzcpXoRGX\nOwWcZzC/fmSABS0yH56ow+I0tjdLIEEMhoa4/kkamioHOJ4yyB+W1DO6/DnMyQlx\ng7y2WsAaIEBoWUARy776k70xPPMtYAxzFXI9KhqRVrPfeaRZ+ojeyLyr3GQGyyoo\ncuGRdMUblsmODv4ixmOxAoIBAQDvkznvVYNdP3Eg5vQeLm/qsP6dLejLijBLeq9i\n7DZH2gRpKcflXZxCkRjsKDDE+fgDcBYEp2zYfRIVvgrxlTQZdaSG+GoDcbjbNQn3\ndjCCtOOACioN/vg2zFlX4Bs6Q+NaV7g5qP5SUaxUBjuHLe7Nc+ZkyheMHuNYVLvk\nHL/IoWyANpZYjMUU3xMbL/J29Gz7CPGr8Si28TihAHGfcNgn8S04OQZhTX+bU805\n/+7B4XW47Mthg/u7hlqFl+YIAaSJYvWkEaVP1A9I7Ve0aMDSMWwzTg9cle2uVaL3\n+PTzWY5coBlHKjqAg9ufhYSDhAqBd/JOSlv8RwcA3PDXJ6C/AoIBAQDABmXXYQky\n7phExXBvkLtJt2TBGjjwulf4R8TC6W5F51jJuoqY/mTqYcLcOn2nYGVwoFvPsy/Q\nCTjfODwJBXzbloXtYFR3PWAeL1Y6+7Cm+koMWIPJyVbD5Fzm+gZStM0GwP8FhDt2\nWt8fWEyXmoLdAy6RAwiEmCagEh8o+13oBfwnBllbz7TxaErsUuR+XVgl/iHwztdv\ncdJKyRgaFfWSh9aiO7EMV2rBGWsoX09SRvprPFAGx8Ffm7YcqIk34QXsQyc45Dyn\nCwkvypxHoaB3ot/48FeFm9IubApb/ctv+EgkBfL4S4bdwRXS1rt+0+QihBoFyP2o\nJ91cdm4hEWCPAoIBAQC6l11hFaYZo0bWDGsHcr2B+dZkzxPoKznQH76n+jeQoLIc\nwgjJkK4afm39yJOrZtEOxGaxu0CgIFFMk9ZsL/wC9EhvQt02z4TdXiLkFK5VrtMd\nr0zv16y06VWQhqBOMf/KJlX6uq9RqADi9HO6pkC+zc0cpPXQEWKaMmygju+kMG2U\nMm/IieMZjWCRJTfgBCE5J88qTsqaKagkZXcZakdAXKwOhQN+F2EStiM6UCZB5PrO\nS8dfrO8ML+ki8Zqck8L1qhiNb5zkXtKExy4u+gNr8khGcT6vqqoSxOoH3mPRgOfL\nJnppne8wlwIf7Vq3H8ka6zPSXEHma999gZcmy9t7AoIBAGbQhiLl79j3a0wXMvZp\nVf5IVYgXFDnAbG2hb7a06bhAAIgyexcjzsC4C2+DWdgOgwHkuoPg+062QV8zauGh\nsJKaa6cHlvIpSJeg3NjD/nfJN3CYzCd0yCIm2Z9Ka6xI5iYhm+pGPNhIG4Na8deS\ngVL46yv1pc/o73VxfoGg5UzgN3xlp97Cva0sHEGguHr4W8Qr59xZw3wGQ4SLW35M\nF6qXVNKUh12GSMCPbZK2RXBWVKqqJmca+WzJoJ6DlsT2lQdFhXCus9L007xlDXxF\nC/hCmw1dEl+VaNo2Ou26W/zdwTKYhNlxBwsg4SB8nPNxXIsmlBBY54froFhriNfn\nx/0CggEAUzz+VMtjoEWw2HSHLOXrO4EmwJniNgiiwfX3DfZE4tMNZgqZwLkq67ns\nT0n3b0XfAOOkLgMZrUoOxPHkxFeyLLf7pAEJe7QNB+Qilw8e2zVqtiJrRk6uDIGJ\nSv+yM52zkImZAe2jOdU3KeUZxSMmb5vIoiPBm+tb2WupAg3YdpKn1/jWTpVmV/+G\nUtTLVE6YpAyFp1gMxhutE9vfIS94ek+vt03AoEOlltt6hqZfv3xmY8vGuAjlnj12\nzHaq+fhCRPsbsZkzJ9nIVdXYnNIEGtMGNnxax7tYRej/UXqyazbxHiJ0iPF4PeDn\ndzxtGxpeTBi+KhKlca8SlCdCqYwG6Q==\n-----END PRIVATE KEY-----", - "key-id": "rsa-4096" - }, - "rsa-4096-public": { - "encrypt": true, - "decrypt": false, - "algorithm": "rsa", - "type": "public", - "bits": 4096, - "encoding": "pem", - "material": "-----BEGIN PUBLIC KEY-----\nMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAs7RoNYEPAIws89VV+kra\nrVv/4wbdmUAaAKWgWuxZi5na9GJSmnhCkqyLRm7wPbQY4LCoa5/IMUxkHLsYDPdu\nudY0Qm0GcoxOlvJKHYo4RjF7HyiS34D6dvyO4Gd3aq0mZHoxSGCxW/7hf03wEMzc\niVJXWHXhaI0lD6nrzIEgLrE4L+3V2LeAQjvZsTKd+bYMqeZOL2syiVVIAU8POwAG\nGVBroJoveFm/SUp6lCiN0M2kTeyQA2ax3QTtZSAa8nwrI7U52XOzVmdMicJsy2Pg\nuW98te3MuODdK24yNkHIkYameP/Umf/SJshUJQd5a/TUp3XE+HhOWAumx22tIDlC\nvZS11cuk2fp0WeHUnXaC19N5qWKfvHEKSugzty/z3lGP7ItFhrF2X1qJHeAAsL11\nkjo6Lc48KsE1vKvbnW4VLyB3wdNiVvmUNO29tPXwaR0Q5Gbr3jk3nUzdkEHouHWQ\n41lubOHCCBN3V13mh/MgtNhESHjfmmOnh54ErD9saA1d7CjTf8g2wqmjEqvGSW6N\nq7zhcWR2tp1olflS7oHzul4/I3hnkfL6Kb2xAWWaQKvg3mtsY2OPlzFEP0tR5UcH\nPfp5CeS1Xzg7hN6vRICW6m4l3u2HJFld2akDMm1vnSz8RCbPW7jp7YBxUkWJmypM\ntG7Yv2aGZXGbUtM8o1cZarECAwEAAQ==\n-----END PUBLIC KEY-----", - "key-id": "rsa-4096" - }, - "us-west-2-decryptable": { - "encrypt": true, - "decrypt": true, - "type": "aws-kms", - "key-id": "arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f" - }, - "us-west-2-encrypt-only": { - "encrypt": true, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws:kms:us-west-2:658956600833:key/590fd781-ddde-4036-abec-3e1ab5a5d2ad" - }, - "us-west-2-mrk": { - "encrypt": true, - "decrypt": true, - "type": "aws-kms", - "key-id": "arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "us-east-1-mrk": { - "encrypt": true, - "decrypt": true, - "type": "aws-kms", - "key-id": "arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "aws:kms:us-west-2:658956600833:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "aws:kms:us-west-2:658956600833:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - ":aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": ":aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "arn-not:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn-not:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "arn:kms:us-west-2:658956600833:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:kms:us-west-2:658956600833:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "arn::kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn::kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "arn:aws-not:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws-not:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "arn:aws:us-west-2:658956600833:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws:us-west-2:658956600833:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "arn:aws::us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws::us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "arn:aws:kms-not:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws:kms-not:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "arn:aws:kms:658956600833:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws:kms:658956600833:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "arn:aws:kms::658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws:kms::658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "arn:aws:kms:us-west-2:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws:kms:us-west-2:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "arn:aws:kms:us-west-2::key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws:kms:us-west-2::key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "arn:aws:kms:us-west-2:658956600833-not:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws:kms:us-west-2:658956600833-not:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "arn:aws:kms:us-west-2:658956600833:mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws:kms:us-west-2:658956600833:mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "arn:aws:kms:us-west-2:658956600833:/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws:kms:us-west-2:658956600833:/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "arn:aws:kms:us-west-2:658956600833:key-not/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws:kms:us-west-2:658956600833:key-not/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "arn:aws:kms:us-west-2:658956600833:key": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws:kms:us-west-2:658956600833:key" - }, - "arn:aws:kms:us-west-2:658956600833:key/": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws:kms:us-west-2:658956600833:key/" - }, - "arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7-not": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7-not" - }, - "arn:aws:kms:us-west-2:658956600833:alias/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws:kms:us-west-2:658956600833:alias/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - } - } -} diff --git a/test_vector_handlers/test/mpl/__init__.py b/test_vector_handlers/test/mpl/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/test_vector_handlers/test/mpl/integration/__init__.py b/test_vector_handlers/test/mpl/integration/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/test_vector_handlers/test/mpl/integration/commands/__init__.py b/test_vector_handlers/test/mpl/integration/commands/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index 70819dd5f..cdb1137fb 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -36,7 +36,7 @@ envlist = # release :: Builds dist files and uploads to pypi pypirc profile. [testenv:base-command] -commands = pytest --basetemp={envtmpdir} -l --cov awses_test_vectors {posargs} +commands = pytest --basetemp={envtmpdir} -l --cov awses_test_vectors test/ --ignore test/mpl {posargs} [testenv] passenv = @@ -53,6 +53,7 @@ deps = mpl: -rrequirements_mpl.txt .. commands = + awses_local: {[testenv:base-command]commands} full_decrypt_generate: awses-full-message-decrypt-generate {posargs} full_decrypt: awses-full-message-decrypt {posargs} full_encrypt: awses-full-message-encrypt {posargs} From 67f0179ebccbdb44d9a02545f0276b73d52c08ce Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 15:59:28 -0700 Subject: [PATCH 360/376] cleanup --- .../materials_managers/mpl/cmm.py | 1 - .../internal/keyvectors_provider.py | 27 --- .../internal/tampering_mpl_materials.py | 169 ------------------ .../full_message/decrypt_generation.py | 2 +- .../manifests/mpl_keyring.py | 2 +- test_vector_handlers/test/__init__.py | 0 .../test/integration/__init__.py | 2 - .../test/integration/commands/__init__.py | 2 - 8 files changed, 2 insertions(+), 203 deletions(-) delete mode 100644 test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py delete mode 100644 test_vector_handlers/src/awses_test_vectors/internal/tampering_mpl_materials.py delete mode 100644 test_vector_handlers/test/__init__.py diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index 71e9adf8b..ebef5f7ac 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -67,7 +67,6 @@ def get_encryption_materials( CryptoMaterialsManagerFromMPL._native_to_mpl_get_encryption_materials( request ) - mpl_output: MPL_GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) return EncryptionMaterialsFromMPL(mpl_output.encryption_materials) except AwsCryptographicMaterialProvidersException as mpl_exception: diff --git a/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py b/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py deleted file mode 100644 index 305459026..000000000 --- a/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Singleton provider for the KeyVectors client.""" -# # Ignore missing MPL TestVectors for pylint, but the MPL TestVectors is required for this file -# pylint: disable=import-error -from aws_cryptography_materialproviderstestvectorkeys.smithygenerated.\ - aws_cryptography_materialproviderstestvectorkeys.client import ( - KeyVectors, - ) -from aws_cryptography_materialproviderstestvectorkeys.smithygenerated.\ - aws_cryptography_materialproviderstestvectorkeys.config import ( - KeyVectorsConfig - ) - -keyvectors_instances = {} - - -# pylint: disable=too-few-public-methods -class KeyVectorsProvider: - """Singleton manager for the KeyVectors client.""" - - instance: KeyVectors - - @classmethod - def get_keyvectors(cls, keys_path): - """Return the singleton KeyVectors client.""" - if keys_path not in keyvectors_instances: - keyvectors_instances[keys_path] = KeyVectors(KeyVectorsConfig(key_manifest_path=keys_path)) - return keyvectors_instances[keys_path] diff --git a/test_vector_handlers/src/awses_test_vectors/internal/tampering_mpl_materials.py b/test_vector_handlers/src/awses_test_vectors/internal/tampering_mpl_materials.py deleted file mode 100644 index 4f7bc658e..000000000 --- a/test_vector_handlers/src/awses_test_vectors/internal/tampering_mpl_materials.py +++ /dev/null @@ -1,169 +0,0 @@ -"""Allows using ESDK-MPL interfaces with the tampering tests. -These must ONLY be used in testing and NOT in production. -""" -import attr -import six -from copy import copy - - -from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager - -# Ignore missing MPL for pylint, but the MPL is required for this class -# pylint: disable=import-error,no-name-in-module -from aws_encryption_sdk.materials_managers.mpl.materials import ( - EncryptionMaterialsFromMPL -) -from aws_encryption_sdk.materials_managers.mpl.cmm import ( - CryptoMaterialsManagerFromMPL -) -from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders -from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig -from aws_cryptographic_materialproviders.mpl.models import ( - CreateDefaultCryptographicMaterialsManagerInput, -) - -try: - from aws_encryption_sdk.identifiers import AlgorithmSuite -except ImportError: - from aws_encryption_sdk.identifiers import Algorithm as AlgorithmSuite - -class HalfSigningCryptoMaterialsManagerFromMPL(CryptoMaterialsManagerFromMPL): - """ - Custom CMM that uses HalfSigningEncryptionMaterialsFromMPL. - This extends CryptoMaterialsManagerFromMPL so ESDK-internal checks - follow MPL logic. - - THIS IS ONLY USED TO CREATE INVALID MESSAGES and should never be used in - production! - """ - - wrapped_default_cmm = attr.ib(validator=attr.validators.instance_of(CryptoMaterialsManagerFromMPL)) - - def __init__(self, master_key_provider): - """Create a new CMM that wraps a the given CMM.""" - mpl = AwsCryptographicMaterialProviders(MaterialProvidersConfig()) - mpl_cmm = mpl.create_default_cryptographic_materials_manager( - CreateDefaultCryptographicMaterialsManagerInput( - keyring=master_key_provider - ) - ) - self.wrapped_default_cmm = CryptoMaterialsManagerFromMPL(mpl_cmm=mpl_cmm) - - def get_encryption_materials(self, request): - """ - Generate half-signing materials by requesting signing materials - from the wrapped default CMM, and then changing the algorithm suite - and removing the signing key from teh result. - """ - if request.algorithm == AlgorithmSuite.AES_256_GCM_HKDF_SHA512_COMMIT_KEY: - signing_request = copy(request) - signing_request.algorithm = AlgorithmSuite.AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384 - - result = HalfSigningEncryptionMaterialsFromMPL( - self.wrapped_default_cmm.get_encryption_materials(signing_request) - ) - - result.algorithm = request.algorithm - result.signing_key = None - - return result - - raise NotImplementedError( - "The half-sign tampering method is only supported on the " - "AES_256_GCM_HKDF_SHA512_COMMIT_KEY algorithm suite." - ) - - def decrypt_materials(self, request): - """Thunks to the wrapped default CMM""" - return self.wrapped_default_cmm.decrypt_materials(request) - - -class HalfSigningEncryptionMaterialsFromMPL(EncryptionMaterialsFromMPL): - """Allows overriding the algorithm and signing_key for EncryptionMaterialsFromMPL. - This must ONLY be used in testing and NOT in production. - This is used in testing malicious message modification (HalfSigningTampering). - """ - - _underlying_materials: EncryptionMaterialsFromMPL - - def __init__(self, underling_materials): - self._underlying_materials = underling_materials - - # pylint thinks EncryptionMaterialsFromMPL.algorithm is a method - # pylint: disable=invalid-overridden-method - @property - def algorithm(self): - """Return any previously-provided overriden algorithm; - if none was provided, returns underlying algorithm from encryption materials. - """ - if hasattr(self, "set_algorithm"): - return self.set_algorithm - return self._underlying_materials.algorithm - - @algorithm.setter - def algorithm(self, algorithm): - self.set_algorithm = algorithm - - # pylint thinks EncryptionMaterialsFromMPL.signing_key is a method - # pylint: disable=invalid-overridden-method - @property - def signing_key(self): - """Return any previously-provided overriden signing_key; - if none was provided, returns underlying signing_key from encryption materials. - """ - if hasattr(self, "set_signing_key"): - return self.set_signing_key - return self._underlying_materials.algorithm - - @signing_key.setter - def signing_key(self, signing_key): - self.set_signing_key = signing_key - - @property - def encryption_context(self): - return self._underlying_materials.encryption_context - - @property - def encrypted_data_keys(self): - return self._underlying_materials.encrypted_data_keys - - @property - def data_encryption_key(self): - return self._underlying_materials.data_encryption_key - - @property - def required_encryption_context_keys(self): - return self._underlying_materials.required_encryption_context_keys - - -class ProviderInfoChangingCryptoMaterialsManagerFromMPL(CryptoMaterialsManagerFromMPL): - """ - Custom CMM that modifies the provider info field on EDKs. - This extends CryptoMaterialsManagerFromMPL so ESDK-internal checks - follow MPL logic. - - THIS IS ONLY USED TO CREATE INVALID MESSAGES and should never be used in - production! - """ - - wrapped_cmm = attr.ib(validator=attr.validators.instance_of(CryptoMaterialsManager)) - new_provider_info = attr.ib(validator=attr.validators.instance_of(six.string_types)) - - def __init__(self, materials_manager, new_provider_info): - """Create a new CMM that wraps a the given CMM.""" - self.wrapped_cmm = materials_manager - self.new_provider_info = new_provider_info - - def get_encryption_materials(self, request): - """ - Request materials from the wrapped CMM, and then change the provider info - on each EDK. - """ - result = self.wrapped_cmm.get_encryption_materials(request) - for encrypted_data_key in result.encrypted_data_keys: - encrypted_data_key.key_provider.key_info = self.new_provider_info - return result - - def decrypt_materials(self, request): - """Thunks to the wrapped CMM""" - return self.wrapped_cmm.decrypt_materials(request) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index f94facf13..e7aa747a7 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -44,7 +44,7 @@ from aws_encryption_sdk.materials_managers.mpl.materials import ( EncryptionMaterialsFromMPL ) - from awses_test_vectors.internal.tampering_mpl_materials import ( + from awses_test_vectors.internal.mpl.tampering_mpl_materials import ( HalfSigningEncryptionMaterialsFromMPL, ProviderInfoChangingCryptoMaterialsManagerFromMPL, HalfSigningCryptoMaterialsManagerFromMPL, diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index 55a9276c9..c05c14714 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -33,7 +33,7 @@ import _dafny import UTF8 -from awses_test_vectors.internal.keyvectors_provider import KeyVectorsProvider +from awses_test_vectors.internal.mpl.keyvectors_provider import KeyVectorsProvider from awses_test_vectors.manifests.keys import KeysManifest # noqa: disable=F401 diff --git a/test_vector_handlers/test/__init__.py b/test_vector_handlers/test/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/test_vector_handlers/test/integration/__init__.py b/test_vector_handlers/test/integration/__init__.py index 76a5b798a..e69de29bb 100644 --- a/test_vector_handlers/test/integration/__init__.py +++ b/test_vector_handlers/test/integration/__init__.py @@ -1,2 +0,0 @@ -# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 \ No newline at end of file diff --git a/test_vector_handlers/test/integration/commands/__init__.py b/test_vector_handlers/test/integration/commands/__init__.py index 76a5b798a..e69de29bb 100644 --- a/test_vector_handlers/test/integration/commands/__init__.py +++ b/test_vector_handlers/test/integration/commands/__init__.py @@ -1,2 +0,0 @@ -# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 \ No newline at end of file From 7d8a515fbb8589243f6538e68f7110994d7727ee Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 16:08:20 -0700 Subject: [PATCH 361/376] cleanup --- .../manifests/full_message/decrypt_generation.py | 2 +- .../src/awses_test_vectors/manifests/mpl_keyring.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index e7aa747a7..6213fc3a6 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -51,7 +51,7 @@ ) _HAS_MPL = True -except ImportError as e: +except ImportError:" _HAS_MPL = False diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index c05c14714..ec35147c0 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -33,6 +33,8 @@ import _dafny import UTF8 +# Ignore pylint not being able to read a module that requires the MPL +# pylint: disable=no-name-in-module from awses_test_vectors.internal.mpl.keyvectors_provider import KeyVectorsProvider from awses_test_vectors.manifests.keys import KeysManifest # noqa: disable=F401 From 48974b051d01003135761a405ec66fe03e68b385 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 16:11:13 -0700 Subject: [PATCH 362/376] debug --- .../src/awses_test_vectors/manifests/full_message/decrypt.py | 3 ++- .../manifests/full_message/decrypt_generation.py | 3 ++- .../src/awses_test_vectors/manifests/full_message/encrypt.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index 2aaaf1bca..6f8b43592 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -46,7 +46,8 @@ ) _HAS_MPL = True -except ImportError: +except ImportError as e: + print(f"ImportError: {e}") _HAS_MPL = False diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 6213fc3a6..45cd3b9d0 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -51,7 +51,8 @@ ) _HAS_MPL = True -except ImportError:" +except ImportError as e: + print(f"ImportError: {e}") _HAS_MPL = False diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 57de8504c..d06d543a2 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -50,7 +50,8 @@ from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_from_master_key_specs _HAS_MPL = True -except ImportError: +except ImportError as e: + print(f"ImportError: {e}") _HAS_MPL = False From 1bb55273a0e9a48e8d9cd29ff4e6ff216184a9ff Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 16:14:28 -0700 Subject: [PATCH 363/376] cleanup --- .../internal/mpl/keyvectors_provider.py | 27 +++ .../internal/mpl/tampering_mpl_materials.py | 172 ++++++++++++++++++ .../manifests/full_message/decrypt.py | 3 +- .../full_message/decrypt_generation.py | 3 +- .../manifests/full_message/encrypt.py | 3 +- 5 files changed, 202 insertions(+), 6 deletions(-) create mode 100644 test_vector_handlers/src/awses_test_vectors/internal/mpl/keyvectors_provider.py create mode 100644 test_vector_handlers/src/awses_test_vectors/internal/mpl/tampering_mpl_materials.py diff --git a/test_vector_handlers/src/awses_test_vectors/internal/mpl/keyvectors_provider.py b/test_vector_handlers/src/awses_test_vectors/internal/mpl/keyvectors_provider.py new file mode 100644 index 000000000..305459026 --- /dev/null +++ b/test_vector_handlers/src/awses_test_vectors/internal/mpl/keyvectors_provider.py @@ -0,0 +1,27 @@ +"""Singleton provider for the KeyVectors client.""" +# # Ignore missing MPL TestVectors for pylint, but the MPL TestVectors is required for this file +# pylint: disable=import-error +from aws_cryptography_materialproviderstestvectorkeys.smithygenerated.\ + aws_cryptography_materialproviderstestvectorkeys.client import ( + KeyVectors, + ) +from aws_cryptography_materialproviderstestvectorkeys.smithygenerated.\ + aws_cryptography_materialproviderstestvectorkeys.config import ( + KeyVectorsConfig + ) + +keyvectors_instances = {} + + +# pylint: disable=too-few-public-methods +class KeyVectorsProvider: + """Singleton manager for the KeyVectors client.""" + + instance: KeyVectors + + @classmethod + def get_keyvectors(cls, keys_path): + """Return the singleton KeyVectors client.""" + if keys_path not in keyvectors_instances: + keyvectors_instances[keys_path] = KeyVectors(KeyVectorsConfig(key_manifest_path=keys_path)) + return keyvectors_instances[keys_path] diff --git a/test_vector_handlers/src/awses_test_vectors/internal/mpl/tampering_mpl_materials.py b/test_vector_handlers/src/awses_test_vectors/internal/mpl/tampering_mpl_materials.py new file mode 100644 index 000000000..7ba471506 --- /dev/null +++ b/test_vector_handlers/src/awses_test_vectors/internal/mpl/tampering_mpl_materials.py @@ -0,0 +1,172 @@ +"""Allows using ESDK-MPL interfaces with the tampering tests. +These must ONLY be used in testing and NOT in production. +""" +import attr +import six +from copy import copy + + +from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager + +# Ignore missing MPL for pylint, but the MPL is required for this class +# pylint: disable=import-error,no-name-in-module +from aws_encryption_sdk.materials_managers.mpl.materials import ( + EncryptionMaterialsFromMPL +) +from aws_encryption_sdk.materials_managers.mpl.cmm import ( + CryptoMaterialsManagerFromMPL +) +from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders +from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig +from aws_cryptographic_materialproviders.mpl.models import ( + CreateDefaultCryptographicMaterialsManagerInput, +) + +try: + from aws_encryption_sdk.identifiers import AlgorithmSuite +except ImportError: + from aws_encryption_sdk.identifiers import Algorithm as AlgorithmSuite + + +class HalfSigningCryptoMaterialsManagerFromMPL(CryptoMaterialsManagerFromMPL): + """ + Custom CMM that uses HalfSigningEncryptionMaterialsFromMPL. + This extends CryptoMaterialsManagerFromMPL so ESDK-internal checks + follow MPL logic. + + THIS IS ONLY USED TO CREATE INVALID MESSAGES and should never be used in + production! + """ + + wrapped_default_cmm = attr.ib(validator=attr.validators.instance_of(CryptoMaterialsManagerFromMPL)) + + def __init__(self, master_key_provider): + """Create a new CMM that wraps a the given CMM.""" + mpl = AwsCryptographicMaterialProviders(MaterialProvidersConfig()) + mpl_cmm = mpl.create_default_cryptographic_materials_manager( + CreateDefaultCryptographicMaterialsManagerInput( + keyring=master_key_provider + ) + ) + self.wrapped_default_cmm = CryptoMaterialsManagerFromMPL(mpl_cmm=mpl_cmm) + + def get_encryption_materials(self, request): + """ + Generate half-signing materials by requesting signing materials + from the wrapped default CMM, and then changing the algorithm suite + and removing the signing key from teh result. + """ + if request.algorithm == AlgorithmSuite.AES_256_GCM_HKDF_SHA512_COMMIT_KEY: + signing_request = copy(request) + signing_request.algorithm = AlgorithmSuite.AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384 + + result = HalfSigningEncryptionMaterialsFromMPL( + self.wrapped_default_cmm.get_encryption_materials(signing_request) + ) + + result.algorithm = request.algorithm + result.signing_key = None + + return result + + raise NotImplementedError( + "The half-sign tampering method is only supported on the " + "AES_256_GCM_HKDF_SHA512_COMMIT_KEY algorithm suite." + ) + + def decrypt_materials(self, request): + """Thunks to the wrapped default CMM""" + return self.wrapped_default_cmm.decrypt_materials(request) + + +class HalfSigningEncryptionMaterialsFromMPL(EncryptionMaterialsFromMPL): + """Allows overriding properties inside the EncryptionMaterialsFromMPL. + The test vectors to this to "tamper" with the messages + and ensure they fail with expected errors. + This must ONLY be used in testing and NOT in production. + This is used in testing malicious message modification (HalfSigningTampering). + """ + + _underlying_materials: EncryptionMaterialsFromMPL + + def __init__(self, underling_materials): + self._underlying_materials = underling_materials + + # pylint thinks EncryptionMaterialsFromMPL.algorithm is a method + # pylint: disable=invalid-overridden-method + @property + def algorithm(self): + """Return any previously-provided overriden algorithm; + if none was provided, returns underlying algorithm from encryption materials. + """ + if hasattr(self, "set_algorithm"): + return self.set_algorithm + return self._underlying_materials.algorithm + + @algorithm.setter + def algorithm(self, algorithm): + self.set_algorithm = algorithm + + # pylint thinks EncryptionMaterialsFromMPL.signing_key is a method + # pylint: disable=invalid-overridden-method + @property + def signing_key(self): + """Return any previously-provided overriden signing_key; + if none was provided, returns underlying signing_key from encryption materials. + """ + if hasattr(self, "set_signing_key"): + return self.set_signing_key + return self._underlying_materials.algorithm + + @signing_key.setter + def signing_key(self, signing_key): + self.set_signing_key = signing_key + + @property + def encryption_context(self): + return self._underlying_materials.encryption_context + + @property + def encrypted_data_keys(self): + return self._underlying_materials.encrypted_data_keys + + @property + def data_encryption_key(self): + return self._underlying_materials.data_encryption_key + + @property + def required_encryption_context_keys(self): + return self._underlying_materials.required_encryption_context_keys + + +class ProviderInfoChangingCryptoMaterialsManagerFromMPL(CryptoMaterialsManagerFromMPL): + """ + Custom CMM that modifies the provider info field on EDKs. + This extends CryptoMaterialsManagerFromMPL so ESDK-internal checks + follow MPL logic. + + THIS IS ONLY USED TO CREATE INVALID MESSAGES and should never be used in + production! + """ + + wrapped_cmm = attr.ib(validator=attr.validators.instance_of(CryptoMaterialsManager)) + new_provider_info = attr.ib(validator=attr.validators.instance_of(six.string_types)) + + def __init__(self, materials_manager, new_provider_info): + """Create a new CMM that wraps a the given CMM.""" + self.wrapped_cmm = materials_manager + self.new_provider_info = new_provider_info + + def get_encryption_materials(self, request): + """ + Request materials from the wrapped CMM, and then change the provider info + on each EDK. + """ + result = self.wrapped_cmm.get_encryption_materials(request) + for encrypted_data_key in result.encrypted_data_keys: + encrypted_data_key.key_provider.key_info = self.new_provider_info + return result + + def decrypt_materials(self, request): + """Thunks to the wrapped CMM""" + return self.wrapped_cmm.decrypt_materials(request) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index 6f8b43592..2aaaf1bca 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -46,8 +46,7 @@ ) _HAS_MPL = True -except ImportError as e: - print(f"ImportError: {e}") +except ImportError: _HAS_MPL = False diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 45cd3b9d0..50c14a091 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -51,8 +51,7 @@ ) _HAS_MPL = True -except ImportError as e: - print(f"ImportError: {e}") +except ImportError: _HAS_MPL = False diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index d06d543a2..57de8504c 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -50,8 +50,7 @@ from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_from_master_key_specs _HAS_MPL = True -except ImportError as e: - print(f"ImportError: {e}") +except ImportError: _HAS_MPL = False From 72de35b7a20e80114001dd061ddb039038c344f2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 16:20:14 -0700 Subject: [PATCH 364/376] cleanup --- .../internal/mpl/tampering_mpl_materials.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/internal/mpl/tampering_mpl_materials.py b/test_vector_handlers/src/awses_test_vectors/internal/mpl/tampering_mpl_materials.py index 7ba471506..8b540f8dd 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/mpl/tampering_mpl_materials.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/mpl/tampering_mpl_materials.py @@ -89,8 +89,10 @@ class HalfSigningEncryptionMaterialsFromMPL(EncryptionMaterialsFromMPL): _underlying_materials: EncryptionMaterialsFromMPL - def __init__(self, underling_materials): - self._underlying_materials = underling_materials + def __init__(self, underlying_materials): + """Creates a HalfSigningEncryptionMaterialsFromMPL wrapper + around underlying_materials.""" + self._underlying_materials = underlying_materials # pylint thinks EncryptionMaterialsFromMPL.algorithm is a method # pylint: disable=invalid-overridden-method @@ -124,18 +126,22 @@ def signing_key(self, signing_key): @property def encryption_context(self): + """Get encryption_context from _underlying_materials.""" return self._underlying_materials.encryption_context @property def encrypted_data_keys(self): + """Get encrypted_data_keys from _underlying_materials.""" return self._underlying_materials.encrypted_data_keys @property def data_encryption_key(self): + """Get data_encryption_key from _underlying_materials.""" return self._underlying_materials.data_encryption_key @property def required_encryption_context_keys(self): + """Get required_encryption_context_keys from _underlying_materials.""" return self._underlying_materials.required_encryption_context_keys From f2792bccea7980b4b990e2bc1cc87b5cc768eba6 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 16:22:27 -0700 Subject: [PATCH 365/376] cleanup --- .../src/awses_test_vectors/internal/mpl/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 test_vector_handlers/src/awses_test_vectors/internal/mpl/__init__.py diff --git a/test_vector_handlers/src/awses_test_vectors/internal/mpl/__init__.py b/test_vector_handlers/src/awses_test_vectors/internal/mpl/__init__.py new file mode 100644 index 000000000..a9f648dff --- /dev/null +++ b/test_vector_handlers/src/awses_test_vectors/internal/mpl/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Internal modules that require the aws-cryptographic-material-providers library.""" From df45d5db58691d70e2aa4f17c7f4ab9ab5887628 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 16:25:45 -0700 Subject: [PATCH 366/376] cleanup --- .../awses_test_vectors/internal/mpl/__init__.py | 14 ++------------ .../internal/mpl/tampering_mpl_materials.py | 7 ++++--- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/internal/mpl/__init__.py b/test_vector_handlers/src/awses_test_vectors/internal/mpl/__init__.py index a9f648dff..11e9569d9 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/mpl/__init__.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/mpl/__init__.py @@ -1,13 +1,3 @@ -# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 """Internal modules that require the aws-cryptographic-material-providers library.""" diff --git a/test_vector_handlers/src/awses_test_vectors/internal/mpl/tampering_mpl_materials.py b/test_vector_handlers/src/awses_test_vectors/internal/mpl/tampering_mpl_materials.py index 8b540f8dd..29a820ecc 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/mpl/tampering_mpl_materials.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/mpl/tampering_mpl_materials.py @@ -1,9 +1,9 @@ """Allows using ESDK-MPL interfaces with the tampering tests. These must ONLY be used in testing and NOT in production. """ +from copy import copy import attr import six -from copy import copy from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager @@ -90,8 +90,9 @@ class HalfSigningEncryptionMaterialsFromMPL(EncryptionMaterialsFromMPL): _underlying_materials: EncryptionMaterialsFromMPL def __init__(self, underlying_materials): - """Creates a HalfSigningEncryptionMaterialsFromMPL wrapper - around underlying_materials.""" + """Create a HalfSigningEncryptionMaterialsFromMPL wrapper + around underlying_materials. + """ self._underlying_materials = underlying_materials # pylint thinks EncryptionMaterialsFromMPL.algorithm is a method From c66938f22081d483b790c2fa00137f0b3759803e Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 16:47:33 -0700 Subject: [PATCH 367/376] cleanup --- codebuild/py311/mplawses_local_mpl.yml | 26 -------------------------- codebuild/py312/mplawses_local_mpl.yml | 26 -------------------------- codebuild/py39/awses_local.yml | 25 ------------------------- 3 files changed, 77 deletions(-) delete mode 100644 codebuild/py311/mplawses_local_mpl.yml delete mode 100644 codebuild/py312/mplawses_local_mpl.yml delete mode 100644 codebuild/py39/awses_local.yml diff --git a/codebuild/py311/mplawses_local_mpl.yml b/codebuild/py311/mplawses_local_mpl.yml deleted file mode 100644 index 92dbdb086..000000000 --- a/codebuild/py311/mplawses_local_mpl.yml +++ /dev/null @@ -1,26 +0,0 @@ -version: 0.2 - -env: - variables: - TOXENV: "py311-mplvectors-mpl" - REGION: "us-west-2" - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- - arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- - arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- - arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- - arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" - -phases: - install: - runtime-versions: - python: 3.11 - build: - commands: - - pip install "tox < 4.0" - - cd test_vector_handlers - - tox diff --git a/codebuild/py312/mplawses_local_mpl.yml b/codebuild/py312/mplawses_local_mpl.yml deleted file mode 100644 index 8a7d5f5c6..000000000 --- a/codebuild/py312/mplawses_local_mpl.yml +++ /dev/null @@ -1,26 +0,0 @@ -version: 0.2 - -env: - variables: - TOXENV: "py312-mplvectors-mpl" - REGION: "us-west-2" - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- - arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- - arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- - arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- - arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" - -phases: - install: - runtime-versions: - python: 3.12 - build: - commands: - - pip install "tox < 4.0" - - cd test_vector_handlers - - tox diff --git a/codebuild/py39/awses_local.yml b/codebuild/py39/awses_local.yml deleted file mode 100644 index e56a9ff45..000000000 --- a/codebuild/py39/awses_local.yml +++ /dev/null @@ -1,25 +0,0 @@ -version: 0.2 - -env: - variables: - TOXENV: "py39-awses_local" - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- - arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- - arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- - arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- - arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" - -phases: - install: - runtime-versions: - python: 3.9 - build: - commands: - - pip install "tox < 4.0" - - cd test_vector_handlers - - tox From 30ed6fa401df7b3d221e2db7825af703cd511343 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 1 May 2024 16:51:15 -0700 Subject: [PATCH 368/376] fix: Try all master key providers when decrypting raw RSA data key --- src/aws_encryption_sdk/internal/crypto/wrapping_keys.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/aws_encryption_sdk/internal/crypto/wrapping_keys.py b/src/aws_encryption_sdk/internal/crypto/wrapping_keys.py index 91f9fd834..ba6135965 100644 --- a/src/aws_encryption_sdk/internal/crypto/wrapping_keys.py +++ b/src/aws_encryption_sdk/internal/crypto/wrapping_keys.py @@ -98,9 +98,12 @@ def decrypt(self, encrypted_wrapped_data_key, encryption_context): if self.wrapping_key_type is EncryptionKeyType.PUBLIC: raise IncorrectMasterKeyError("Public key cannot decrypt") if self.wrapping_key_type is EncryptionKeyType.PRIVATE: - return self._wrapping_key.decrypt( - ciphertext=encrypted_wrapped_data_key.ciphertext, padding=self.wrapping_algorithm.padding - ) + try: + return self._wrapping_key.decrypt( + ciphertext=encrypted_wrapped_data_key.ciphertext, padding=self.wrapping_algorithm.padding + ) + except ValueError as e: + raise IncorrectMasterKeyError("_wrapping_key cannot decrypt provided ciphertext") serialized_encryption_context = serialize_encryption_context(encryption_context=encryption_context) return decrypt( algorithm=self.wrapping_algorithm.algorithm, From a30bceb0130e9afbfbc2aaa8e29a192d7e2199fc Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 3 May 2024 14:24:07 -0700 Subject: [PATCH 369/376] resovle merge --- .../materials_managers/__init__.py | 5 -- .../materials_managers/mpl/cmm.py | 39 ---------- .../unit/test_material_managers_mpl_cmm.py | 71 ------------------- .../test_material_managers_mpl_materials.py | 6 -- test/unit/test_streaming_client_configs.py | 43 ----------- .../test_streaming_client_stream_encryptor.py | 3 - 6 files changed, 167 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/__init__.py b/src/aws_encryption_sdk/materials_managers/__init__.py index 0254381be..950dd87cd 100644 --- a/src/aws_encryption_sdk/materials_managers/__init__.py +++ b/src/aws_encryption_sdk/materials_managers/__init__.py @@ -90,12 +90,7 @@ class DecryptionMaterialsRequest(object): :type encrypted_data_keys: set of `aws_encryption_sdk.structures.EncryptedDataKey` :param dict encryption_context: Encryption context to provide to master keys for underlying decrypt requests :param dict reproduced_encryption_context: Encryption context to provide on decrypt. -<<<<<<< HEAD - This is ONLY processed if using the required encryption context CMM from the - aws-cryptographic-materialproviders library. -======= This is ONLY processed if using a CMM from the aws-cryptographic-materialproviders library. ->>>>>>> mpl-reviewed """ algorithm = attr.ib(validator=attr.validators.instance_of(Algorithm)) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index f3bc60853..49a743f89 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -7,14 +7,10 @@ # pylint should pass even if the MPL isn't installed # Also thinks these imports aren't used if it can't import them # noqa pylint: disable=import-error,unused-import -<<<<<<< HEAD -from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException -======= from aws_cryptographic_materialproviders.mpl.errors import ( AwsCryptographicMaterialProvidersException, CollectionOfErrors, ) ->>>>>>> mpl-reviewed from aws_cryptographic_materialproviders.mpl.models import ( AlgorithmSuiteIdESDK as MPL_AlgorithmSuiteIdESDK, CommitmentPolicyESDK as MPL_CommitmentPolicyESDK, @@ -43,11 +39,7 @@ class CryptoMaterialsManagerFromMPL(CryptoMaterialsManager): """ In instances where encryption materials are provided by an implementation of the MPL's `aws_cryptographic_materialproviders.mpl.references.MPL_ICryptographicMaterialsManager`, -<<<<<<< HEAD - this maps the ESDK CMM interfaces to the MPL CMM. -======= this maps the ESDK-Python CMM interfaces to the MPL CMM. ->>>>>>> mpl-reviewed """ mpl_cmm: 'MPL_ICryptographicMaterialsManager' @@ -89,18 +81,6 @@ def get_encryption_materials( def _native_to_mpl_get_encryption_materials( request: EncryptionMaterialsRequest ) -> 'MPL_GetEncryptionMaterialsInput': -<<<<<<< HEAD - commitment_policy = CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy( - request.commitment_policy - ) - mpl_input_kwargs = { - "encryption_context": request.encryption_context, - "commitment_policy": commitment_policy, - "max_plaintext_length": request.plaintext_length, - } - if request.algorithm is not None: - mpl_input_kwargs["algorithm_suite_id"] = \ -======= output_kwargs = { "encryption_context": request.encryption_context, "max_plaintext_length": request.plaintext_length, @@ -111,25 +91,14 @@ def _native_to_mpl_get_encryption_materials( if request.algorithm is not None: output_kwargs["algorithm_suite_id"] = \ ->>>>>>> mpl-reviewed CryptoMaterialsManagerFromMPL._native_algorithm_id_to_mpl_algorithm_id( request.algorithm.algorithm_id ) -<<<<<<< HEAD - output: MPL_GetEncryptionMaterialsInput = MPL_GetEncryptionMaterialsInput( - **mpl_input_kwargs - ) - return output - - @staticmethod - def _native_to_mpl_commmitment_policy( -======= return MPL_GetEncryptionMaterialsInput(**output_kwargs) @staticmethod def _native_to_mpl_commitment_policy( ->>>>>>> mpl-reviewed native_commitment_policy: CommitmentPolicy ) -> 'MPL_CommitmentPolicyESDK': if native_commitment_policy == CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT: @@ -154,11 +123,7 @@ def decrypt_materials( CryptoMaterialsManagerFromMPL._create_mpl_decrypt_materials_input_from_request(request) mpl_output: 'MPL_DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(mpl_input) return DecryptionMaterialsFromMPL(mpl_output.decryption_materials) -<<<<<<< HEAD - except AwsCryptographicMaterialProvidersException as mpl_exception: -======= except (AwsCryptographicMaterialProvidersException, CollectionOfErrors) as mpl_exception: ->>>>>>> mpl-reviewed # Wrap MPL error into the ESDK error type # so customers only have to catch ESDK error types. raise AWSEncryptionSDKClientError(mpl_exception) @@ -182,11 +147,7 @@ def _create_mpl_decrypt_materials_input_from_request( algorithm_suite_id=CryptoMaterialsManagerFromMPL._native_algorithm_id_to_mpl_algorithm_id( request.algorithm.algorithm_id ), -<<<<<<< HEAD - commitment_policy=CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy( -======= commitment_policy=CryptoMaterialsManagerFromMPL._native_to_mpl_commitment_policy( ->>>>>>> mpl-reviewed request.commitment_policy ), encrypted_data_keys=list_edks, diff --git a/test/mpl/unit/test_material_managers_mpl_cmm.py b/test/mpl/unit/test_material_managers_mpl_cmm.py index 92f3ba656..603446550 100644 --- a/test/mpl/unit/test_material_managers_mpl_cmm.py +++ b/test/mpl/unit/test_material_managers_mpl_cmm.py @@ -38,10 +38,7 @@ mock_mpl_cmm = MagicMock(__class__=MPL_ICryptographicMaterialsManager) mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) -<<<<<<< HEAD -======= mock_reproduced_encryption_context = MagicMock(__class_=dict) ->>>>>>> mpl-reviewed mock_edk = MagicMock(__class__=Native_EncryptedDataKey) @@ -100,12 +97,6 @@ def test_GIVEN_valid_request_WHEN_get_encryption_materials_THEN_return_Encryptio @patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL" -<<<<<<< HEAD - "._native_to_mpl_get_encryption_materials") -def test_GIVEN_mpl_cmm_raises_MPLException_WHEN_get_encryption_materials_THEN_raise_ESDKException( - _ -): -======= "._native_algorithm_id_to_mpl_algorithm_id") @patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL" "._native_to_mpl_commitment_policy") @@ -117,7 +108,6 @@ def test_GIVEN_mpl_cmm_raises_MPLException_WHEN_get_encryption_materials_THEN_ra mock_algorithm_id = "0x1234" # Some fake algorithm ID that fits the format mock_mpl_algorithm_id.return_value = mock_algorithm_id ->>>>>>> mpl-reviewed # Then: Raises AWSEncryptionSDKClientError with pytest.raises(AWSEncryptionSDKClientError): # Given: mpl_cmm.get_encryption_materials raises MPL exception @@ -131,13 +121,6 @@ def test_GIVEN_mpl_cmm_raises_MPLException_WHEN_get_encryption_materials_THEN_ra @patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL" "._native_algorithm_id_to_mpl_algorithm_id") @patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL" -<<<<<<< HEAD - "._native_to_mpl_commmitment_policy") -def test_GIVEN_valid_mpl_commitment_policy_WHEN_native_to_mpl_get_encryption_materials_THEN_returns_MPL_GetEncryptionMaterialsInput( # noqa: E501 - mock_mpl_commitment_policy, - mock_mpl_algorithm, -): -======= "._native_to_mpl_commitment_policy") def test_GIVEN_valid_mpl_commitment_policy_WHEN_native_to_mpl_get_encryption_materials_THEN_returns_MPL_GetEncryptionMaterialsInput( # noqa: E501 mock_mpl_commitment_policy, @@ -147,7 +130,6 @@ def test_GIVEN_valid_mpl_commitment_policy_WHEN_native_to_mpl_get_encryption_mat mock_algorithm_id = "0x1234" # Some fake algorithm ID that fits the format mock_mpl_algorithm_id.return_value = mock_algorithm_id ->>>>>>> mpl-reviewed # Given: commitment policy is some MPL ESDK commitment policy mock_commitment_policy = MagicMock(__class__=MPL_CommitmentPolicyESDK) mock_mpl_commitment_policy.return_value = mock_commitment_policy @@ -162,17 +144,6 @@ def test_GIVEN_valid_mpl_commitment_policy_WHEN_native_to_mpl_get_encryption_mat assert output.encryption_context == mock_encryption_materials_request.encryption_context assert output.commitment_policy == mock_commitment_policy assert output.max_plaintext_length == mock_encryption_materials_request.plaintext_length -<<<<<<< HEAD - assert output.algorithm_suite_id == mock_mpl_algorithm() - - -def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_MPL_CommitmentPolicyESDK_FORBID_ENCRYPT_ALLOW_DECRYPT(): # noqa: E501 - # Given: native FORBID_ENCRYPT_ALLOW_DECRYPT - native_commitment_policy = CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT - - # When: _native_to_mpl_commmitment_policy - output = CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy(native_commitment_policy) -======= def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_commitment_policy_THEN_returns_MPL_CommitmentPolicyESDK_FORBID_ENCRYPT_ALLOW_DECRYPT(): # noqa: E501 @@ -181,72 +152,44 @@ def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_ # When: _native_to_mpl_commitment_policy output = CryptoMaterialsManagerFromMPL._native_to_mpl_commitment_policy(native_commitment_policy) ->>>>>>> mpl-reviewed # Then: Returns MPL FORBID_ENCRYPT_ALLOW_DECRYPT assert isinstance(output, MPL_CommitmentPolicyESDK) assert output.value == "FORBID_ENCRYPT_ALLOW_DECRYPT" -<<<<<<< HEAD -def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_MPL_CommitmentPolicyESDK_REQUIRE_ENCRYPT_ALLOW_DECRYPT(): # noqa: E501 - # Given: native REQUIRE_ENCRYPT_ALLOW_DECRYPT - native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT - - # When: _native_to_mpl_commmitment_policy - output = CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy(native_commitment_policy) -======= def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_commitment_policy_THEN_returns_MPL_CommitmentPolicyESDK_REQUIRE_ENCRYPT_ALLOW_DECRYPT(): # noqa: E501 # Given: native REQUIRE_ENCRYPT_ALLOW_DECRYPT native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT # When: _native_to_mpl_commitment_policy output = CryptoMaterialsManagerFromMPL._native_to_mpl_commitment_policy(native_commitment_policy) ->>>>>>> mpl-reviewed # Then: Returns MPL REQUIRE_ENCRYPT_ALLOW_DECRYPT assert isinstance(output, MPL_CommitmentPolicyESDK) assert output.value == "REQUIRE_ENCRYPT_ALLOW_DECRYPT" -<<<<<<< HEAD -def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_REQUIRE_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_MPL_CommitmentPolicyESDK_REQUIRE_ENCRYPT_REQUIRE_DECRYPT(): # noqa: E501 - # Given: native REQUIRE_ENCRYPT_REQUIRE_DECRYPT - native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT - - # When: _native_to_mpl_commmitment_policy - output = CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy(native_commitment_policy) -======= def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_REQUIRE_DECRYPT_WHEN_native_to_mpl_commitment_policy_THEN_returns_MPL_CommitmentPolicyESDK_REQUIRE_ENCRYPT_REQUIRE_DECRYPT(): # noqa: E501 # Given: native REQUIRE_ENCRYPT_REQUIRE_DECRYPT native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT # When: _native_to_mpl_commitment_policy output = CryptoMaterialsManagerFromMPL._native_to_mpl_commitment_policy(native_commitment_policy) ->>>>>>> mpl-reviewed # Then: Returns MPL REQUIRE_ENCRYPT_REQUIRE_DECRYPT assert isinstance(output, MPL_CommitmentPolicyESDK) assert output.value == "REQUIRE_ENCRYPT_REQUIRE_DECRYPT" -<<<<<<< HEAD -def test_GIVEN_CommitmentPolicy_unrecognized_WHEN_native_to_mpl_commmitment_policy_THEN_raise_ValueError(): -======= def test_GIVEN_CommitmentPolicy_unrecognized_WHEN_native_to_mpl_commitment_policy_THEN_raise_ValueError(): ->>>>>>> mpl-reviewed # Given: invalid native commitment policy native_commitment_policy = "not a commitment policy" # Then: Raises ValueError with pytest.raises(ValueError): -<<<<<<< HEAD - # When: _native_to_mpl_commmitment_policy - CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy(native_commitment_policy) -======= # When: _native_to_mpl_commitment_policy CryptoMaterialsManagerFromMPL._native_to_mpl_commitment_policy(native_commitment_policy) ->>>>>>> mpl-reviewed @patch.object(mock_mpl_cmm, "decrypt_materials") @@ -310,11 +253,7 @@ def test_GIVEN_valid_native_algorithm_id_WHEN_native_algorithm_id_to_mpl_algorit @patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL" "._native_algorithm_id_to_mpl_algorithm_id") @patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL" -<<<<<<< HEAD - "._native_to_mpl_commmitment_policy") -======= "._native_to_mpl_commitment_policy") ->>>>>>> mpl-reviewed def test_GIVEN_valid_request_WHEN_create_mpl_decrypt_materials_input_from_request_THEN_returns_MPL_MPL_DecryptMaterialsInput( # noqa: E501 mock_mpl_commitment_policy, mock_mpl_algorithm_id, @@ -323,11 +262,7 @@ def test_GIVEN_valid_request_WHEN_create_mpl_decrypt_materials_input_from_reques mock_algorithm_id = "0x1234" # Some fake algorithm ID that fits the format mock_mpl_algorithm_id.return_value = mock_algorithm_id -<<<<<<< HEAD - # Given: _native_to_mpl_commmitment_policy returns some MPL commitment policy -======= # Given: _native_to_mpl_commitment_policy returns some MPL commitment policy ->>>>>>> mpl-reviewed mock_commitment_policy = MagicMock(__class__=MPL_CommitmentPolicyESDK) mock_mpl_commitment_policy.return_value = mock_commitment_policy @@ -339,10 +274,7 @@ def test_GIVEN_valid_request_WHEN_create_mpl_decrypt_materials_input_from_reques for mock_edks in [no_mock_edks, one_mock_edk, two_mock_edks]: mock_decryption_materials_request.encrypted_data_keys = mock_edks -<<<<<<< HEAD -======= mock_decryption_materials_request.reproduced_encryption_context = mock_reproduced_encryption_context ->>>>>>> mpl-reviewed # When: _create_mpl_decrypt_materials_input_from_request output = CryptoMaterialsManagerFromMPL._create_mpl_decrypt_materials_input_from_request( @@ -355,10 +287,7 @@ def test_GIVEN_valid_request_WHEN_create_mpl_decrypt_materials_input_from_reques assert output.algorithm_suite_id == mock_algorithm_id assert output.commitment_policy == mock_commitment_policy assert output.encryption_context == mock_decryption_materials_request.encryption_context -<<<<<<< HEAD -======= assert output.reproduced_encryption_context == mock_reproduced_encryption_context ->>>>>>> mpl-reviewed assert len(output.encrypted_data_keys) == len(mock_edks) for i in range(len(output.encrypted_data_keys)): diff --git a/test/mpl/unit/test_material_managers_mpl_materials.py b/test/mpl/unit/test_material_managers_mpl_materials.py index 0c31bff17..8d9052c0a 100644 --- a/test/mpl/unit/test_material_managers_mpl_materials.py +++ b/test/mpl/unit/test_material_managers_mpl_materials.py @@ -160,8 +160,6 @@ def test_GIVEN_valid_signing_key_WHEN_EncryptionMaterials_get_signing_key_THEN_r assert output == mock_signing_key -<<<<<<< HEAD -======= def test_GIVEN_valid_required_encryption_context_keys_WHEN_EncryptionMaterials_get_required_encryption_context_keys_THEN_returns_required_encryption_context_keys(): # noqa pylint: disable=line-too-long # Given: valid required encryption context keys mock_required_encryption_context_keys = MagicMock(__class__=bytes) @@ -175,7 +173,6 @@ def test_GIVEN_valid_required_encryption_context_keys_WHEN_EncryptionMaterials_g assert output == mock_required_encryption_context_keys ->>>>>>> mpl-reviewed def test_GIVEN_valid_data_key_WHEN_DecryptionMaterials_get_data_key_THEN_returns_data_key(): # Given: valid MPL data key mock_data_key = MagicMock(__class__=bytes) @@ -203,8 +200,6 @@ def test_GIVEN_valid_verification_key_WHEN_DecryptionMaterials_get_verification_ # Then: returns verification key assert output == mock_verification_key -<<<<<<< HEAD -======= def test_GIVEN_valid_encryption_context_WHEN_DecryptionMaterials_get_encryption_context_THEN_returns_encryption_context(): # noqa pylint: disable=line-too-long @@ -231,4 +226,3 @@ def test_GIVEN_valid_required_encryption_context_keys_WHEN_DecryptionMaterials_g # Then: returns required encryption context keys assert output == mock_required_encryption_context_keys ->>>>>>> mpl-reviewed diff --git a/test/unit/test_streaming_client_configs.py b/test/unit/test_streaming_client_configs.py index 0521139aa..435aff0da 100644 --- a/test/unit/test_streaming_client_configs.py +++ b/test/unit/test_streaming_client_configs.py @@ -15,11 +15,7 @@ import pytest import six -<<<<<<< HEAD -from mock import patch -======= from mock import MagicMock, patch ->>>>>>> mpl-reviewed from aws_encryption_sdk import CommitmentPolicy from aws_encryption_sdk.internal.defaults import ALGORITHM, FRAME_LENGTH, LINE_LENGTH @@ -37,11 +33,7 @@ # Ideally, this logic would be based on mocking imports and testing logic, # but doing that introduces errors that cause other tests to fail. try: -<<<<<<< HEAD - from aws_cryptographic_materialproviders.mpl.references import IKeyring -======= from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager, IKeyring ->>>>>>> mpl-reviewed HAS_MPL = True from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL @@ -244,13 +236,6 @@ def test_client_configs_with_mpl( assert test.materials_manager is not None # If materials manager was provided, it should be directly used -<<<<<<< HEAD - if hasattr(kwargs, "materials_manager"): - assert kwargs["materials_manager"] == test.materials_manager - - # If MPL keyring was provided, it should be wrapped in MPL materials manager - if hasattr(kwargs, "keyring"): -======= if "materials_manager" in kwargs: assert kwargs["materials_manager"] == test.materials_manager @@ -262,29 +247,17 @@ def test_client_configs_with_mpl( # If MPL keyring was provided, it should be wrapped in MPL materials manager elif "keyring" in kwargs: ->>>>>>> mpl-reviewed assert test.keyring is not None assert test.keyring == kwargs["keyring"] assert isinstance(test.keyring, IKeyring) assert isinstance(test.materials_manager, CryptoMaterialsManagerFromMPL) -<<<<<<< HEAD - # If native key_provider was provided, it should be wrapped in native materials manager - if hasattr(kwargs, "key_provider"): - assert test.key_provider is not None - assert test.key_provider == kwargs["key_provider"] - assert isinstance(test.materials_manager, DefaultCryptoMaterialsManager) - - -# This needs its own test; pytest parametrize cannot use a conditionally-loaded type -======= else: raise ValueError(f"Test did not find materials_manager or key_provider. {kwargs}") # This is an addition to test_client_configs_with_mpl; # This needs its own test; pytest's parametrize cannot use a conditionally-loaded type (IKeyring) ->>>>>>> mpl-reviewed @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") def test_keyring_client_config_with_mpl( ): @@ -296,21 +269,6 @@ def test_keyring_client_config_with_mpl( test = _ClientConfig(**kwargs) -<<<<<<< HEAD - # In all cases, config should have a materials manager - assert test.materials_manager is not None - - # If materials manager was provided, it should be directly used - if hasattr(kwargs, "materials_manager"): - assert kwargs["materials_manager"] == test.materials_manager - - # If MPL keyring was provided, it should be wrapped in MPL materials manager - if hasattr(kwargs, "keyring"): - assert test.keyring is not None - assert test.keyring == kwargs["keyring"] - assert isinstance(test.keyring, IKeyring) - assert isinstance(test.materials_manager, CryptoMaterialsManagerFromMPL) -======= assert test.materials_manager is not None assert test.keyring is not None @@ -338,4 +296,3 @@ def test_mpl_cmm_client_config_with_mpl( assert isinstance(test.materials_manager, CryptoMaterialsManagerFromMPL) # Assert the MPL CMM is used by the native interface assert test.materials_manager.mpl_cmm == mock_mpl_cmm ->>>>>>> mpl-reviewed diff --git a/test/unit/test_streaming_client_stream_encryptor.py b/test/unit/test_streaming_client_stream_encryptor.py index f56ec35a2..4df79e146 100644 --- a/test/unit/test_streaming_client_stream_encryptor.py +++ b/test/unit/test_streaming_client_stream_encryptor.py @@ -452,8 +452,6 @@ def test_GIVEN_has_mpl_AND_has_MPLCMM_AND_uses_signer_WHEN_prep_message_THEN_sig encoding=serialization.Encoding.PEM ) -<<<<<<< HEAD -======= # Given: has MPL @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") def test_GIVEN_has_mpl_AND_encryption_materials_has_required_EC_keys_WHEN_prep_message_THEN_paritions_stored_and_required_EC( # noqa pylint: disable=line-too-long @@ -567,7 +565,6 @@ def test_GIVEN_has_mpl_AND_encryption_materials_does_not_have_required_EC_keys_W # Then: _required_encryption_context is None assert test_encryptor._required_encryption_context is None ->>>>>>> mpl-reviewed def test_prep_message_no_signer(self): self.mock_encryption_materials.algorithm = Algorithm.AES_128_GCM_IV12_TAG16 test_encryptor = StreamEncryptor( From 5393825d7a0a269b4ee61c31bd5977e12b166768 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 6 May 2024 13:28:58 -0700 Subject: [PATCH 370/376] cleanup --- test/unit/test_streaming_client_stream_decryptor.py | 4 ---- test_vector_handlers/tox.ini | 4 ---- 2 files changed, 8 deletions(-) diff --git a/test/unit/test_streaming_client_stream_decryptor.py b/test/unit/test_streaming_client_stream_decryptor.py index a291cab54..be9304006 100644 --- a/test/unit/test_streaming_client_stream_decryptor.py +++ b/test/unit/test_streaming_client_stream_decryptor.py @@ -195,12 +195,8 @@ def test_read_header(self, mock_derive_datakey, mock_decrypt_materials_request, test_decryptor._stream_length = len(VALUES["data_128"]) # Mock: hasattr(self.config, "encryption_context") returns False -<<<<<<< HEAD - del test_decryptor.config.encryption_context -======= if hasattr(test_decryptor.config, "encryption_context"): del test_decryptor.config.encryption_context ->>>>>>> mpl-reviewed test_header, test_header_auth = test_decryptor._read_header() diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index 75c9c7a76..ed9ad993e 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -50,11 +50,7 @@ sitepackages = False deps = -rtest/requirements.txt # Install the MPL requirements if the `-mpl` suffix is present -<<<<<<< HEAD - mpl: -rrequirements_mpl.txt -======= mpl: -r../requirements_mpl.txt ->>>>>>> mpl-reviewed .. commands = awses_local: {[testenv:base-command]commands} From 709fb3ae46ed22af340c604aab84b0ce69d1214d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 6 May 2024 14:06:31 -0700 Subject: [PATCH 371/376] cleanup --- .../test_streaming_client_stream_decryptor.py | 39 ------------------- 1 file changed, 39 deletions(-) diff --git a/test/unit/test_streaming_client_stream_decryptor.py b/test/unit/test_streaming_client_stream_decryptor.py index be9304006..83ce22c29 100644 --- a/test/unit/test_streaming_client_stream_decryptor.py +++ b/test/unit/test_streaming_client_stream_decryptor.py @@ -365,45 +365,6 @@ def test_GIVEN_verification_key_AND_has_mpl_AND_has_MPLCMM_WHEN_read_header_THEN algorithm=self.mock_header.algorithm, encoded_point=mock_b64encoding() ) - @patch("aws_encryption_sdk.streaming_client.derive_data_encryption_key") - @patch("aws_encryption_sdk.streaming_client.DecryptionMaterialsRequest") - @patch("aws_encryption_sdk.streaming_client.Verifier") - # Given: no MPL - @pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") - def test_GIVEN_decrypt_config_has_ec_WHEN_read_header_THEN_calls_decrypt_materials_with_reproduced_ec( - self, - mock_verifier, - mock_decrypt_materials_request, - *_, - ): - - mock_verifier_instance = MagicMock() - mock_verifier.from_key_bytes.return_value = mock_verifier_instance - ct_stream = io.BytesIO(VALUES["data_128"]) - mock_commitment_policy = MagicMock(__class__=CommitmentPolicy) - test_decryptor = StreamDecryptor( - materials_manager=self.mock_materials_manager, - source=ct_stream, - commitment_policy=mock_commitment_policy, - ) - test_decryptor.source_stream = ct_stream - test_decryptor._stream_length = len(VALUES["data_128"]) - # Given: self.config has "encryption_context" - any_reproduced_ec = {"some": "ec"} - test_decryptor.config.encryption_context = any_reproduced_ec - - # When: read header - test_decryptor._read_header() - - # Then: calls decrypt_materials with reproduced_encryption_context - mock_decrypt_materials_request.assert_called_once_with( - encrypted_data_keys=self.mock_header.encrypted_data_keys, - algorithm=self.mock_header.algorithm, - encryption_context=sentinel.encryption_context, - commitment_policy=mock_commitment_policy, - reproduced_encryption_context=any_reproduced_ec, - ) - @patch("aws_encryption_sdk.streaming_client.DecryptionMaterialsRequest") @patch("aws_encryption_sdk.streaming_client.derive_data_encryption_key") @patch("aws_encryption_sdk.streaming_client.Verifier") From d64dc81796519189f625f26f1d633f75a00e3ac7 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 6 May 2024 14:08:53 -0700 Subject: [PATCH 372/376] cleanup --- .../test_streaming_client_stream_decryptor.py | 97 ------------------- 1 file changed, 97 deletions(-) diff --git a/test/unit/test_streaming_client_stream_decryptor.py b/test/unit/test_streaming_client_stream_decryptor.py index 83ce22c29..ce3d6ee3c 100644 --- a/test/unit/test_streaming_client_stream_decryptor.py +++ b/test/unit/test_streaming_client_stream_decryptor.py @@ -365,103 +365,6 @@ def test_GIVEN_verification_key_AND_has_mpl_AND_has_MPLCMM_WHEN_read_header_THEN algorithm=self.mock_header.algorithm, encoded_point=mock_b64encoding() ) - @patch("aws_encryption_sdk.streaming_client.DecryptionMaterialsRequest") - @patch("aws_encryption_sdk.streaming_client.derive_data_encryption_key") - @patch("aws_encryption_sdk.streaming_client.Verifier") - # Given: no MPL - @pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") - def test_GIVEN_verification_key_AND_no_mpl_WHEN_read_header_THEN_calls_from_key_bytes( - self, - mock_verifier, - *_, - ): - # Given: verification key - mock_verifier_instance = MagicMock() - mock_verifier.from_key_bytes.return_value = mock_verifier_instance - ct_stream = io.BytesIO(VALUES["data_128"]) - mock_commitment_policy = MagicMock(__class__=CommitmentPolicy) - test_decryptor = StreamDecryptor( - materials_manager=self.mock_materials_manager, - source=ct_stream, - commitment_policy=mock_commitment_policy, - ) - test_decryptor.source_stream = ct_stream - test_decryptor._stream_length = len(VALUES["data_128"]) - - # When: read header - test_decryptor._read_header() - - # Then: calls from_key_bytes - mock_verifier.from_key_bytes.assert_called_once_with( - algorithm=self.mock_header.algorithm, key_bytes=sentinel.verification_key - ) - - @patch("aws_encryption_sdk.streaming_client.DecryptionMaterialsRequest") - @patch("aws_encryption_sdk.streaming_client.derive_data_encryption_key") - @patch("aws_encryption_sdk.streaming_client.Verifier") - # Given: has MPL - @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") - def test_GIVEN_verification_key_AND_has_mpl_AND_not_MPLCMM_WHEN_read_header_THEN_calls_from_key_bytes( - self, - mock_verifier, - *_, - ): - # Given: verification key - mock_verifier_instance = MagicMock() - mock_verifier.from_key_bytes.return_value = mock_verifier_instance - ct_stream = io.BytesIO(VALUES["data_128"]) - mock_commitment_policy = MagicMock(__class__=CommitmentPolicy) - test_decryptor = StreamDecryptor( - # Given: native CMM - materials_manager=self.mock_materials_manager, - source=ct_stream, - commitment_policy=mock_commitment_policy, - ) - test_decryptor.source_stream = ct_stream - test_decryptor._stream_length = len(VALUES["data_128"]) - - # When: read_header - test_decryptor._read_header() - - # Then: calls from_key_bytess - mock_verifier.from_key_bytes.assert_called_once_with( - algorithm=self.mock_header.algorithm, key_bytes=sentinel.verification_key - ) - - @patch("aws_encryption_sdk.streaming_client.DecryptionMaterialsRequest") - @patch("aws_encryption_sdk.streaming_client.derive_data_encryption_key") - @patch("aws_encryption_sdk.streaming_client.Verifier") - @patch("base64.b64encode") - # Given: has MPL - @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") - def test_GIVEN_verification_key_AND_has_mpl_AND_has_MPLCMM_WHEN_read_header_THEN_calls_from_encoded_point( - self, - mock_b64encoding, - mock_verifier, - *_, - ): - # Given: Verification key - mock_verifier_instance = MagicMock() - mock_verifier.from_key_bytes.return_value = mock_verifier_instance - ct_stream = io.BytesIO(VALUES["data_128"]) - mock_commitment_policy = MagicMock(__class__=CommitmentPolicy) - test_decryptor = StreamDecryptor( - # Given: MPL CMM - materials_manager=self.mock_mpl_materials_manager, - source=ct_stream, - commitment_policy=mock_commitment_policy, - ) - test_decryptor.source_stream = ct_stream - test_decryptor._stream_length = len(VALUES["data_128"]) - - # When: read header - test_decryptor._read_header() - - # Then: calls from_encoded_point - mock_verifier.from_encoded_point.assert_called_once_with( - algorithm=self.mock_header.algorithm, encoded_point=mock_b64encoding() - ) - @patch("aws_encryption_sdk.streaming_client.derive_data_encryption_key") def test_read_header_frame_too_large(self, mock_derive_datakey): self.mock_header.content_type = ContentType.FRAMED_DATA From c953b215494162055bdbd40f3d1df4f3fa948f24 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 6 May 2024 14:12:36 -0700 Subject: [PATCH 373/376] fix --- src/aws_encryption_sdk/internal/crypto/wrapping_keys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/internal/crypto/wrapping_keys.py b/src/aws_encryption_sdk/internal/crypto/wrapping_keys.py index ba6135965..da9bc9b6b 100644 --- a/src/aws_encryption_sdk/internal/crypto/wrapping_keys.py +++ b/src/aws_encryption_sdk/internal/crypto/wrapping_keys.py @@ -102,7 +102,7 @@ def decrypt(self, encrypted_wrapped_data_key, encryption_context): return self._wrapping_key.decrypt( ciphertext=encrypted_wrapped_data_key.ciphertext, padding=self.wrapping_algorithm.padding ) - except ValueError as e: + except ValueError: raise IncorrectMasterKeyError("_wrapping_key cannot decrypt provided ciphertext") serialized_encryption_context = serialize_encryption_context(encryption_context=encryption_context) return decrypt( From c7c6a5648333c51bb075f0d347f4c3d60422613d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 6 May 2024 14:14:07 -0700 Subject: [PATCH 374/376] fix --- test_vector_handlers/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index ed9ad993e..cdb1137fb 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -50,7 +50,7 @@ sitepackages = False deps = -rtest/requirements.txt # Install the MPL requirements if the `-mpl` suffix is present - mpl: -r../requirements_mpl.txt + mpl: -rrequirements_mpl.txt .. commands = awses_local: {[testenv:base-command]commands} From 990e2b8a37f33474b3d74052bfbe152d1d0b389d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 6 May 2024 14:15:04 -0700 Subject: [PATCH 375/376] rv vectors --- .../test/aws-crypto-tools-test-vector-framework | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_vector_handlers/test/aws-crypto-tools-test-vector-framework b/test_vector_handlers/test/aws-crypto-tools-test-vector-framework index fc793e257..c3d73fae2 160000 --- a/test_vector_handlers/test/aws-crypto-tools-test-vector-framework +++ b/test_vector_handlers/test/aws-crypto-tools-test-vector-framework @@ -1 +1 @@ -Subproject commit fc793e257f4a58ae49b92f95a519ba2c31ccff12 +Subproject commit c3d73fae260fd9e9cc9e746f09a7ffbab83576e2 From 6979419a3f81ec5aecfa51ba98b214abc21d007f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 6 May 2024 14:18:05 -0700 Subject: [PATCH 376/376] fix --- .../test/aws-crypto-tools-test-vector-framework | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_vector_handlers/test/aws-crypto-tools-test-vector-framework b/test_vector_handlers/test/aws-crypto-tools-test-vector-framework index c3d73fae2..9eb2fcbbe 160000 --- a/test_vector_handlers/test/aws-crypto-tools-test-vector-framework +++ b/test_vector_handlers/test/aws-crypto-tools-test-vector-framework @@ -1 +1 @@ -Subproject commit c3d73fae260fd9e9cc9e746f09a7ffbab83576e2 +Subproject commit 9eb2fcbbe47ab30c29d6ad9a8125b1064e0db42a