From 441c8320e701411df4073a3de2444c4d8e4ed093 Mon Sep 17 00:00:00 2001 From: Adam Tankanow Date: Wed, 6 Oct 2021 08:06:54 -0400 Subject: [PATCH 1/4] ISSUE-694: add sort key to DynamoDBPersistenceLayer --- .../idempotency/persistence/dynamodb.py | 24 +- tests/functional/idempotency/conftest.py | 72 +++-- .../idempotency/test_idempotency.py | 304 +++++++++++------- 3 files changed, 253 insertions(+), 147 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py index 0ce307ab503..ab0640bd36d 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py @@ -20,6 +20,8 @@ def __init__( self, table_name: str, key_attr: str = "id", + key_attr_value: str = "powertools#idempotency", + sort_key_attr: Optional[str] = None, expiry_attr: str = "expiration", status_attr: str = "status", data_attr: str = "data", @@ -35,7 +37,12 @@ def __init__( table_name: str Name of the table to use for storing execution records key_attr: str, optional - DynamoDB attribute name for key, by default "id" + DynamoDB attribute name for partition key, by default "id" + key_attr_value: str, optional + DynamoDB attribute value for partition key, by default "powertools#idempotency". + This will be used if the sort_key_attr is set. + sort_key_attr: str, optional + DynamoDB attribute name for the sort key expiry_attr: str, optional DynamoDB attribute name for expiry timestamp, by default "expiration" status_attr: str, optional @@ -68,6 +75,8 @@ def __init__( self._table = None self.table_name = table_name self.key_attr = key_attr + self.key_attr_value = key_attr_value + self.sort_key_attr = sort_key_attr self.expiry_attr = expiry_attr self.status_attr = status_attr self.data_attr = data_attr @@ -93,6 +102,11 @@ def table(self, table): """ self._table = table + def _get_key(self, idempotency_key: str) -> dict: + if self.sort_key_attr: + return {self.key_attr: self.key_attr_value, self.sort_key_attr: idempotency_key} + return {self.key_attr: idempotency_key} + def _item_to_data_record(self, item: Dict[str, Any]) -> DataRecord: """ Translate raw item records from DynamoDB to DataRecord @@ -117,7 +131,7 @@ def _item_to_data_record(self, item: Dict[str, Any]) -> DataRecord: ) def _get_record(self, idempotency_key) -> DataRecord: - response = self.table.get_item(Key={self.key_attr: idempotency_key}, ConsistentRead=True) + response = self.table.get_item(Key=self._get_key(idempotency_key), ConsistentRead=True) try: item = response["Item"] @@ -127,7 +141,7 @@ def _get_record(self, idempotency_key) -> DataRecord: def _put_record(self, data_record: DataRecord) -> None: item = { - self.key_attr: data_record.idempotency_key, + **self._get_key(data_record.idempotency_key), self.expiry_attr: data_record.expiry_timestamp, self.status_attr: data_record.status, } @@ -168,7 +182,7 @@ def _update_record(self, data_record: DataRecord): expression_attr_names["#validation_key"] = self.validation_key_attr kwargs = { - "Key": {self.key_attr: data_record.idempotency_key}, + "Key": self._get_key(data_record.idempotency_key), "UpdateExpression": update_expression, "ExpressionAttributeValues": expression_attr_values, "ExpressionAttributeNames": expression_attr_names, @@ -178,4 +192,4 @@ def _update_record(self, data_record: DataRecord): def _delete_record(self, data_record: DataRecord) -> None: logger.debug(f"Deleting record for idempotency key: {data_record.idempotency_key}") - self.table.delete_item(Key={self.key_attr: data_record.idempotency_key}) + self.table.delete_item(Key=self._get_key(data_record.idempotency_key)) diff --git a/tests/functional/idempotency/conftest.py b/tests/functional/idempotency/conftest.py index 71b5978497c..0955f05584b 100644 --- a/tests/functional/idempotency/conftest.py +++ b/tests/functional/idempotency/conftest.py @@ -2,7 +2,9 @@ import hashlib import json from collections import namedtuple +from dataclasses import dataclass from decimal import Decimal +from typing import Callable from unittest import mock import jmespath @@ -80,25 +82,23 @@ def default_jmespath(): @pytest.fixture -def expected_params_update_item(serialized_lambda_response, hashed_idempotency_key): - return { +def expected_params_update_item(serialized_lambda_response): + return lambda key: { "ExpressionAttributeNames": {"#expiry": "expiration", "#response_data": "data", "#status": "status"}, "ExpressionAttributeValues": { ":expiry": stub.ANY, ":response_data": serialized_lambda_response, ":status": "COMPLETED", }, - "Key": {"id": hashed_idempotency_key}, + "Key": key, "TableName": "TEST_TABLE", "UpdateExpression": "SET #response_data = :response_data, " "#expiry = :expiry, #status = :status", } @pytest.fixture -def expected_params_update_item_with_validation( - serialized_lambda_response, hashed_idempotency_key, hashed_validation_key -): - return { +def expected_params_update_item_with_validation(serialized_lambda_response, hashed_validation_key): + return lambda key: { "ExpressionAttributeNames": { "#expiry": "expiration", "#response_data": "data", @@ -111,7 +111,7 @@ def expected_params_update_item_with_validation( ":status": "COMPLETED", ":validation_key": hashed_validation_key, }, - "Key": {"id": hashed_idempotency_key}, + "Key": key, "TableName": "TEST_TABLE", "UpdateExpression": "SET #response_data = :response_data, " "#expiry = :expiry, #status = :status, " @@ -120,28 +120,23 @@ def expected_params_update_item_with_validation( @pytest.fixture -def expected_params_put_item(hashed_idempotency_key): - return { +def expected_params_put_item(): + return lambda key_attr, key: { "ConditionExpression": "attribute_not_exists(#id) OR #now < :now", - "ExpressionAttributeNames": {"#id": "id", "#now": "expiration"}, + "ExpressionAttributeNames": {"#id": key_attr, "#now": "expiration"}, "ExpressionAttributeValues": {":now": stub.ANY}, - "Item": {"expiration": stub.ANY, "id": hashed_idempotency_key, "status": "INPROGRESS"}, + "Item": {"expiration": stub.ANY, "status": "INPROGRESS", **key}, "TableName": "TEST_TABLE", } @pytest.fixture -def expected_params_put_item_with_validation(hashed_idempotency_key, hashed_validation_key): - return { +def expected_params_put_item_with_validation(hashed_validation_key): + return lambda key_attr, key: { "ConditionExpression": "attribute_not_exists(#id) OR #now < :now", - "ExpressionAttributeNames": {"#id": "id", "#now": "expiration"}, + "ExpressionAttributeNames": {"#id": key_attr, "#now": "expiration"}, "ExpressionAttributeValues": {":now": stub.ANY}, - "Item": { - "expiration": stub.ANY, - "id": hashed_idempotency_key, - "status": "INPROGRESS", - "validation": hashed_validation_key, - }, + "Item": {"expiration": stub.ANY, "status": "INPROGRESS", "validation": hashed_validation_key, **key}, "TableName": "TEST_TABLE", } @@ -166,9 +161,40 @@ def hashed_validation_key(lambda_apigw_event): return hashlib.md5(serialize(lambda_apigw_event["requestContext"]).encode()).hexdigest() +@dataclass(eq=True, frozen=True) +class TestPersistenceStore: + persistence_layer: DynamoDBPersistenceLayer + key_attr: str + expected_key: Callable[[str], dict] + expected_key_values: Callable[[str], dict] + + +@pytest.fixture(params=[{}, {"key_attr": "PK", "key_attr_value": "powertools#idempotency", "sort_key_attr": "SK"}]) +def test_persistence_store(config, request) -> TestPersistenceStore: + expected_key = ( + lambda idempotency_key: {"PK": "powertools#idempotency", "SK": idempotency_key} + if request.param + else {"id": idempotency_key} + ) + expected_key_values = ( + lambda idempotency_key: {"PK": {"S": "powertools#idempotency"}, "SK": {"S": idempotency_key}} + if request.param + else {"id": {"S": idempotency_key}} + ) + return TestPersistenceStore( + persistence_layer=DynamoDBPersistenceLayer(table_name=TABLE_NAME, boto_config=config, **request.param), + key_attr="PK" if request.param else "id", + expected_key=expected_key, + expected_key_values=expected_key_values, + ) + + @pytest.fixture -def persistence_store(config): - return DynamoDBPersistenceLayer(table_name=TABLE_NAME, boto_config=config) +def persistence_store_with_composite_key(config): + return DynamoDBPersistenceLayer( + table_name=TABLE_NAME, + boto_config=config, + ) @pytest.fixture diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/test_idempotency.py index b1d0914d181..a3dd00595bc 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/test_idempotency.py @@ -10,7 +10,7 @@ from botocore import stub from aws_lambda_powertools.utilities.data_classes import APIGatewayProxyEventV2, event_source -from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer, IdempotencyConfig +from aws_lambda_powertools.utilities.idempotency import IdempotencyConfig from aws_lambda_powertools.utilities.idempotency.exceptions import ( IdempotencyAlreadyInProgressError, IdempotencyInconsistentStateError, @@ -22,7 +22,7 @@ from aws_lambda_powertools.utilities.idempotency.idempotency import idempotent, idempotent_function from aws_lambda_powertools.utilities.idempotency.persistence.base import BasePersistenceLayer, DataRecord from aws_lambda_powertools.utilities.validation import envelopes, validator -from tests.functional.idempotency.conftest import serialize +from tests.functional.idempotency.conftest import TestPersistenceStore, serialize from tests.functional.utils import load_event TABLE_NAME = "TEST_TABLE" @@ -33,7 +33,7 @@ @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_already_completed( idempotency_config: IdempotencyConfig, - persistence_store: DynamoDBPersistenceLayer, + test_persistence_store: TestPersistenceStore, lambda_apigw_event, timestamp_future, hashed_idempotency_key, @@ -45,10 +45,10 @@ def test_idempotent_lambda_already_completed( Test idempotent decorator where event with matching event key has already been succesfully processed """ - stubber = stub.Stubber(persistence_store.table.meta.client) + stubber = stub.Stubber(test_persistence_store.persistence_layer.table.meta.client) ddb_response = { "Item": { - "id": {"S": hashed_idempotency_key}, + **test_persistence_store.expected_key_values(hashed_idempotency_key), "expiration": {"N": timestamp_future}, "data": {"S": serialized_lambda_response}, "status": {"S": "COMPLETED"}, @@ -57,14 +57,14 @@ def test_idempotent_lambda_already_completed( expected_params = { "TableName": TABLE_NAME, - "Key": {"id": hashed_idempotency_key}, + "Key": test_persistence_store.expected_key(hashed_idempotency_key), "ConsistentRead": True, } stubber.add_client_error("put_item", "ConditionalCheckFailedException") stubber.add_response("get_item", ddb_response, expected_params) stubber.activate() - @idempotent(config=idempotency_config, persistence_store=persistence_store) + @idempotent(config=idempotency_config, persistence_store=test_persistence_store.persistence_layer) def lambda_handler(event, context): raise Exception @@ -78,7 +78,7 @@ def lambda_handler(event, context): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_in_progress( idempotency_config: IdempotencyConfig, - persistence_store: DynamoDBPersistenceLayer, + test_persistence_store: TestPersistenceStore, lambda_apigw_event, lambda_response, timestamp_future, @@ -89,16 +89,16 @@ def test_idempotent_lambda_in_progress( Test idempotent decorator where lambda_handler is already processing an event with matching event key """ - stubber = stub.Stubber(persistence_store.table.meta.client) + stubber = stub.Stubber(test_persistence_store.persistence_layer.table.meta.client) expected_params = { "TableName": TABLE_NAME, - "Key": {"id": hashed_idempotency_key}, + "Key": test_persistence_store.expected_key(hashed_idempotency_key), "ConsistentRead": True, } ddb_response = { "Item": { - "id": {"S": hashed_idempotency_key}, + **test_persistence_store.expected_key_values(hashed_idempotency_key), "expiration": {"N": timestamp_future}, "status": {"S": "INPROGRESS"}, } @@ -108,7 +108,7 @@ def test_idempotent_lambda_in_progress( stubber.add_response("get_item", ddb_response, expected_params) stubber.activate() - @idempotent(config=idempotency_config, persistence_store=persistence_store) + @idempotent(config=idempotency_config, persistence_store=test_persistence_store.persistence_layer) def lambda_handler(event, context): return lambda_response @@ -127,7 +127,7 @@ def lambda_handler(event, context): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) def test_idempotent_lambda_in_progress_with_cache( idempotency_config: IdempotencyConfig, - persistence_store: DynamoDBPersistenceLayer, + test_persistence_store: TestPersistenceStore, lambda_apigw_event, lambda_response, timestamp_future, @@ -139,18 +139,18 @@ def test_idempotent_lambda_in_progress_with_cache( Test idempotent decorator where lambda_handler is already processing an event with matching event key, cache enabled. """ - save_to_cache_spy = mocker.spy(persistence_store, "_save_to_cache") - retrieve_from_cache_spy = mocker.spy(persistence_store, "_retrieve_from_cache") - stubber = stub.Stubber(persistence_store.table.meta.client) + save_to_cache_spy = mocker.spy(test_persistence_store.persistence_layer, "_save_to_cache") + retrieve_from_cache_spy = mocker.spy(test_persistence_store.persistence_layer, "_retrieve_from_cache") + stubber = stub.Stubber(test_persistence_store.persistence_layer.table.meta.client) expected_params = { "TableName": TABLE_NAME, - "Key": {"id": hashed_idempotency_key}, + "Key": test_persistence_store.expected_key(hashed_idempotency_key), "ConsistentRead": True, } ddb_response = { "Item": { - "id": {"S": hashed_idempotency_key}, + **test_persistence_store.expected_key_values(hashed_idempotency_key), "expiration": {"N": timestamp_future}, "status": {"S": "INPROGRESS"}, } @@ -166,7 +166,7 @@ def test_idempotent_lambda_in_progress_with_cache( stubber.add_response("get_item", copy.deepcopy(ddb_response), copy.deepcopy(expected_params)) stubber.activate() - @idempotent(config=idempotency_config, persistence_store=persistence_store) + @idempotent(config=idempotency_config, persistence_store=test_persistence_store.persistence_layer) def lambda_handler(event, context): return lambda_response @@ -183,7 +183,7 @@ def lambda_handler(event, context): retrieve_from_cache_spy.assert_called_with(idempotency_key=hashed_idempotency_key) save_to_cache_spy.assert_called() - assert persistence_store._cache.get(hashed_idempotency_key) is None + assert test_persistence_store.persistence_layer._cache.get(hashed_idempotency_key) is None stubber.assert_no_pending_responses() stubber.deactivate() @@ -192,7 +192,7 @@ def lambda_handler(event, context): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_first_execution( idempotency_config: IdempotencyConfig, - persistence_store: DynamoDBPersistenceLayer, + test_persistence_store: TestPersistenceStore, lambda_apigw_event, expected_params_update_item, expected_params_put_item, @@ -206,14 +206,24 @@ def test_idempotent_lambda_first_execution( Test idempotent decorator when lambda is executed with an event with a previously unknown event key """ - stubber = stub.Stubber(persistence_store.table.meta.client) + stubber = stub.Stubber(test_persistence_store.persistence_layer.table.meta.client) ddb_response = {} - stubber.add_response("put_item", ddb_response, expected_params_put_item) - stubber.add_response("update_item", ddb_response, expected_params_update_item) + stubber.add_response( + "put_item", + ddb_response, + expected_params_put_item( + test_persistence_store.key_attr, test_persistence_store.expected_key(hashed_idempotency_key) + ), + ) + stubber.add_response( + "update_item", + ddb_response, + expected_params_update_item(test_persistence_store.expected_key(hashed_idempotency_key)), + ) stubber.activate() - @idempotent(config=idempotency_config, persistence_store=persistence_store) + @idempotent(config=idempotency_config, persistence_store=test_persistence_store.persistence_layer) def lambda_handler(event, context): return lambda_response @@ -226,7 +236,7 @@ def lambda_handler(event, context): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) def test_idempotent_lambda_first_execution_cached( idempotency_config: IdempotencyConfig, - persistence_store: DynamoDBPersistenceLayer, + test_persistence_store: TestPersistenceStore, lambda_apigw_event, expected_params_update_item, expected_params_put_item, @@ -239,16 +249,26 @@ def test_idempotent_lambda_first_execution_cached( Test idempotent decorator when lambda is executed with an event with a previously unknown event key. Ensure result is cached locally on the persistence store instance. """ - save_to_cache_spy = mocker.spy(persistence_store, "_save_to_cache") - retrieve_from_cache_spy = mocker.spy(persistence_store, "_retrieve_from_cache") - stubber = stub.Stubber(persistence_store.table.meta.client) + save_to_cache_spy = mocker.spy(test_persistence_store.persistence_layer, "_save_to_cache") + retrieve_from_cache_spy = mocker.spy(test_persistence_store.persistence_layer, "_retrieve_from_cache") + stubber = stub.Stubber(test_persistence_store.persistence_layer.table.meta.client) ddb_response = {} - stubber.add_response("put_item", ddb_response, expected_params_put_item) - stubber.add_response("update_item", ddb_response, expected_params_update_item) + stubber.add_response( + "put_item", + ddb_response, + expected_params_put_item( + test_persistence_store.key_attr, test_persistence_store.expected_key(hashed_idempotency_key) + ), + ) + stubber.add_response( + "update_item", + ddb_response, + expected_params_update_item(test_persistence_store.expected_key(hashed_idempotency_key)), + ) stubber.activate() - @idempotent(config=idempotency_config, persistence_store=persistence_store) + @idempotent(config=idempotency_config, persistence_store=test_persistence_store.persistence_layer) def lambda_handler(event, context): return lambda_response @@ -257,7 +277,7 @@ def lambda_handler(event, context): retrieve_from_cache_spy.assert_called_once() save_to_cache_spy.assert_called_once() assert save_to_cache_spy.call_args[1]["data_record"].status == "COMPLETED" - assert persistence_store._cache.get(hashed_idempotency_key).status == "COMPLETED" + assert test_persistence_store.persistence_layer._cache.get(hashed_idempotency_key).status == "COMPLETED" # This lambda call should not call AWS API lambda_handler(lambda_apigw_event, lambda_context) @@ -272,7 +292,7 @@ def lambda_handler(event, context): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_expired( idempotency_config: IdempotencyConfig, - persistence_store: DynamoDBPersistenceLayer, + test_persistence_store: TestPersistenceStore, lambda_apigw_event, timestamp_expired, lambda_response, @@ -286,15 +306,25 @@ def test_idempotent_lambda_expired( expiry window """ - stubber = stub.Stubber(persistence_store.table.meta.client) + stubber = stub.Stubber(test_persistence_store.persistence_layer.table.meta.client) ddb_response = {} - stubber.add_response("put_item", ddb_response, expected_params_put_item) - stubber.add_response("update_item", ddb_response, expected_params_update_item) + stubber.add_response( + "put_item", + ddb_response, + expected_params_put_item( + test_persistence_store.key_attr, test_persistence_store.expected_key(hashed_idempotency_key) + ), + ) + stubber.add_response( + "update_item", + ddb_response, + expected_params_update_item(test_persistence_store.expected_key(hashed_idempotency_key)), + ) stubber.activate() - @idempotent(config=idempotency_config, persistence_store=persistence_store) + @idempotent(config=idempotency_config, persistence_store=test_persistence_store.persistence_layer) def lambda_handler(event, context): return lambda_response @@ -307,7 +337,7 @@ def lambda_handler(event, context): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_exception( idempotency_config: IdempotencyConfig, - persistence_store: DynamoDBPersistenceLayer, + test_persistence_store: TestPersistenceStore, lambda_apigw_event, timestamp_future, lambda_response, @@ -323,16 +353,25 @@ def test_idempotent_lambda_exception( # Create a new provider # Stub the boto3 client - stubber = stub.Stubber(persistence_store.table.meta.client) + stubber = stub.Stubber(test_persistence_store.persistence_layer.table.meta.client) ddb_response = {} - expected_params_delete_item = {"TableName": TABLE_NAME, "Key": {"id": hashed_idempotency_key}} + expected_params_delete_item = { + "TableName": TABLE_NAME, + "Key": test_persistence_store.expected_key(hashed_idempotency_key), + } - stubber.add_response("put_item", ddb_response, expected_params_put_item) + stubber.add_response( + "put_item", + ddb_response, + expected_params_put_item( + test_persistence_store.key_attr, test_persistence_store.expected_key(hashed_idempotency_key) + ), + ) stubber.add_response("delete_item", ddb_response, expected_params_delete_item) stubber.activate() - @idempotent(config=idempotency_config, persistence_store=persistence_store) + @idempotent(config=idempotency_config, persistence_store=test_persistence_store.persistence_layer) def lambda_handler(event, context): raise Exception("Something went wrong!") @@ -348,7 +387,7 @@ def lambda_handler(event, context): ) def test_idempotent_lambda_already_completed_with_validation_bad_payload( config_with_validation: IdempotencyConfig, - persistence_store: DynamoDBPersistenceLayer, + test_persistence_store: TestPersistenceStore, lambda_apigw_event, timestamp_future, lambda_response, @@ -360,10 +399,10 @@ def test_idempotent_lambda_already_completed_with_validation_bad_payload( Test idempotent decorator where event with matching event key has already been successfully processed """ - stubber = stub.Stubber(persistence_store.table.meta.client) + stubber = stub.Stubber(test_persistence_store.persistence_layer.table.meta.client) ddb_response = { "Item": { - "id": {"S": hashed_idempotency_key}, + **test_persistence_store.expected_key_values(hashed_idempotency_key), "expiration": {"N": timestamp_future}, "data": {"S": '{"message": "test", "statusCode": 200}'}, "status": {"S": "COMPLETED"}, @@ -371,13 +410,17 @@ def test_idempotent_lambda_already_completed_with_validation_bad_payload( } } - expected_params = {"TableName": TABLE_NAME, "Key": {"id": hashed_idempotency_key}, "ConsistentRead": True} + expected_params = { + "TableName": TABLE_NAME, + "Key": test_persistence_store.expected_key(hashed_idempotency_key), + "ConsistentRead": True, + } stubber.add_client_error("put_item", "ConditionalCheckFailedException") stubber.add_response("get_item", ddb_response, expected_params) stubber.activate() - @idempotent(config=config_with_validation, persistence_store=persistence_store) + @idempotent(config=config_with_validation, persistence_store=test_persistence_store.persistence_layer) def lambda_handler(event, context): return lambda_response @@ -392,7 +435,7 @@ def lambda_handler(event, context): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_expired_during_request( idempotency_config: IdempotencyConfig, - persistence_store: DynamoDBPersistenceLayer, + test_persistence_store: TestPersistenceStore, lambda_apigw_event, timestamp_expired, lambda_response, @@ -404,11 +447,11 @@ def test_idempotent_lambda_expired_during_request( returns inconsistent/rapidly changing result between put_item and get_item calls. """ - stubber = stub.Stubber(persistence_store.table.meta.client) + stubber = stub.Stubber(test_persistence_store.persistence_layer.table.meta.client) ddb_response_get_item = { "Item": { - "id": {"S": hashed_idempotency_key}, + **test_persistence_store.expected_key_values(hashed_idempotency_key), "expiration": {"N": timestamp_expired}, "data": {"S": '{"message": "test", "statusCode": 200}'}, "status": {"S": "INPROGRESS"}, @@ -417,7 +460,7 @@ def test_idempotent_lambda_expired_during_request( ddb_response_get_item_missing = {} expected_params_get_item = { "TableName": TABLE_NAME, - "Key": {"id": hashed_idempotency_key}, + "Key": test_persistence_store.expected_key(hashed_idempotency_key), "ConsistentRead": True, } @@ -433,7 +476,7 @@ def test_idempotent_lambda_expired_during_request( stubber.activate() - @idempotent(config=idempotency_config, persistence_store=persistence_store) + @idempotent(config=idempotency_config, persistence_store=test_persistence_store.persistence_layer) def lambda_handler(event, context): return lambda_response @@ -448,7 +491,7 @@ def lambda_handler(event, context): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_persistence_exception_deleting( idempotency_config: IdempotencyConfig, - persistence_store: DynamoDBPersistenceLayer, + test_persistence_store: TestPersistenceStore, lambda_apigw_event, timestamp_future, lambda_response, @@ -460,15 +503,21 @@ def test_idempotent_persistence_exception_deleting( Test idempotent decorator when lambda is executed with an event with a previously unknown event key, but lambda_handler raises an exception which is retryable. """ - stubber = stub.Stubber(persistence_store.table.meta.client) + stubber = stub.Stubber(test_persistence_store.persistence_layer.table.meta.client) ddb_response = {} - stubber.add_response("put_item", ddb_response, expected_params_put_item) + stubber.add_response( + "put_item", + ddb_response, + expected_params_put_item( + test_persistence_store.key_attr, test_persistence_store.expected_key(hashed_idempotency_key) + ), + ) stubber.add_client_error("delete_item", "UnrecoverableError") stubber.activate() - @idempotent(config=idempotency_config, persistence_store=persistence_store) + @idempotent(config=idempotency_config, persistence_store=test_persistence_store.persistence_layer) def lambda_handler(event, context): raise Exception("Something went wrong!") @@ -483,7 +532,7 @@ def lambda_handler(event, context): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_persistence_exception_updating( idempotency_config: IdempotencyConfig, - persistence_store: DynamoDBPersistenceLayer, + test_persistence_store: TestPersistenceStore, lambda_apigw_event, timestamp_future, lambda_response, @@ -495,15 +544,21 @@ def test_idempotent_persistence_exception_updating( Test idempotent decorator when lambda is executed with an event with a previously unknown event key, but lambda_handler raises an exception which is retryable. """ - stubber = stub.Stubber(persistence_store.table.meta.client) + stubber = stub.Stubber(test_persistence_store.persistence_layer.table.meta.client) ddb_response = {} - stubber.add_response("put_item", ddb_response, expected_params_put_item) + stubber.add_response( + "put_item", + ddb_response, + expected_params_put_item( + test_persistence_store.key_attr, test_persistence_store.expected_key(hashed_idempotency_key) + ), + ) stubber.add_client_error("update_item", "UnrecoverableError") stubber.activate() - @idempotent(config=idempotency_config, persistence_store=persistence_store) + @idempotent(config=idempotency_config, persistence_store=test_persistence_store.persistence_layer) def lambda_handler(event, context): return {"message": "success!"} @@ -518,7 +573,7 @@ def lambda_handler(event, context): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_persistence_exception_getting( idempotency_config: IdempotencyConfig, - persistence_store: DynamoDBPersistenceLayer, + test_persistence_store: TestPersistenceStore, lambda_apigw_event, timestamp_future, lambda_response, @@ -530,13 +585,13 @@ def test_idempotent_persistence_exception_getting( Test idempotent decorator when lambda is executed with an event with a previously unknown event key, but lambda_handler raises an exception which is retryable. """ - stubber = stub.Stubber(persistence_store.table.meta.client) + stubber = stub.Stubber(test_persistence_store.persistence_layer.table.meta.client) stubber.add_client_error("put_item", "ConditionalCheckFailedException") stubber.add_client_error("get_item", "UnexpectedException") stubber.activate() - @idempotent(config=idempotency_config, persistence_store=persistence_store) + @idempotent(config=idempotency_config, persistence_store=test_persistence_store.persistence_layer) def lambda_handler(event, context): return {"message": "success!"} @@ -553,7 +608,7 @@ def lambda_handler(event, context): ) def test_idempotent_lambda_first_execution_with_validation( config_with_validation: IdempotencyConfig, - persistence_store: DynamoDBPersistenceLayer, + test_persistence_store: TestPersistenceStore, lambda_apigw_event, expected_params_update_item_with_validation, expected_params_put_item_with_validation, @@ -565,14 +620,24 @@ def test_idempotent_lambda_first_execution_with_validation( """ Test idempotent decorator when lambda is executed with an event with a previously unknown event key """ - stubber = stub.Stubber(persistence_store.table.meta.client) + stubber = stub.Stubber(test_persistence_store.persistence_layer.table.meta.client) ddb_response = {} - stubber.add_response("put_item", ddb_response, expected_params_put_item_with_validation) - stubber.add_response("update_item", ddb_response, expected_params_update_item_with_validation) + stubber.add_response( + "put_item", + ddb_response, + expected_params_put_item_with_validation( + test_persistence_store.key_attr, test_persistence_store.expected_key(hashed_idempotency_key) + ), + ) + stubber.add_response( + "update_item", + ddb_response, + expected_params_update_item_with_validation(test_persistence_store.expected_key(hashed_idempotency_key)), + ) stubber.activate() - @idempotent(config=config_with_validation, persistence_store=persistence_store) + @idempotent(config=config_with_validation, persistence_store=test_persistence_store.persistence_layer) def lambda_handler(event, context): return lambda_response @@ -587,7 +652,7 @@ def lambda_handler(event, context): ) def test_idempotent_lambda_with_validator_util( config_without_jmespath: IdempotencyConfig, - persistence_store: DynamoDBPersistenceLayer, + test_persistence_store: TestPersistenceStore, lambda_apigw_event, timestamp_future, serialized_lambda_response, @@ -601,10 +666,10 @@ def test_idempotent_lambda_with_validator_util( validator utility to unwrap the event """ - stubber = stub.Stubber(persistence_store.table.meta.client) + stubber = stub.Stubber(test_persistence_store.persistence_layer.table.meta.client) ddb_response = { "Item": { - "id": {"S": hashed_idempotency_key_with_envelope}, + **test_persistence_store.expected_key_values(hashed_idempotency_key_with_envelope), "expiration": {"N": timestamp_future}, "data": {"S": serialized_lambda_response}, "status": {"S": "COMPLETED"}, @@ -613,7 +678,7 @@ def test_idempotent_lambda_with_validator_util( expected_params = { "TableName": TABLE_NAME, - "Key": {"id": hashed_idempotency_key_with_envelope}, + "Key": test_persistence_store.expected_key(hashed_idempotency_key_with_envelope), "ConsistentRead": True, } stubber.add_client_error("put_item", "ConditionalCheckFailedException") @@ -621,7 +686,7 @@ def test_idempotent_lambda_with_validator_util( stubber.activate() @validator(envelope=envelopes.API_GATEWAY_HTTP) - @idempotent(config=config_without_jmespath, persistence_store=persistence_store) + @idempotent(config=config_without_jmespath, persistence_store=test_persistence_store.persistence_layer) def lambda_handler(event, context): mock_function() return "shouldn't get here!" @@ -644,50 +709,50 @@ def test_data_record_invalid_status_value(): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) def test_in_progress_never_saved_to_cache( - idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer + idempotency_config: IdempotencyConfig, test_persistence_store: TestPersistenceStore ): # GIVEN a data record with status "INPROGRESS" - # and persistence_store has use_local_cache = True - persistence_store.configure(idempotency_config) + # and test_persistence_store.persistence_layerhas use_local_cache = True + test_persistence_store.persistence_layer.configure(idempotency_config) data_record = DataRecord("key", status="INPROGRESS") # WHEN saving to local cache - persistence_store._save_to_cache(data_record) + test_persistence_store.persistence_layer._save_to_cache(data_record) # THEN don't save to local cache - assert persistence_store._cache.get("key") is None + assert test_persistence_store.persistence_layer._cache.get("key") is None @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}], indirect=True) -def test_user_local_disabled(idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer): - # GIVEN a persistence_store with use_local_cache = False - persistence_store.configure(idempotency_config) +def test_user_local_disabled(idempotency_config: IdempotencyConfig, test_persistence_store: TestPersistenceStore): + # GIVEN a test_persistence_store.persistence_layerwith use_local_cache = False + test_persistence_store.persistence_layer.configure(idempotency_config) # WHEN calling any local cache options data_record = DataRecord("key", status="COMPLETED") try: - persistence_store._save_to_cache(data_record) - cache_value = persistence_store._retrieve_from_cache("key") + test_persistence_store.persistence_layer._save_to_cache(data_record) + cache_value = test_persistence_store.persistence_layer._retrieve_from_cache("key") assert cache_value is None - persistence_store._delete_from_cache("key") + test_persistence_store.persistence_layer._delete_from_cache("key") except AttributeError as e: pytest.fail(f"AttributeError should not be raised: {e}") # THEN raise AttributeError # AND don't have a _cache attribute - assert not hasattr("persistence_store", "_cache") + assert not hasattr("test_persistence_store.persistence_layer", "_cache") @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) def test_delete_from_cache_when_empty( - idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer + idempotency_config: IdempotencyConfig, test_persistence_store: TestPersistenceStore ): # GIVEN use_local_cache is True AND the local cache is empty - persistence_store.configure(idempotency_config) + test_persistence_store.persistence_layer.configure(idempotency_config) try: # WHEN we _delete_from_cache - persistence_store._delete_from_cache("key_does_not_exist") + test_persistence_store.persistence_layer._delete_from_cache("key_does_not_exist") except KeyError: # THEN we should not get a KeyError pytest.fail("KeyError should not happen") @@ -732,15 +797,15 @@ def test_is_missing_idempotency_key(): "idempotency_config", [{"use_local_cache": False, "event_key_jmespath": "body"}], indirect=True ) def test_default_no_raise_on_missing_idempotency_key( - idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, lambda_context + idempotency_config: IdempotencyConfig, test_persistence_store: TestPersistenceStore, lambda_context ): - # GIVEN a persistence_store with use_local_cache = False and event_key_jmespath = "body" - persistence_store.configure(idempotency_config) - assert persistence_store.use_local_cache is False - assert "body" in persistence_store.event_key_jmespath + # GIVEN a test_persistence_store.persistence_layerwith use_local_cache = False and event_key_jmespath = "body" + test_persistence_store.persistence_layer.configure(idempotency_config) + assert test_persistence_store.persistence_layer.use_local_cache is False + assert "body" in test_persistence_store.persistence_layer.event_key_jmespath # WHEN getting the hashed idempotency key for an event with no `body` key - hashed_key = persistence_store._get_hashed_idempotency_key({}) + hashed_key = test_persistence_store.persistence_layer._get_hashed_idempotency_key({}) # THEN return the hash of None expected_value = "test-func#" + md5(serialize(None).encode()).hexdigest() @@ -751,17 +816,18 @@ def test_default_no_raise_on_missing_idempotency_key( "idempotency_config", [{"use_local_cache": False, "event_key_jmespath": "[body, x]"}], indirect=True ) def test_raise_on_no_idempotency_key( - idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, lambda_context + idempotency_config: IdempotencyConfig, test_persistence_store: TestPersistenceStore, lambda_context ): - # GIVEN a persistence_store with raise_on_no_idempotency_key and no idempotency key in the request - persistence_store.configure(idempotency_config) - persistence_store.raise_on_no_idempotency_key = True - assert persistence_store.use_local_cache is False - assert "body" in persistence_store.event_key_jmespath + # GIVEN a test_persistence_store.persistence_layer with raise_on_no_idempotency_key + # and no idempotency key in the request + test_persistence_store.persistence_layer.configure(idempotency_config) + test_persistence_store.persistence_layer.raise_on_no_idempotency_key = True + assert test_persistence_store.persistence_layer.use_local_cache is False + assert "body" in test_persistence_store.persistence_layer.event_key_jmespath # WHEN getting the hashed idempotency key for an event with no `body` key with pytest.raises(IdempotencyKeyError) as excinfo: - persistence_store._get_hashed_idempotency_key({}) + test_persistence_store.persistence_layer._get_hashed_idempotency_key({}) # THEN raise IdempotencyKeyError error assert "No data found to create a hashed idempotency_key" in str(excinfo.value) @@ -778,10 +844,10 @@ def test_raise_on_no_idempotency_key( indirect=True, ) def test_jmespath_with_powertools_json( - idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, lambda_context + idempotency_config: IdempotencyConfig, test_persistence_store: TestPersistenceStore, lambda_context ): # GIVEN an event_key_jmespath with powertools_json custom function - persistence_store.configure(idempotency_config) + test_persistence_store.persistence_layer.configure(idempotency_config) sub_attr_value = "cognito_user" key_attr_value = "some_key" expected_value = [sub_attr_value, key_attr_value] @@ -791,34 +857,34 @@ def test_jmespath_with_powertools_json( } # WHEN calling _get_hashed_idempotency_key - result = persistence_store._get_hashed_idempotency_key(api_gateway_proxy_event) + result = test_persistence_store.persistence_layer._get_hashed_idempotency_key(api_gateway_proxy_event) # THEN the hashed idempotency key should match the extracted values generated hash - assert result == "test-func#" + persistence_store._generate_hash(expected_value) + assert result == "test-func#" + test_persistence_store.persistence_layer._generate_hash(expected_value) @pytest.mark.parametrize("config_with_jmespath_options", ["powertools_json(data).payload"], indirect=True) def test_custom_jmespath_function_overrides_builtin_functions( - config_with_jmespath_options: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, lambda_context + config_with_jmespath_options: IdempotencyConfig, test_persistence_store: TestPersistenceStore, lambda_context ): # GIVEN an persistence store with a custom jmespath_options # AND use a builtin powertools custom function - persistence_store.configure(config_with_jmespath_options) + test_persistence_store.persistence_layer.configure(config_with_jmespath_options) with pytest.raises(jmespath.exceptions.UnknownFunctionError, match="Unknown function: powertools_json()"): # WHEN calling _get_hashed_idempotency_key # THEN raise unknown function - persistence_store._get_hashed_idempotency_key({}) + test_persistence_store.persistence_layer._get_hashed_idempotency_key({}) -def test_idempotent_lambda_save_inprogress_error(persistence_store: DynamoDBPersistenceLayer, lambda_context): +def test_idempotent_lambda_save_inprogress_error(test_persistence_store: TestPersistenceStore, lambda_context): # GIVEN a miss configured persistence layer # like no table was created for the idempotency persistence layer - stubber = stub.Stubber(persistence_store.table.meta.client) + stubber = stub.Stubber(test_persistence_store.persistence_layer.table.meta.client) stubber.add_client_error("put_item", "ResourceNotFoundException") stubber.activate() - @idempotent(persistence_store=persistence_store) + @idempotent(persistence_store=test_persistence_store.persistence_layer) def lambda_handler(event, context): return {} @@ -833,13 +899,13 @@ def lambda_handler(event, context): assert "Failed to save in progress record to idempotency store" == e.value.args[0] -def test_handler_raise_idempotency_key_error(persistence_store: DynamoDBPersistenceLayer, lambda_context): +def test_handler_raise_idempotency_key_error(test_persistence_store: TestPersistenceStore, lambda_context): # GIVEN raise_on_no_idempotency_key is True idempotency_config = IdempotencyConfig(event_key_jmespath="idemKey", raise_on_no_idempotency_key=True) # WHEN handling the idempotent call # AND save_inprogress raises a IdempotencyKeyError - @idempotent(persistence_store=persistence_store, config=idempotency_config) + @idempotent(persistence_store=test_persistence_store.persistence_layer, config=idempotency_config) def handler(event, context): raise ValueError("Should not be raised") @@ -997,23 +1063,23 @@ def dummy(payload): dummy(payload=data_two) -def test_idempotency_disabled_envvar(monkeypatch, lambda_context, persistence_store: DynamoDBPersistenceLayer): +def test_idempotency_disabled_envvar(monkeypatch, lambda_context, test_persistence_store: TestPersistenceStore): # Scenario to validate no requests sent to dynamodb table when 'POWERTOOLS_IDEMPOTENCY_DISABLED' is set mock_event = {"data": "value"} - persistence_store.table = MagicMock() + test_persistence_store.persistence_layer.table = MagicMock() monkeypatch.setenv("POWERTOOLS_IDEMPOTENCY_DISABLED", "1") - @idempotent_function(data_keyword_argument="data", persistence_store=persistence_store) + @idempotent_function(data_keyword_argument="data", persistence_store=test_persistence_store.persistence_layer) def dummy(data): return {"message": "hello"} - @idempotent(persistence_store=persistence_store) + @idempotent(persistence_store=test_persistence_store.persistence_layer) def dummy_handler(event, context): return {"message": "hi"} dummy(data=mock_event) dummy_handler(mock_event, lambda_context) - assert len(persistence_store.table.method_calls) == 0 + assert len(test_persistence_store.persistence_layer.table.method_calls) == 0 From 9548e228f8342071df30cb7a07441cc6165313d0 Mon Sep 17 00:00:00 2001 From: Adam Tankanow Date: Wed, 13 Oct 2021 17:54:01 -0400 Subject: [PATCH 2/4] ISSUE-694: change default pk; revert test changes for now --- .../idempotency/persistence/dynamodb.py | 8 +- tests/functional/idempotency/conftest.py | 72 ++--- .../idempotency/test_idempotency.py | 304 +++++++----------- 3 files changed, 148 insertions(+), 236 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py index ab0640bd36d..87aa524adcb 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py @@ -1,10 +1,12 @@ import datetime import logging +import os from typing import Any, Dict, Optional import boto3 from botocore.config import Config +from aws_lambda_powertools.shared import constants from aws_lambda_powertools.utilities.idempotency import BasePersistenceLayer from aws_lambda_powertools.utilities.idempotency.exceptions import ( IdempotencyItemAlreadyExistsError, @@ -20,7 +22,7 @@ def __init__( self, table_name: str, key_attr: str = "id", - key_attr_value: str = "powertools#idempotency", + key_attr_value: str = f"idempotency#{os.getenv(constants.LAMBDA_FUNCTION_NAME_ENV, 'test-func')}", sort_key_attr: Optional[str] = None, expiry_attr: str = "expiration", status_attr: str = "status", @@ -39,7 +41,7 @@ def __init__( key_attr: str, optional DynamoDB attribute name for partition key, by default "id" key_attr_value: str, optional - DynamoDB attribute value for partition key, by default "powertools#idempotency". + DynamoDB attribute value for partition key, by default "idempotency#". This will be used if the sort_key_attr is set. sort_key_attr: str, optional DynamoDB attribute name for the sort key @@ -71,6 +73,8 @@ def __init__( self._boto_config = boto_config or Config() self._boto3_session = boto3_session or boto3.session.Session() + if sort_key_attr == key_attr: + raise ValueError(f"key_attr [{key_attr}] and sort_key_attr [{sort_key_attr}] cannot be the same!") self._table = None self.table_name = table_name diff --git a/tests/functional/idempotency/conftest.py b/tests/functional/idempotency/conftest.py index 0955f05584b..71b5978497c 100644 --- a/tests/functional/idempotency/conftest.py +++ b/tests/functional/idempotency/conftest.py @@ -2,9 +2,7 @@ import hashlib import json from collections import namedtuple -from dataclasses import dataclass from decimal import Decimal -from typing import Callable from unittest import mock import jmespath @@ -82,23 +80,25 @@ def default_jmespath(): @pytest.fixture -def expected_params_update_item(serialized_lambda_response): - return lambda key: { +def expected_params_update_item(serialized_lambda_response, hashed_idempotency_key): + return { "ExpressionAttributeNames": {"#expiry": "expiration", "#response_data": "data", "#status": "status"}, "ExpressionAttributeValues": { ":expiry": stub.ANY, ":response_data": serialized_lambda_response, ":status": "COMPLETED", }, - "Key": key, + "Key": {"id": hashed_idempotency_key}, "TableName": "TEST_TABLE", "UpdateExpression": "SET #response_data = :response_data, " "#expiry = :expiry, #status = :status", } @pytest.fixture -def expected_params_update_item_with_validation(serialized_lambda_response, hashed_validation_key): - return lambda key: { +def expected_params_update_item_with_validation( + serialized_lambda_response, hashed_idempotency_key, hashed_validation_key +): + return { "ExpressionAttributeNames": { "#expiry": "expiration", "#response_data": "data", @@ -111,7 +111,7 @@ def expected_params_update_item_with_validation(serialized_lambda_response, hash ":status": "COMPLETED", ":validation_key": hashed_validation_key, }, - "Key": key, + "Key": {"id": hashed_idempotency_key}, "TableName": "TEST_TABLE", "UpdateExpression": "SET #response_data = :response_data, " "#expiry = :expiry, #status = :status, " @@ -120,23 +120,28 @@ def expected_params_update_item_with_validation(serialized_lambda_response, hash @pytest.fixture -def expected_params_put_item(): - return lambda key_attr, key: { +def expected_params_put_item(hashed_idempotency_key): + return { "ConditionExpression": "attribute_not_exists(#id) OR #now < :now", - "ExpressionAttributeNames": {"#id": key_attr, "#now": "expiration"}, + "ExpressionAttributeNames": {"#id": "id", "#now": "expiration"}, "ExpressionAttributeValues": {":now": stub.ANY}, - "Item": {"expiration": stub.ANY, "status": "INPROGRESS", **key}, + "Item": {"expiration": stub.ANY, "id": hashed_idempotency_key, "status": "INPROGRESS"}, "TableName": "TEST_TABLE", } @pytest.fixture -def expected_params_put_item_with_validation(hashed_validation_key): - return lambda key_attr, key: { +def expected_params_put_item_with_validation(hashed_idempotency_key, hashed_validation_key): + return { "ConditionExpression": "attribute_not_exists(#id) OR #now < :now", - "ExpressionAttributeNames": {"#id": key_attr, "#now": "expiration"}, + "ExpressionAttributeNames": {"#id": "id", "#now": "expiration"}, "ExpressionAttributeValues": {":now": stub.ANY}, - "Item": {"expiration": stub.ANY, "status": "INPROGRESS", "validation": hashed_validation_key, **key}, + "Item": { + "expiration": stub.ANY, + "id": hashed_idempotency_key, + "status": "INPROGRESS", + "validation": hashed_validation_key, + }, "TableName": "TEST_TABLE", } @@ -161,40 +166,9 @@ def hashed_validation_key(lambda_apigw_event): return hashlib.md5(serialize(lambda_apigw_event["requestContext"]).encode()).hexdigest() -@dataclass(eq=True, frozen=True) -class TestPersistenceStore: - persistence_layer: DynamoDBPersistenceLayer - key_attr: str - expected_key: Callable[[str], dict] - expected_key_values: Callable[[str], dict] - - -@pytest.fixture(params=[{}, {"key_attr": "PK", "key_attr_value": "powertools#idempotency", "sort_key_attr": "SK"}]) -def test_persistence_store(config, request) -> TestPersistenceStore: - expected_key = ( - lambda idempotency_key: {"PK": "powertools#idempotency", "SK": idempotency_key} - if request.param - else {"id": idempotency_key} - ) - expected_key_values = ( - lambda idempotency_key: {"PK": {"S": "powertools#idempotency"}, "SK": {"S": idempotency_key}} - if request.param - else {"id": {"S": idempotency_key}} - ) - return TestPersistenceStore( - persistence_layer=DynamoDBPersistenceLayer(table_name=TABLE_NAME, boto_config=config, **request.param), - key_attr="PK" if request.param else "id", - expected_key=expected_key, - expected_key_values=expected_key_values, - ) - - @pytest.fixture -def persistence_store_with_composite_key(config): - return DynamoDBPersistenceLayer( - table_name=TABLE_NAME, - boto_config=config, - ) +def persistence_store(config): + return DynamoDBPersistenceLayer(table_name=TABLE_NAME, boto_config=config) @pytest.fixture diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/test_idempotency.py index a3dd00595bc..b1d0914d181 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/test_idempotency.py @@ -10,7 +10,7 @@ from botocore import stub from aws_lambda_powertools.utilities.data_classes import APIGatewayProxyEventV2, event_source -from aws_lambda_powertools.utilities.idempotency import IdempotencyConfig +from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer, IdempotencyConfig from aws_lambda_powertools.utilities.idempotency.exceptions import ( IdempotencyAlreadyInProgressError, IdempotencyInconsistentStateError, @@ -22,7 +22,7 @@ from aws_lambda_powertools.utilities.idempotency.idempotency import idempotent, idempotent_function from aws_lambda_powertools.utilities.idempotency.persistence.base import BasePersistenceLayer, DataRecord from aws_lambda_powertools.utilities.validation import envelopes, validator -from tests.functional.idempotency.conftest import TestPersistenceStore, serialize +from tests.functional.idempotency.conftest import serialize from tests.functional.utils import load_event TABLE_NAME = "TEST_TABLE" @@ -33,7 +33,7 @@ @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_already_completed( idempotency_config: IdempotencyConfig, - test_persistence_store: TestPersistenceStore, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, timestamp_future, hashed_idempotency_key, @@ -45,10 +45,10 @@ def test_idempotent_lambda_already_completed( Test idempotent decorator where event with matching event key has already been succesfully processed """ - stubber = stub.Stubber(test_persistence_store.persistence_layer.table.meta.client) + stubber = stub.Stubber(persistence_store.table.meta.client) ddb_response = { "Item": { - **test_persistence_store.expected_key_values(hashed_idempotency_key), + "id": {"S": hashed_idempotency_key}, "expiration": {"N": timestamp_future}, "data": {"S": serialized_lambda_response}, "status": {"S": "COMPLETED"}, @@ -57,14 +57,14 @@ def test_idempotent_lambda_already_completed( expected_params = { "TableName": TABLE_NAME, - "Key": test_persistence_store.expected_key(hashed_idempotency_key), + "Key": {"id": hashed_idempotency_key}, "ConsistentRead": True, } stubber.add_client_error("put_item", "ConditionalCheckFailedException") stubber.add_response("get_item", ddb_response, expected_params) stubber.activate() - @idempotent(config=idempotency_config, persistence_store=test_persistence_store.persistence_layer) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): raise Exception @@ -78,7 +78,7 @@ def lambda_handler(event, context): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_in_progress( idempotency_config: IdempotencyConfig, - test_persistence_store: TestPersistenceStore, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, lambda_response, timestamp_future, @@ -89,16 +89,16 @@ def test_idempotent_lambda_in_progress( Test idempotent decorator where lambda_handler is already processing an event with matching event key """ - stubber = stub.Stubber(test_persistence_store.persistence_layer.table.meta.client) + stubber = stub.Stubber(persistence_store.table.meta.client) expected_params = { "TableName": TABLE_NAME, - "Key": test_persistence_store.expected_key(hashed_idempotency_key), + "Key": {"id": hashed_idempotency_key}, "ConsistentRead": True, } ddb_response = { "Item": { - **test_persistence_store.expected_key_values(hashed_idempotency_key), + "id": {"S": hashed_idempotency_key}, "expiration": {"N": timestamp_future}, "status": {"S": "INPROGRESS"}, } @@ -108,7 +108,7 @@ def test_idempotent_lambda_in_progress( stubber.add_response("get_item", ddb_response, expected_params) stubber.activate() - @idempotent(config=idempotency_config, persistence_store=test_persistence_store.persistence_layer) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): return lambda_response @@ -127,7 +127,7 @@ def lambda_handler(event, context): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) def test_idempotent_lambda_in_progress_with_cache( idempotency_config: IdempotencyConfig, - test_persistence_store: TestPersistenceStore, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, lambda_response, timestamp_future, @@ -139,18 +139,18 @@ def test_idempotent_lambda_in_progress_with_cache( Test idempotent decorator where lambda_handler is already processing an event with matching event key, cache enabled. """ - save_to_cache_spy = mocker.spy(test_persistence_store.persistence_layer, "_save_to_cache") - retrieve_from_cache_spy = mocker.spy(test_persistence_store.persistence_layer, "_retrieve_from_cache") - stubber = stub.Stubber(test_persistence_store.persistence_layer.table.meta.client) + save_to_cache_spy = mocker.spy(persistence_store, "_save_to_cache") + retrieve_from_cache_spy = mocker.spy(persistence_store, "_retrieve_from_cache") + stubber = stub.Stubber(persistence_store.table.meta.client) expected_params = { "TableName": TABLE_NAME, - "Key": test_persistence_store.expected_key(hashed_idempotency_key), + "Key": {"id": hashed_idempotency_key}, "ConsistentRead": True, } ddb_response = { "Item": { - **test_persistence_store.expected_key_values(hashed_idempotency_key), + "id": {"S": hashed_idempotency_key}, "expiration": {"N": timestamp_future}, "status": {"S": "INPROGRESS"}, } @@ -166,7 +166,7 @@ def test_idempotent_lambda_in_progress_with_cache( stubber.add_response("get_item", copy.deepcopy(ddb_response), copy.deepcopy(expected_params)) stubber.activate() - @idempotent(config=idempotency_config, persistence_store=test_persistence_store.persistence_layer) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): return lambda_response @@ -183,7 +183,7 @@ def lambda_handler(event, context): retrieve_from_cache_spy.assert_called_with(idempotency_key=hashed_idempotency_key) save_to_cache_spy.assert_called() - assert test_persistence_store.persistence_layer._cache.get(hashed_idempotency_key) is None + assert persistence_store._cache.get(hashed_idempotency_key) is None stubber.assert_no_pending_responses() stubber.deactivate() @@ -192,7 +192,7 @@ def lambda_handler(event, context): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_first_execution( idempotency_config: IdempotencyConfig, - test_persistence_store: TestPersistenceStore, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, expected_params_update_item, expected_params_put_item, @@ -206,24 +206,14 @@ def test_idempotent_lambda_first_execution( Test idempotent decorator when lambda is executed with an event with a previously unknown event key """ - stubber = stub.Stubber(test_persistence_store.persistence_layer.table.meta.client) + stubber = stub.Stubber(persistence_store.table.meta.client) ddb_response = {} - stubber.add_response( - "put_item", - ddb_response, - expected_params_put_item( - test_persistence_store.key_attr, test_persistence_store.expected_key(hashed_idempotency_key) - ), - ) - stubber.add_response( - "update_item", - ddb_response, - expected_params_update_item(test_persistence_store.expected_key(hashed_idempotency_key)), - ) + stubber.add_response("put_item", ddb_response, expected_params_put_item) + stubber.add_response("update_item", ddb_response, expected_params_update_item) stubber.activate() - @idempotent(config=idempotency_config, persistence_store=test_persistence_store.persistence_layer) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): return lambda_response @@ -236,7 +226,7 @@ def lambda_handler(event, context): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) def test_idempotent_lambda_first_execution_cached( idempotency_config: IdempotencyConfig, - test_persistence_store: TestPersistenceStore, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, expected_params_update_item, expected_params_put_item, @@ -249,26 +239,16 @@ def test_idempotent_lambda_first_execution_cached( Test idempotent decorator when lambda is executed with an event with a previously unknown event key. Ensure result is cached locally on the persistence store instance. """ - save_to_cache_spy = mocker.spy(test_persistence_store.persistence_layer, "_save_to_cache") - retrieve_from_cache_spy = mocker.spy(test_persistence_store.persistence_layer, "_retrieve_from_cache") - stubber = stub.Stubber(test_persistence_store.persistence_layer.table.meta.client) + save_to_cache_spy = mocker.spy(persistence_store, "_save_to_cache") + retrieve_from_cache_spy = mocker.spy(persistence_store, "_retrieve_from_cache") + stubber = stub.Stubber(persistence_store.table.meta.client) ddb_response = {} - stubber.add_response( - "put_item", - ddb_response, - expected_params_put_item( - test_persistence_store.key_attr, test_persistence_store.expected_key(hashed_idempotency_key) - ), - ) - stubber.add_response( - "update_item", - ddb_response, - expected_params_update_item(test_persistence_store.expected_key(hashed_idempotency_key)), - ) + stubber.add_response("put_item", ddb_response, expected_params_put_item) + stubber.add_response("update_item", ddb_response, expected_params_update_item) stubber.activate() - @idempotent(config=idempotency_config, persistence_store=test_persistence_store.persistence_layer) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): return lambda_response @@ -277,7 +257,7 @@ def lambda_handler(event, context): retrieve_from_cache_spy.assert_called_once() save_to_cache_spy.assert_called_once() assert save_to_cache_spy.call_args[1]["data_record"].status == "COMPLETED" - assert test_persistence_store.persistence_layer._cache.get(hashed_idempotency_key).status == "COMPLETED" + assert persistence_store._cache.get(hashed_idempotency_key).status == "COMPLETED" # This lambda call should not call AWS API lambda_handler(lambda_apigw_event, lambda_context) @@ -292,7 +272,7 @@ def lambda_handler(event, context): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_expired( idempotency_config: IdempotencyConfig, - test_persistence_store: TestPersistenceStore, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, timestamp_expired, lambda_response, @@ -306,25 +286,15 @@ def test_idempotent_lambda_expired( expiry window """ - stubber = stub.Stubber(test_persistence_store.persistence_layer.table.meta.client) + stubber = stub.Stubber(persistence_store.table.meta.client) ddb_response = {} - stubber.add_response( - "put_item", - ddb_response, - expected_params_put_item( - test_persistence_store.key_attr, test_persistence_store.expected_key(hashed_idempotency_key) - ), - ) - stubber.add_response( - "update_item", - ddb_response, - expected_params_update_item(test_persistence_store.expected_key(hashed_idempotency_key)), - ) + stubber.add_response("put_item", ddb_response, expected_params_put_item) + stubber.add_response("update_item", ddb_response, expected_params_update_item) stubber.activate() - @idempotent(config=idempotency_config, persistence_store=test_persistence_store.persistence_layer) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): return lambda_response @@ -337,7 +307,7 @@ def lambda_handler(event, context): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_exception( idempotency_config: IdempotencyConfig, - test_persistence_store: TestPersistenceStore, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, timestamp_future, lambda_response, @@ -353,25 +323,16 @@ def test_idempotent_lambda_exception( # Create a new provider # Stub the boto3 client - stubber = stub.Stubber(test_persistence_store.persistence_layer.table.meta.client) + stubber = stub.Stubber(persistence_store.table.meta.client) ddb_response = {} - expected_params_delete_item = { - "TableName": TABLE_NAME, - "Key": test_persistence_store.expected_key(hashed_idempotency_key), - } + expected_params_delete_item = {"TableName": TABLE_NAME, "Key": {"id": hashed_idempotency_key}} - stubber.add_response( - "put_item", - ddb_response, - expected_params_put_item( - test_persistence_store.key_attr, test_persistence_store.expected_key(hashed_idempotency_key) - ), - ) + stubber.add_response("put_item", ddb_response, expected_params_put_item) stubber.add_response("delete_item", ddb_response, expected_params_delete_item) stubber.activate() - @idempotent(config=idempotency_config, persistence_store=test_persistence_store.persistence_layer) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): raise Exception("Something went wrong!") @@ -387,7 +348,7 @@ def lambda_handler(event, context): ) def test_idempotent_lambda_already_completed_with_validation_bad_payload( config_with_validation: IdempotencyConfig, - test_persistence_store: TestPersistenceStore, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, timestamp_future, lambda_response, @@ -399,10 +360,10 @@ def test_idempotent_lambda_already_completed_with_validation_bad_payload( Test idempotent decorator where event with matching event key has already been successfully processed """ - stubber = stub.Stubber(test_persistence_store.persistence_layer.table.meta.client) + stubber = stub.Stubber(persistence_store.table.meta.client) ddb_response = { "Item": { - **test_persistence_store.expected_key_values(hashed_idempotency_key), + "id": {"S": hashed_idempotency_key}, "expiration": {"N": timestamp_future}, "data": {"S": '{"message": "test", "statusCode": 200}'}, "status": {"S": "COMPLETED"}, @@ -410,17 +371,13 @@ def test_idempotent_lambda_already_completed_with_validation_bad_payload( } } - expected_params = { - "TableName": TABLE_NAME, - "Key": test_persistence_store.expected_key(hashed_idempotency_key), - "ConsistentRead": True, - } + expected_params = {"TableName": TABLE_NAME, "Key": {"id": hashed_idempotency_key}, "ConsistentRead": True} stubber.add_client_error("put_item", "ConditionalCheckFailedException") stubber.add_response("get_item", ddb_response, expected_params) stubber.activate() - @idempotent(config=config_with_validation, persistence_store=test_persistence_store.persistence_layer) + @idempotent(config=config_with_validation, persistence_store=persistence_store) def lambda_handler(event, context): return lambda_response @@ -435,7 +392,7 @@ def lambda_handler(event, context): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_expired_during_request( idempotency_config: IdempotencyConfig, - test_persistence_store: TestPersistenceStore, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, timestamp_expired, lambda_response, @@ -447,11 +404,11 @@ def test_idempotent_lambda_expired_during_request( returns inconsistent/rapidly changing result between put_item and get_item calls. """ - stubber = stub.Stubber(test_persistence_store.persistence_layer.table.meta.client) + stubber = stub.Stubber(persistence_store.table.meta.client) ddb_response_get_item = { "Item": { - **test_persistence_store.expected_key_values(hashed_idempotency_key), + "id": {"S": hashed_idempotency_key}, "expiration": {"N": timestamp_expired}, "data": {"S": '{"message": "test", "statusCode": 200}'}, "status": {"S": "INPROGRESS"}, @@ -460,7 +417,7 @@ def test_idempotent_lambda_expired_during_request( ddb_response_get_item_missing = {} expected_params_get_item = { "TableName": TABLE_NAME, - "Key": test_persistence_store.expected_key(hashed_idempotency_key), + "Key": {"id": hashed_idempotency_key}, "ConsistentRead": True, } @@ -476,7 +433,7 @@ def test_idempotent_lambda_expired_during_request( stubber.activate() - @idempotent(config=idempotency_config, persistence_store=test_persistence_store.persistence_layer) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): return lambda_response @@ -491,7 +448,7 @@ def lambda_handler(event, context): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_persistence_exception_deleting( idempotency_config: IdempotencyConfig, - test_persistence_store: TestPersistenceStore, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, timestamp_future, lambda_response, @@ -503,21 +460,15 @@ def test_idempotent_persistence_exception_deleting( Test idempotent decorator when lambda is executed with an event with a previously unknown event key, but lambda_handler raises an exception which is retryable. """ - stubber = stub.Stubber(test_persistence_store.persistence_layer.table.meta.client) + stubber = stub.Stubber(persistence_store.table.meta.client) ddb_response = {} - stubber.add_response( - "put_item", - ddb_response, - expected_params_put_item( - test_persistence_store.key_attr, test_persistence_store.expected_key(hashed_idempotency_key) - ), - ) + stubber.add_response("put_item", ddb_response, expected_params_put_item) stubber.add_client_error("delete_item", "UnrecoverableError") stubber.activate() - @idempotent(config=idempotency_config, persistence_store=test_persistence_store.persistence_layer) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): raise Exception("Something went wrong!") @@ -532,7 +483,7 @@ def lambda_handler(event, context): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_persistence_exception_updating( idempotency_config: IdempotencyConfig, - test_persistence_store: TestPersistenceStore, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, timestamp_future, lambda_response, @@ -544,21 +495,15 @@ def test_idempotent_persistence_exception_updating( Test idempotent decorator when lambda is executed with an event with a previously unknown event key, but lambda_handler raises an exception which is retryable. """ - stubber = stub.Stubber(test_persistence_store.persistence_layer.table.meta.client) + stubber = stub.Stubber(persistence_store.table.meta.client) ddb_response = {} - stubber.add_response( - "put_item", - ddb_response, - expected_params_put_item( - test_persistence_store.key_attr, test_persistence_store.expected_key(hashed_idempotency_key) - ), - ) + stubber.add_response("put_item", ddb_response, expected_params_put_item) stubber.add_client_error("update_item", "UnrecoverableError") stubber.activate() - @idempotent(config=idempotency_config, persistence_store=test_persistence_store.persistence_layer) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): return {"message": "success!"} @@ -573,7 +518,7 @@ def lambda_handler(event, context): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_persistence_exception_getting( idempotency_config: IdempotencyConfig, - test_persistence_store: TestPersistenceStore, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, timestamp_future, lambda_response, @@ -585,13 +530,13 @@ def test_idempotent_persistence_exception_getting( Test idempotent decorator when lambda is executed with an event with a previously unknown event key, but lambda_handler raises an exception which is retryable. """ - stubber = stub.Stubber(test_persistence_store.persistence_layer.table.meta.client) + stubber = stub.Stubber(persistence_store.table.meta.client) stubber.add_client_error("put_item", "ConditionalCheckFailedException") stubber.add_client_error("get_item", "UnexpectedException") stubber.activate() - @idempotent(config=idempotency_config, persistence_store=test_persistence_store.persistence_layer) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): return {"message": "success!"} @@ -608,7 +553,7 @@ def lambda_handler(event, context): ) def test_idempotent_lambda_first_execution_with_validation( config_with_validation: IdempotencyConfig, - test_persistence_store: TestPersistenceStore, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, expected_params_update_item_with_validation, expected_params_put_item_with_validation, @@ -620,24 +565,14 @@ def test_idempotent_lambda_first_execution_with_validation( """ Test idempotent decorator when lambda is executed with an event with a previously unknown event key """ - stubber = stub.Stubber(test_persistence_store.persistence_layer.table.meta.client) + stubber = stub.Stubber(persistence_store.table.meta.client) ddb_response = {} - stubber.add_response( - "put_item", - ddb_response, - expected_params_put_item_with_validation( - test_persistence_store.key_attr, test_persistence_store.expected_key(hashed_idempotency_key) - ), - ) - stubber.add_response( - "update_item", - ddb_response, - expected_params_update_item_with_validation(test_persistence_store.expected_key(hashed_idempotency_key)), - ) + stubber.add_response("put_item", ddb_response, expected_params_put_item_with_validation) + stubber.add_response("update_item", ddb_response, expected_params_update_item_with_validation) stubber.activate() - @idempotent(config=config_with_validation, persistence_store=test_persistence_store.persistence_layer) + @idempotent(config=config_with_validation, persistence_store=persistence_store) def lambda_handler(event, context): return lambda_response @@ -652,7 +587,7 @@ def lambda_handler(event, context): ) def test_idempotent_lambda_with_validator_util( config_without_jmespath: IdempotencyConfig, - test_persistence_store: TestPersistenceStore, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, timestamp_future, serialized_lambda_response, @@ -666,10 +601,10 @@ def test_idempotent_lambda_with_validator_util( validator utility to unwrap the event """ - stubber = stub.Stubber(test_persistence_store.persistence_layer.table.meta.client) + stubber = stub.Stubber(persistence_store.table.meta.client) ddb_response = { "Item": { - **test_persistence_store.expected_key_values(hashed_idempotency_key_with_envelope), + "id": {"S": hashed_idempotency_key_with_envelope}, "expiration": {"N": timestamp_future}, "data": {"S": serialized_lambda_response}, "status": {"S": "COMPLETED"}, @@ -678,7 +613,7 @@ def test_idempotent_lambda_with_validator_util( expected_params = { "TableName": TABLE_NAME, - "Key": test_persistence_store.expected_key(hashed_idempotency_key_with_envelope), + "Key": {"id": hashed_idempotency_key_with_envelope}, "ConsistentRead": True, } stubber.add_client_error("put_item", "ConditionalCheckFailedException") @@ -686,7 +621,7 @@ def test_idempotent_lambda_with_validator_util( stubber.activate() @validator(envelope=envelopes.API_GATEWAY_HTTP) - @idempotent(config=config_without_jmespath, persistence_store=test_persistence_store.persistence_layer) + @idempotent(config=config_without_jmespath, persistence_store=persistence_store) def lambda_handler(event, context): mock_function() return "shouldn't get here!" @@ -709,50 +644,50 @@ def test_data_record_invalid_status_value(): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) def test_in_progress_never_saved_to_cache( - idempotency_config: IdempotencyConfig, test_persistence_store: TestPersistenceStore + idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer ): # GIVEN a data record with status "INPROGRESS" - # and test_persistence_store.persistence_layerhas use_local_cache = True - test_persistence_store.persistence_layer.configure(idempotency_config) + # and persistence_store has use_local_cache = True + persistence_store.configure(idempotency_config) data_record = DataRecord("key", status="INPROGRESS") # WHEN saving to local cache - test_persistence_store.persistence_layer._save_to_cache(data_record) + persistence_store._save_to_cache(data_record) # THEN don't save to local cache - assert test_persistence_store.persistence_layer._cache.get("key") is None + assert persistence_store._cache.get("key") is None @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}], indirect=True) -def test_user_local_disabled(idempotency_config: IdempotencyConfig, test_persistence_store: TestPersistenceStore): - # GIVEN a test_persistence_store.persistence_layerwith use_local_cache = False - test_persistence_store.persistence_layer.configure(idempotency_config) +def test_user_local_disabled(idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer): + # GIVEN a persistence_store with use_local_cache = False + persistence_store.configure(idempotency_config) # WHEN calling any local cache options data_record = DataRecord("key", status="COMPLETED") try: - test_persistence_store.persistence_layer._save_to_cache(data_record) - cache_value = test_persistence_store.persistence_layer._retrieve_from_cache("key") + persistence_store._save_to_cache(data_record) + cache_value = persistence_store._retrieve_from_cache("key") assert cache_value is None - test_persistence_store.persistence_layer._delete_from_cache("key") + persistence_store._delete_from_cache("key") except AttributeError as e: pytest.fail(f"AttributeError should not be raised: {e}") # THEN raise AttributeError # AND don't have a _cache attribute - assert not hasattr("test_persistence_store.persistence_layer", "_cache") + assert not hasattr("persistence_store", "_cache") @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) def test_delete_from_cache_when_empty( - idempotency_config: IdempotencyConfig, test_persistence_store: TestPersistenceStore + idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer ): # GIVEN use_local_cache is True AND the local cache is empty - test_persistence_store.persistence_layer.configure(idempotency_config) + persistence_store.configure(idempotency_config) try: # WHEN we _delete_from_cache - test_persistence_store.persistence_layer._delete_from_cache("key_does_not_exist") + persistence_store._delete_from_cache("key_does_not_exist") except KeyError: # THEN we should not get a KeyError pytest.fail("KeyError should not happen") @@ -797,15 +732,15 @@ def test_is_missing_idempotency_key(): "idempotency_config", [{"use_local_cache": False, "event_key_jmespath": "body"}], indirect=True ) def test_default_no_raise_on_missing_idempotency_key( - idempotency_config: IdempotencyConfig, test_persistence_store: TestPersistenceStore, lambda_context + idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, lambda_context ): - # GIVEN a test_persistence_store.persistence_layerwith use_local_cache = False and event_key_jmespath = "body" - test_persistence_store.persistence_layer.configure(idempotency_config) - assert test_persistence_store.persistence_layer.use_local_cache is False - assert "body" in test_persistence_store.persistence_layer.event_key_jmespath + # GIVEN a persistence_store with use_local_cache = False and event_key_jmespath = "body" + persistence_store.configure(idempotency_config) + assert persistence_store.use_local_cache is False + assert "body" in persistence_store.event_key_jmespath # WHEN getting the hashed idempotency key for an event with no `body` key - hashed_key = test_persistence_store.persistence_layer._get_hashed_idempotency_key({}) + hashed_key = persistence_store._get_hashed_idempotency_key({}) # THEN return the hash of None expected_value = "test-func#" + md5(serialize(None).encode()).hexdigest() @@ -816,18 +751,17 @@ def test_default_no_raise_on_missing_idempotency_key( "idempotency_config", [{"use_local_cache": False, "event_key_jmespath": "[body, x]"}], indirect=True ) def test_raise_on_no_idempotency_key( - idempotency_config: IdempotencyConfig, test_persistence_store: TestPersistenceStore, lambda_context + idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, lambda_context ): - # GIVEN a test_persistence_store.persistence_layer with raise_on_no_idempotency_key - # and no idempotency key in the request - test_persistence_store.persistence_layer.configure(idempotency_config) - test_persistence_store.persistence_layer.raise_on_no_idempotency_key = True - assert test_persistence_store.persistence_layer.use_local_cache is False - assert "body" in test_persistence_store.persistence_layer.event_key_jmespath + # GIVEN a persistence_store with raise_on_no_idempotency_key and no idempotency key in the request + persistence_store.configure(idempotency_config) + persistence_store.raise_on_no_idempotency_key = True + assert persistence_store.use_local_cache is False + assert "body" in persistence_store.event_key_jmespath # WHEN getting the hashed idempotency key for an event with no `body` key with pytest.raises(IdempotencyKeyError) as excinfo: - test_persistence_store.persistence_layer._get_hashed_idempotency_key({}) + persistence_store._get_hashed_idempotency_key({}) # THEN raise IdempotencyKeyError error assert "No data found to create a hashed idempotency_key" in str(excinfo.value) @@ -844,10 +778,10 @@ def test_raise_on_no_idempotency_key( indirect=True, ) def test_jmespath_with_powertools_json( - idempotency_config: IdempotencyConfig, test_persistence_store: TestPersistenceStore, lambda_context + idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, lambda_context ): # GIVEN an event_key_jmespath with powertools_json custom function - test_persistence_store.persistence_layer.configure(idempotency_config) + persistence_store.configure(idempotency_config) sub_attr_value = "cognito_user" key_attr_value = "some_key" expected_value = [sub_attr_value, key_attr_value] @@ -857,34 +791,34 @@ def test_jmespath_with_powertools_json( } # WHEN calling _get_hashed_idempotency_key - result = test_persistence_store.persistence_layer._get_hashed_idempotency_key(api_gateway_proxy_event) + result = persistence_store._get_hashed_idempotency_key(api_gateway_proxy_event) # THEN the hashed idempotency key should match the extracted values generated hash - assert result == "test-func#" + test_persistence_store.persistence_layer._generate_hash(expected_value) + assert result == "test-func#" + persistence_store._generate_hash(expected_value) @pytest.mark.parametrize("config_with_jmespath_options", ["powertools_json(data).payload"], indirect=True) def test_custom_jmespath_function_overrides_builtin_functions( - config_with_jmespath_options: IdempotencyConfig, test_persistence_store: TestPersistenceStore, lambda_context + config_with_jmespath_options: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer, lambda_context ): # GIVEN an persistence store with a custom jmespath_options # AND use a builtin powertools custom function - test_persistence_store.persistence_layer.configure(config_with_jmespath_options) + persistence_store.configure(config_with_jmespath_options) with pytest.raises(jmespath.exceptions.UnknownFunctionError, match="Unknown function: powertools_json()"): # WHEN calling _get_hashed_idempotency_key # THEN raise unknown function - test_persistence_store.persistence_layer._get_hashed_idempotency_key({}) + persistence_store._get_hashed_idempotency_key({}) -def test_idempotent_lambda_save_inprogress_error(test_persistence_store: TestPersistenceStore, lambda_context): +def test_idempotent_lambda_save_inprogress_error(persistence_store: DynamoDBPersistenceLayer, lambda_context): # GIVEN a miss configured persistence layer # like no table was created for the idempotency persistence layer - stubber = stub.Stubber(test_persistence_store.persistence_layer.table.meta.client) + stubber = stub.Stubber(persistence_store.table.meta.client) stubber.add_client_error("put_item", "ResourceNotFoundException") stubber.activate() - @idempotent(persistence_store=test_persistence_store.persistence_layer) + @idempotent(persistence_store=persistence_store) def lambda_handler(event, context): return {} @@ -899,13 +833,13 @@ def lambda_handler(event, context): assert "Failed to save in progress record to idempotency store" == e.value.args[0] -def test_handler_raise_idempotency_key_error(test_persistence_store: TestPersistenceStore, lambda_context): +def test_handler_raise_idempotency_key_error(persistence_store: DynamoDBPersistenceLayer, lambda_context): # GIVEN raise_on_no_idempotency_key is True idempotency_config = IdempotencyConfig(event_key_jmespath="idemKey", raise_on_no_idempotency_key=True) # WHEN handling the idempotent call # AND save_inprogress raises a IdempotencyKeyError - @idempotent(persistence_store=test_persistence_store.persistence_layer, config=idempotency_config) + @idempotent(persistence_store=persistence_store, config=idempotency_config) def handler(event, context): raise ValueError("Should not be raised") @@ -1063,23 +997,23 @@ def dummy(payload): dummy(payload=data_two) -def test_idempotency_disabled_envvar(monkeypatch, lambda_context, test_persistence_store: TestPersistenceStore): +def test_idempotency_disabled_envvar(monkeypatch, lambda_context, persistence_store: DynamoDBPersistenceLayer): # Scenario to validate no requests sent to dynamodb table when 'POWERTOOLS_IDEMPOTENCY_DISABLED' is set mock_event = {"data": "value"} - test_persistence_store.persistence_layer.table = MagicMock() + persistence_store.table = MagicMock() monkeypatch.setenv("POWERTOOLS_IDEMPOTENCY_DISABLED", "1") - @idempotent_function(data_keyword_argument="data", persistence_store=test_persistence_store.persistence_layer) + @idempotent_function(data_keyword_argument="data", persistence_store=persistence_store) def dummy(data): return {"message": "hello"} - @idempotent(persistence_store=test_persistence_store.persistence_layer) + @idempotent(persistence_store=persistence_store) def dummy_handler(event, context): return {"message": "hi"} dummy(data=mock_event) dummy_handler(mock_event, lambda_context) - assert len(test_persistence_store.persistence_layer.table.method_calls) == 0 + assert len(persistence_store.table.method_calls) == 0 From 1947829eccfc2fe2e904d8ae4af427542b4967c5 Mon Sep 17 00:00:00 2001 From: Adam Tankanow Date: Wed, 20 Oct 2021 21:13:58 -0400 Subject: [PATCH 3/4] ISSUE-694: PR Updates - remove test-func default name --- .../utilities/idempotency/persistence/dynamodb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py index 87aa524adcb..85ee311ca90 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py @@ -22,7 +22,7 @@ def __init__( self, table_name: str, key_attr: str = "id", - key_attr_value: str = f"idempotency#{os.getenv(constants.LAMBDA_FUNCTION_NAME_ENV, 'test-func')}", + key_attr_value: str = f"idempotency#{os.getenv(constants.LAMBDA_FUNCTION_NAME_ENV, '')}", sort_key_attr: Optional[str] = None, expiry_attr: str = "expiration", status_attr: str = "status", From 7f6091b284dfa934acec4cf1ccddbf63055fe0a8 Mon Sep 17 00:00:00 2001 From: Adam Tankanow Date: Wed, 3 Nov 2021 10:55:46 -0400 Subject: [PATCH 4/4] ISSUE-694: Change key_attr_value based on PR feedback --- .../utilities/idempotency/persistence/dynamodb.py | 8 ++++---- tests/functional/idempotency/test_idempotency.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py index 85ee311ca90..8a470c0f910 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py @@ -22,7 +22,7 @@ def __init__( self, table_name: str, key_attr: str = "id", - key_attr_value: str = f"idempotency#{os.getenv(constants.LAMBDA_FUNCTION_NAME_ENV, '')}", + static_pk_value: str = f"idempotency#{os.getenv(constants.LAMBDA_FUNCTION_NAME_ENV, '')}", sort_key_attr: Optional[str] = None, expiry_attr: str = "expiration", status_attr: str = "status", @@ -40,7 +40,7 @@ def __init__( Name of the table to use for storing execution records key_attr: str, optional DynamoDB attribute name for partition key, by default "id" - key_attr_value: str, optional + static_pk_value: str, optional DynamoDB attribute value for partition key, by default "idempotency#". This will be used if the sort_key_attr is set. sort_key_attr: str, optional @@ -79,7 +79,7 @@ def __init__( self._table = None self.table_name = table_name self.key_attr = key_attr - self.key_attr_value = key_attr_value + self.static_pk_value = static_pk_value self.sort_key_attr = sort_key_attr self.expiry_attr = expiry_attr self.status_attr = status_attr @@ -108,7 +108,7 @@ def table(self, table): def _get_key(self, idempotency_key: str) -> dict: if self.sort_key_attr: - return {self.key_attr: self.key_attr_value, self.sort_key_attr: idempotency_key} + return {self.key_attr: self.static_pk_value, self.sort_key_attr: idempotency_key} return {self.key_attr: idempotency_key} def _item_to_data_record(self, item: Dict[str, Any]) -> DataRecord: diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/test_idempotency.py index b1d0914d181..043fb06a04a 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/test_idempotency.py @@ -783,11 +783,11 @@ def test_jmespath_with_powertools_json( # GIVEN an event_key_jmespath with powertools_json custom function persistence_store.configure(idempotency_config) sub_attr_value = "cognito_user" - key_attr_value = "some_key" - expected_value = [sub_attr_value, key_attr_value] + static_pk_value = "some_key" + expected_value = [sub_attr_value, static_pk_value] api_gateway_proxy_event = { "requestContext": {"authorizer": {"claims": {"sub": sub_attr_value}}}, - "body": serialize({"id": key_attr_value}), + "body": serialize({"id": static_pk_value}), } # WHEN calling _get_hashed_idempotency_key