Merge branch 'awslabs:develop' into fix-1025-v2

Author: Michael Brewer

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,25 +1,40 @@
-**Issue #, if available:**
+**Issue number:**
 
-## Description of changes:
+## Summary
 
-<!--- One or two sentences as a summary of what's being changed -->
+### Changes
 
-**Checklist**
+> Please provide a summary of what's being changed
 
-<!--- Leave unchecked if your change doesn't seem to apply --> 
+### User experience
+
+> Please share what the user experience looks like before and after this change
+
+## Checklist
+
+If your change doesn't seem to apply, please leave them unchecked.
 
 * [ ] [Meet tenets criteria](https://awslabs.github.io/aws-lambda-powertools-python/#tenets)
-* [ ] Update tests
-* [ ] Update docs
+* [ ] I have performed a self-review of my this change
+* [ ] Changes are tested
+* [ ] Changes are documented
 * [ ] PR title follows [conventional commit semantics](https://github.com/awslabs/aws-lambda-powertools-python/blob/376ec0a2ac0d2a40e0af5717bef42ff84ca0d1b9/.github/semantic.yml#L2)
 
-## Breaking change checklist
 
-<!--- Ignore if it's not a breaking change -->
+<details>
+<summary>Is this a breaking change?</summary>
+
+**RFC issue number**:
 
-**RFC issue #**:
+Checklist:
 
 * [ ] Migration process documented
 * [ ] Implement warnings (if it can live side by side)
 
+</details>
+
+## Acknowledgment
+
 By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.
+
+**Disclaimer**: We value your time and bandwidth. As such, any pull requests created on non-triaged issues might not be successful.

aws_lambda_powertools/event_handler/api_gateway.py

@@ -26,7 +26,7 @@
 _DYNAMIC_ROUTE_PATTERN = r"(<\w+>)"
 _SAFE_URI = "-._~()'!*:@,;"  # https://www.ietf.org/rfc/rfc3986.txt
 # API GW/ALB decode non-safe URI chars; we must support them too
-_UNSAFE_URI = "%<>\[\]{}|^"  # noqa: W605
+_UNSAFE_URI = "%<> \[\]{}|^"  # noqa: W605
 _NAMED_GROUP_BOUNDARY_PATTERN = fr"(?P\1[{_SAFE_URI}{_UNSAFE_URI}\\w]+)"
 
 

aws_lambda_powertools/logging/formatter.py

@@ -127,8 +127,8 @@ def __init__(
 
         super(LambdaPowertoolsFormatter, self).__init__(datefmt=self.datefmt)
 
-        keys_combined = {**self._build_default_keys(), **kwargs}
-        self.log_format.update(**keys_combined)
+        self.keys_combined = {**self._build_default_keys(), **kwargs}
+        self.log_format.update(**self.keys_combined)
 
     def serialize(self, log: Dict) -> str:
         """Serialize structured log dict to JSON str"""
@@ -187,7 +187,7 @@ def remove_keys(self, keys: Iterable[str]):
 
     def clear_state(self):
         self.log_format = dict.fromkeys(self.log_record_order)
-        self.log_format.update(**self._build_default_keys())
+        self.log_format.update(**self.keys_combined)
 
     @staticmethod
     def _build_default_keys():

aws_lambda_powertools/middleware_factory/factory.py

@@ -2,7 +2,7 @@
 import inspect
 import logging
 import os
-from typing import Callable, Optional
+from typing import Any, Callable, Optional
 
 from ..shared import constants
 from ..shared.functions import resolve_truthy_env_var_choice
@@ -12,7 +12,8 @@
 logger = logging.getLogger(__name__)
 
 
-def lambda_handler_decorator(decorator: Optional[Callable] = None, trace_execution: Optional[bool] = None):
+# Maintenance: we can't yet provide an accurate return type without ParamSpec etc. see #1066
+def lambda_handler_decorator(decorator: Optional[Callable] = None, trace_execution: Optional[bool] = None) -> Callable:
     """Decorator factory for decorating Lambda handlers.
 
     You can use lambda_handler_decorator to create your own middlewares,
@@ -110,7 +111,7 @@ def lambda_handler(event, context):
     )
 
     @functools.wraps(decorator)
-    def final_decorator(func: Optional[Callable] = None, **kwargs):
+    def final_decorator(func: Optional[Callable] = None, **kwargs: Any):
         # If called with kwargs return new func with kwargs
         if func is None:
             return functools.partial(final_decorator, **kwargs)

aws_lambda_powertools/utilities/idempotency/base.py

@@ -1,5 +1,6 @@
 import logging
 from typing import Any, Callable, Dict, Optional, Tuple
+from copy import deepcopy
 
 from aws_lambda_powertools.utilities.idempotency.config import IdempotencyConfig
 from aws_lambda_powertools.utilities.idempotency.exceptions import (
@@ -69,7 +70,7 @@ def __init__(
             Function keyword arguments
         """
         self.function = function
-        self.data = _prepare_data(function_payload)
+        self.data = deepcopy(_prepare_data(function_payload))
         self.fn_args = function_args
         self.fn_kwargs = function_kwargs
 

tests/functional/event_handler/test_api_gateway.py

@@ -715,6 +715,42 @@ def get_network_account(account_id: str, network_id: str):
     app.resolve(event, {})
 
 
+def test_similar_dynamic_routes_with_whitespaces():
+    # GIVEN
+    app = ApiGatewayResolver()
+    event = deepcopy(LOAD_GW_EVENT)
+
+    # WHEN
+    # r'^/accounts/(?P<account_id>\\w+\\b)$' # noqa: E800
+    @app.get("/accounts/<account_id>")
+    def get_account(account_id: str):
+        assert account_id == "single account"
+
+    # r'^/accounts/(?P<account_id>\\w+\\b)/source_networks$' # noqa: E800
+    @app.get("/accounts/<account_id>/source_networks")
+    def get_account_networks(account_id: str):
+        assert account_id == "nested account"
+
+    # r'^/accounts/(?P<account_id>\\w+\\b)/source_networks/(?P<network_id>\\w+\\b)$' # noqa: E800
+    @app.get("/accounts/<account_id>/source_networks/<network_id>")
+    def get_network_account(account_id: str, network_id: str):
+        assert account_id == "nested account"
+        assert network_id == "network 123"
+
+    # THEN
+    event["resource"] = "/accounts/{account_id}"
+    event["path"] = "/accounts/single account"
+    assert app.resolve(event, {})["statusCode"] == 200
+
+    event["resource"] = "/accounts/{account_id}/source_networks"
+    event["path"] = "/accounts/nested account/source_networks"
+    assert app.resolve(event, {})["statusCode"] == 200
+
+    event["resource"] = "/accounts/{account_id}/source_networks/{network_id}"
+    event["path"] = "/accounts/nested account/source_networks/network 123"
+    assert app.resolve(event, {})["statusCode"] == 200
+
+
 @pytest.mark.parametrize(
     "req",
     [

tests/functional/idempotency/conftest.py

@@ -14,7 +14,8 @@
 from aws_lambda_powertools.utilities.idempotency.idempotency import IdempotencyConfig
 from aws_lambda_powertools.utilities.jmespath_utils import extract_data_from_envelope
 from aws_lambda_powertools.utilities.validation import envelopes
-from tests.functional.utils import hash_idempotency_key, json_serialize, load_event
+from tests.functional.idempotency.utils import hash_idempotency_key
+from tests.functional.utils import json_serialize, load_event
 
 TABLE_NAME = "TEST_TABLE"
 

tests/functional/idempotency/test_idempotency.py

@@ -22,7 +22,12 @@
 from aws_lambda_powertools.utilities.idempotency.idempotency import idempotent, idempotent_function
 from aws_lambda_powertools.utilities.idempotency.persistence.base import BasePersistenceLayer, DataRecord
 from aws_lambda_powertools.utilities.validation import envelopes, validator
-from tests.functional.utils import hash_idempotency_key, json_serialize, load_event
+from tests.functional.idempotency.utils import (
+    build_idempotency_put_item_stub,
+    build_idempotency_update_item_stub,
+    hash_idempotency_key,
+)
+from tests.functional.utils import json_serialize, load_event
 
 TABLE_NAME = "TEST_TABLE"
 
@@ -275,6 +280,40 @@ def lambda_handler(event, context):
     stubber.deactivate()
 
 
+@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True, "event_key_jmespath": "body"}], indirect=True)
+def test_idempotent_lambda_first_execution_event_mutation(
+    idempotency_config: IdempotencyConfig,
+    persistence_store: DynamoDBPersistenceLayer,
+    lambda_apigw_event,
+    lambda_response,
+    lambda_context,
+):
+    """
+    Test idempotent decorator where lambda_handler mutates the event.
+    Ensures we're passing data by value, not reference.
+    """
+    event = copy.deepcopy(lambda_apigw_event)
+    stubber = stub.Stubber(persistence_store.table.meta.client)
+    ddb_response = {}
+    stubber.add_response("put_item", ddb_response, build_idempotency_put_item_stub(data=event["body"]))
+    stubber.add_response(
+        "update_item",
+        ddb_response,
+        build_idempotency_update_item_stub(data=event["body"], handler_response=lambda_response),
+    )
+    stubber.activate()
+
+    @idempotent(config=idempotency_config, persistence_store=persistence_store)
+    def lambda_handler(event, context):
+        event.pop("body")  # remove exact key we're using for idempotency
+        return lambda_response
+
+    lambda_handler(event, lambda_context)
+
+    stubber.assert_no_pending_responses()
+    stubber.deactivate()
+
+
 @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True)
 def test_idempotent_lambda_expired(
     idempotency_config: IdempotencyConfig,

tests/functional/idempotency/utils.py

@@ -0,0 +1,45 @@
+import hashlib
+from typing import Any, Dict
+
+from botocore import stub
+
+from tests.functional.utils import json_serialize
+
+
+def hash_idempotency_key(data: Any):
+    """Serialize data to JSON, encode, and hash it for idempotency key"""
+    return hashlib.md5(json_serialize(data).encode()).hexdigest()
+
+
+def build_idempotency_put_item_stub(
+    data: Dict, function_name: str = "test-func", handler_name: str = "lambda_handler"
+) -> Dict:
+    idempotency_key_hash = f"{function_name}.{handler_name}#{hash_idempotency_key(data)}"
+    return {
+        "ConditionExpression": "attribute_not_exists(#id) OR #now < :now",
+        "ExpressionAttributeNames": {"#id": "id", "#now": "expiration"},
+        "ExpressionAttributeValues": {":now": stub.ANY},
+        "Item": {"expiration": stub.ANY, "id": idempotency_key_hash, "status": "INPROGRESS"},
+        "TableName": "TEST_TABLE",
+    }
+
+
+def build_idempotency_update_item_stub(
+    data: Dict,
+    handler_response: Dict,
+    function_name: str = "test-func",
+    handler_name: str = "lambda_handler",
+) -> Dict:
+    idempotency_key_hash = f"{function_name}.{handler_name}#{hash_idempotency_key(data)}"
+    serialized_lambda_response = json_serialize(handler_response)
+    return {
+        "ExpressionAttributeNames": {"#expiry": "expiration", "#response_data": "data", "#status": "status"},
+        "ExpressionAttributeValues": {
+            ":expiry": stub.ANY,
+            ":response_data": serialized_lambda_response,
+            ":status": "COMPLETED",
+        },
+        "Key": {"id": idempotency_key_hash},
+        "TableName": "TEST_TABLE",
+        "UpdateExpression": "SET #response_data = :response_data, " "#expiry = :expiry, #status = :status",
+    }

tests/functional/test_logger.py

@@ -683,6 +683,26 @@ def handler(event, context):
         assert key in second_log
 
 
+def test_clear_state_keeps_custom_keys(lambda_context, stdout, service_name):
+    # GIVEN
+    location_format = "%(module)s.%(funcName)s:clear_state"
+    logger = Logger(service=service_name, stream=stdout, location=location_format, custom_key="foo")
+
+    # WHEN clear_state is set
+    @logger.inject_lambda_context(clear_state=True)
+    def handler(event, context):
+        logger.info("Foo")
+
+    # THEN all standard keys should be available as usual
+    handler({}, lambda_context)
+    handler({}, lambda_context)
+
+    first_log, second_log = capture_multiple_logging_statements_output(stdout)
+    for log in (first_log, second_log):
+        assert "foo" == log["custom_key"]
+        assert "test_logger.handler:clear_state" == log["location"]
+
+
 def test_clear_state_keeps_exception_keys(lambda_context, stdout, service_name):
     # GIVEN
     logger = Logger(service=service_name, stream=stdout)

tests/functional/utils.py

@@ -1,5 +1,4 @@
 import base64
-import hashlib
 import json
 from pathlib import Path
 from typing import Any
@@ -22,8 +21,3 @@ def b64_to_str(data: str) -> str:
 
 def json_serialize(data):
     return json.dumps(data, sort_keys=True, cls=Encoder)
-
-
-def hash_idempotency_key(data: Any):
-    """Serialize data to JSON, encode, and hash it for idempotency key"""
-    return hashlib.md5(json_serialize(data).encode()).hexdigest()