From 534d45eb5d33d243603deaf27ea69e9cd4339d29 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Wed, 3 May 2023 15:07:32 +0100 Subject: [PATCH 1/6] feat(event_source): adding support to custom json decode --- .../utilities/data_classes/common.py | 7 ++-- .../utilities/data_classes/kafka_event.py | 5 +-- .../data_classes/kinesis_firehose_event.py | 5 +-- .../utilities/data_classes/sqs_event.py | 12 +++++- tests/functional/test_data_classes.py | 40 +++++++++++++++++++ 5 files changed, 59 insertions(+), 10 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_classes/common.py b/aws_lambda_powertools/utilities/data_classes/common.py index 5c1fea14731..ddfd4910a5a 100644 --- a/aws_lambda_powertools/utilities/data_classes/common.py +++ b/aws_lambda_powertools/utilities/data_classes/common.py @@ -1,7 +1,7 @@ import base64 import json from collections.abc import Mapping -from typing import Any, Dict, Iterator, List, Optional +from typing import Any, Callable, Dict, Iterator, List, Optional from aws_lambda_powertools.shared.headers_serializer import BaseHeadersSerializer @@ -9,9 +9,10 @@ class DictWrapper(Mapping): """Provides a single read only access to a wrapper dict""" - def __init__(self, data: Dict[str, Any]): + def __init__(self, data: Dict[str, Any], json_deserializer: Optional[Callable] = None): self._data = data self._json_data: Optional[Any] = None + self._json_deserializer = json_deserializer or json.loads def __getitem__(self, key: str) -> Any: return self._data[key] @@ -122,7 +123,7 @@ def body(self) -> Optional[str]: def json_body(self) -> Any: """Parses the submitted body as json""" if self._json_data is None: - self._json_data = json.loads(self.decoded_body) + self._json_data = self._json_deserializer(self.decoded_body) return self._json_data @property diff --git a/aws_lambda_powertools/utilities/data_classes/kafka_event.py b/aws_lambda_powertools/utilities/data_classes/kafka_event.py index e52cc5d8dc1..4773d9e50de 100644 --- a/aws_lambda_powertools/utilities/data_classes/kafka_event.py +++ b/aws_lambda_powertools/utilities/data_classes/kafka_event.py @@ -1,5 +1,4 @@ import base64 -import json from typing import Any, Dict, Iterator, List, Optional from aws_lambda_powertools.utilities.data_classes.common import DictWrapper @@ -55,7 +54,7 @@ def decoded_value(self) -> bytes: def json_value(self) -> Any: """Decodes the text encoded data as JSON.""" if self._json_data is None: - self._json_data = json.loads(self.decoded_value.decode("utf-8")) + self._json_data = self._json_deserializer(self.decoded_value.decode("utf-8")) return self._json_data @property @@ -117,7 +116,7 @@ def records(self) -> Iterator[KafkaEventRecord]: """The Kafka records.""" for chunk in self["records"].values(): for record in chunk: - yield KafkaEventRecord(record) + yield KafkaEventRecord(data=record, json_deserializer=self._json_deserializer) @property def record(self) -> KafkaEventRecord: diff --git a/aws_lambda_powertools/utilities/data_classes/kinesis_firehose_event.py b/aws_lambda_powertools/utilities/data_classes/kinesis_firehose_event.py index 5683902f9d0..47dc196856d 100644 --- a/aws_lambda_powertools/utilities/data_classes/kinesis_firehose_event.py +++ b/aws_lambda_powertools/utilities/data_classes/kinesis_firehose_event.py @@ -1,5 +1,4 @@ import base64 -import json from typing import Iterator, Optional from aws_lambda_powertools.utilities.data_classes.common import DictWrapper @@ -75,7 +74,7 @@ def data_as_text(self) -> str: def data_as_json(self) -> dict: """Decoded base64-encoded data loaded to json""" if self._json_data is None: - self._json_data = json.loads(self.data_as_text) + self._json_data = self._json_deserializer(self.data_as_text) return self._json_data @@ -110,4 +109,4 @@ def region(self) -> str: @property def records(self) -> Iterator[KinesisFirehoseRecord]: for record in self["records"]: - yield KinesisFirehoseRecord(record) + yield KinesisFirehoseRecord(data=record, json_deserializer=self._json_deserializer) diff --git a/aws_lambda_powertools/utilities/data_classes/sqs_event.py b/aws_lambda_powertools/utilities/data_classes/sqs_event.py index 7d0dbe49352..1d9b1fb2952 100644 --- a/aws_lambda_powertools/utilities/data_classes/sqs_event.py +++ b/aws_lambda_powertools/utilities/data_classes/sqs_event.py @@ -103,6 +103,16 @@ def body(self) -> str: """The message's contents (not URL-encoded).""" return self["body"] + @property + def json_body(self) -> Dict: + """Parses the submitted body as json""" + try: + if self._json_data is None: + self._json_data = self._json_deserializer(self["body"]) + return self._json_data + except Exception: + return self["body"] + @property def attributes(self) -> SQSRecordAttributes: """A map of the attributes requested in ReceiveMessage to their respective values.""" @@ -157,4 +167,4 @@ class SQSEvent(DictWrapper): @property def records(self) -> Iterator[SQSRecord]: for record in self["Records"]: - yield SQSRecord(record) + yield SQSRecord(data=record, json_deserializer=self._json_deserializer) diff --git a/tests/functional/test_data_classes.py b/tests/functional/test_data_classes.py index 068e8738fad..74e345b8a27 100644 --- a/tests/functional/test_data_classes.py +++ b/tests/functional/test_data_classes.py @@ -113,6 +113,46 @@ def message(self) -> str: assert DataClassSample(data1).raw_event is data1 +def test_dict_wrapper_with_default_custom_json_deserializer(): + class DataClassSample(DictWrapper): + @property + def json_body(self) -> dict: + return self._json_deserializer(self["body"]) + + data = {"body": '{"message": "foo1"}'} + event = DataClassSample(data=data) + assert (event.json_body) == {"message": "foo1"} + + +def test_dict_wrapper_with_valid_custom_json_deserializer(): + class DataClassSample(DictWrapper): + @property + def json_body(self) -> dict: + return self._json_deserializer(self["body"]) + + def fake_json_deserializer(record: dict): + return json.loads(record) + + data = {"body": '{"message": "foo1"}'} + event = DataClassSample(data=data, json_deserializer=fake_json_deserializer) + assert (event.json_body) == {"message": "foo1"} + + +def test_dict_wrapper_with_wrong_custom_json_deserializer(): + class DataClassSample(DictWrapper): + @property + def json_body(self) -> dict: + return self._json_deserializer(self["body"]) + + def fake_json_deserializer() -> None: + pass + + data = {"body": {"message": "foo1"}} + with pytest.raises(TypeError): + event = DataClassSample(data=data, json_deserializer=fake_json_deserializer) + assert (event.json_body) == {"message": "foo1"} + + def test_dict_wrapper_implements_mapping(): class DataClassSample(DictWrapper): pass From 4ae10effdbf11fafc4c6734c2315145ee4891a20 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Wed, 3 May 2023 15:39:01 +0100 Subject: [PATCH 2/6] feat(event_source): fix code coverage --- tests/events/sqsEvent.json | 4 ++-- tests/functional/test_data_classes.py | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/events/sqsEvent.json b/tests/events/sqsEvent.json index ef03b128943..2bfcd1c7b8f 100644 --- a/tests/events/sqsEvent.json +++ b/tests/events/sqsEvent.json @@ -25,7 +25,7 @@ { "messageId": "2e1424d4-f796-459a-8184-9c92662be6da", "receiptHandle": "AQEBzWwaftRI0KuVm4tP+/7q1rGgNqicHq...", - "body": "Test message2.", + "body": "{\"message\": \"foo1\"}", "attributes": { "ApproximateReceiveCount": "1", "SentTimestamp": "1545082650636", @@ -39,4 +39,4 @@ "awsRegion": "us-east-2" } ] -} \ No newline at end of file +} diff --git a/tests/functional/test_data_classes.py b/tests/functional/test_data_classes.py index 74e345b8a27..34c6339a327 100644 --- a/tests/functional/test_data_classes.py +++ b/tests/functional/test_data_classes.py @@ -966,6 +966,9 @@ def test_seq_trigger_event(): assert record.queue_url == "https://sqs.us-east-2.amazonaws.com/123456789012/my-queue" assert record.aws_region == "us-east-2" + record_2 = records[1] + assert record_2.json_body == {"message": "foo1"} + def test_default_api_gateway_proxy_event(): event = APIGatewayProxyEvent(load_event("apiGatewayProxyEvent_noVersionAuth.json")) From fbbae513c2ad11dde377ab11f9f1edfe42e7c2af Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Thu, 4 May 2023 10:34:41 +0200 Subject: [PATCH 3/6] chore: add docstring to DictWrapper parameters --- aws_lambda_powertools/utilities/data_classes/common.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/aws_lambda_powertools/utilities/data_classes/common.py b/aws_lambda_powertools/utilities/data_classes/common.py index ddfd4910a5a..d1ce8f90a07 100644 --- a/aws_lambda_powertools/utilities/data_classes/common.py +++ b/aws_lambda_powertools/utilities/data_classes/common.py @@ -10,6 +10,15 @@ class DictWrapper(Mapping): """Provides a single read only access to a wrapper dict""" def __init__(self, data: Dict[str, Any], json_deserializer: Optional[Callable] = None): + """ + Parameters + ---------- + data : Dict[str, Any] + Lambda Event Source Event payload + json_deserializer : Callable, optional + function to deserialize `str`, `bytes`, bytearray` containing a JSON document to a Python `obj`, + by default json.loads + """ self._data = data self._json_data: Optional[Any] = None self._json_deserializer = json_deserializer or json.loads From f7b166fda70548d3af51f01d7e6b0bc1a4377cc8 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Thu, 4 May 2023 10:59:44 +0200 Subject: [PATCH 4/6] fix: prevent error handling surprises --- .../utilities/data_classes/sqs_event.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_classes/sqs_event.py b/aws_lambda_powertools/utilities/data_classes/sqs_event.py index 1d9b1fb2952..eaf2566a242 100644 --- a/aws_lambda_powertools/utilities/data_classes/sqs_event.py +++ b/aws_lambda_powertools/utilities/data_classes/sqs_event.py @@ -106,12 +106,9 @@ def body(self) -> str: @property def json_body(self) -> Dict: """Parses the submitted body as json""" - try: - if self._json_data is None: - self._json_data = self._json_deserializer(self["body"]) - return self._json_data - except Exception: - return self["body"] + if self._json_data is None: + self._json_data = self._json_deserializer(self["body"]) + return self._json_data @property def attributes(self) -> SQSRecordAttributes: From 708a7e6e6d69abae2e9ecad28add90fa00db1499 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Thu, 4 May 2023 13:45:34 +0200 Subject: [PATCH 5/6] chore(typing): use any to not get in customers' way; docstring for reasoning --- .../utilities/data_classes/sqs_event.py | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_classes/sqs_event.py b/aws_lambda_powertools/utilities/data_classes/sqs_event.py index eaf2566a242..2b3224358d8 100644 --- a/aws_lambda_powertools/utilities/data_classes/sqs_event.py +++ b/aws_lambda_powertools/utilities/data_classes/sqs_event.py @@ -1,4 +1,4 @@ -from typing import Dict, Iterator, Optional +from typing import Any, Dict, Iterator, Optional from aws_lambda_powertools.utilities.data_classes.common import DictWrapper @@ -104,8 +104,30 @@ def body(self) -> str: return self["body"] @property - def json_body(self) -> Dict: - """Parses the submitted body as json""" + def json_body(self) -> Any: + """Deserializes JSON string available in 'body' property + + Notes + ----- + + **Strict typing** + + Caller controls the type as we can't use recursive generics here. + + JSON Union types would force caller to have to cast a type. Instead, + we choose Any to ease ergonomics and other tools receiving this data. + + Examples + -------- + + **Type deserialized data from JSON string** + + ```python + data: dict = record.json_body # {"telemetry": [], ...} + # or + data: list = record.json_body # ["telemetry_values"] + ``` + """ if self._json_data is None: self._json_data = self._json_deserializer(self["body"]) return self._json_data From ec681ec4e5c37f3254e39384b908de3963ae81d5 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Thu, 4 May 2023 14:08:02 +0200 Subject: [PATCH 6/6] chore: peer review before merging --- tests/functional/test_data_classes.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/functional/test_data_classes.py b/tests/functional/test_data_classes.py index 34c6339a327..b3a24b0865a 100644 --- a/tests/functional/test_data_classes.py +++ b/tests/functional/test_data_classes.py @@ -121,7 +121,7 @@ def json_body(self) -> dict: data = {"body": '{"message": "foo1"}'} event = DataClassSample(data=data) - assert (event.json_body) == {"message": "foo1"} + assert event.json_body == json.loads(data["body"]) def test_dict_wrapper_with_valid_custom_json_deserializer(): @@ -135,22 +135,23 @@ def fake_json_deserializer(record: dict): data = {"body": '{"message": "foo1"}'} event = DataClassSample(data=data, json_deserializer=fake_json_deserializer) - assert (event.json_body) == {"message": "foo1"} + assert event.json_body == json.loads(data["body"]) -def test_dict_wrapper_with_wrong_custom_json_deserializer(): +def test_dict_wrapper_with_invalid_custom_json_deserializer(): class DataClassSample(DictWrapper): @property def json_body(self) -> dict: return self._json_deserializer(self["body"]) def fake_json_deserializer() -> None: + # invalid fn signature should raise TypeError pass data = {"body": {"message": "foo1"}} with pytest.raises(TypeError): event = DataClassSample(data=data, json_deserializer=fake_json_deserializer) - assert (event.json_body) == {"message": "foo1"} + assert event.json_body == {"message": "foo1"} def test_dict_wrapper_implements_mapping():