Skip to content

Commit ea91e5c

Browse files
theipsterleandrodamascena
theipster
and
leandrodamascena
authored
feat(parser): add support for SQS-wrapped S3 event notifications (#2108)
Co-authored-by: Leandro Damascena <leandro.damascena@gmail.com>
1 parent 7b158ef commit ea91e5c

File tree

7 files changed

+108
-23
lines changed

7 files changed

+108
-23
lines changed

‎aws_lambda_powertools/utilities/parser/models/__init__.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,10 @@
5050
S3Model,
5151
S3RecordModel,
5252
)
53+
from .s3_event_notification import (
54+
S3SqsEventNotificationModel,
55+
S3SqsEventNotificationRecordModel,
56+
)
5357
from .s3_object_event import (
5458
S3ObjectConfiguration,
5559
S3ObjectContext,
@@ -130,6 +134,8 @@
130134
"SqsRecordModel",
131135
"SqsMsgAttributeModel",
132136
"SqsAttributesModel",
137+
"S3SqsEventNotificationModel",
138+
"S3SqsEventNotificationRecordModel",
133139
"APIGatewayProxyEventModel",
134140
"APIGatewayEventRequestContext",
135141
"APIGatewayEventAuthorizer",

‎aws_lambda_powertools/utilities/parser/models/s3_event_notification.py

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
from typing import List
2+
3+
from pydantic import Json
4+
5+
from aws_lambda_powertools.utilities.parser.models.s3 import S3Model
6+
from aws_lambda_powertools.utilities.parser.models.sqs import SqsModel, SqsRecordModel
7+
8+
9+
class S3SqsEventNotificationRecordModel(SqsRecordModel):
10+
body: Json[S3Model]
11+
12+
13+
class S3SqsEventNotificationModel(SqsModel):
14+
Records: List[S3SqsEventNotificationRecordModel]

‎aws_lambda_powertools/utilities/parser/models/sqs.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
from datetime import datetime
2-
from typing import Dict, List, Optional, Type, Union
2+
from typing import Dict, List, Optional, Sequence, Type, Union
33

44
from pydantic import BaseModel
55

@@ -63,4 +63,4 @@ class SqsRecordModel(BaseModel):
6363

6464

6565
class SqsModel(BaseModel):
66-
Records: List[SqsRecordModel]
66+
Records: Sequence[SqsRecordModel]

‎docs/utilities/parser.md

Lines changed: 20 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -156,25 +156,26 @@ def my_function():
156156

157157
Parser comes with the following built-in models:
158158

159-
| Model name | Description |
160-
| --------------------------------------- | ---------------------------------------------------------------------------- |
161-
| **DynamoDBStreamModel** | Lambda Event Source payload for Amazon DynamoDB Streams |
162-
| **EventBridgeModel** | Lambda Event Source payload for Amazon EventBridge |
163-
| **SqsModel** | Lambda Event Source payload for Amazon SQS |
164-
| **AlbModel** | Lambda Event Source payload for Amazon Application Load Balancer |
165-
| **CloudwatchLogsModel** | Lambda Event Source payload for Amazon CloudWatch Logs |
166-
| **S3Model** | Lambda Event Source payload for Amazon S3 |
167-
| **S3ObjectLambdaEvent** | Lambda Event Source payload for Amazon S3 Object Lambda |
168-
| **S3EventNotificationEventBridgeModel** | Lambda Event Source payload for Amazon S3 Event Notification to EventBridge. |
169-
| **KinesisDataStreamModel** | Lambda Event Source payload for Amazon Kinesis Data Streams |
170-
| **KinesisFirehoseModel** | Lambda Event Source payload for Amazon Kinesis Firehose |
171-
| **SesModel** | Lambda Event Source payload for Amazon Simple Email Service |
172-
| **SnsModel** | Lambda Event Source payload for Amazon Simple Notification Service |
173-
| **APIGatewayProxyEventModel** | Lambda Event Source payload for Amazon API Gateway |
174-
| **APIGatewayProxyEventV2Model** | Lambda Event Source payload for Amazon API Gateway v2 payload |
175-
| **LambdaFunctionUrlModel** | Lambda Event Source payload for Lambda Function URL payload |
176-
| **KafkaSelfManagedEventModel** | Lambda Event Source payload for self managed Kafka payload |
177-
| **KafkaMskEventModel** | Lambda Event Source payload for AWS MSK payload |
159+
| Model name | Description |
160+
| --------------------------------------- | ------------------------------------------------------------------------------------- |
161+
| **AlbModel** | Lambda Event Source payload for Amazon Application Load Balancer |
162+
| **APIGatewayProxyEventModel** | Lambda Event Source payload for Amazon API Gateway |
163+
| **APIGatewayProxyEventV2Model** | Lambda Event Source payload for Amazon API Gateway v2 payload |
164+
| **CloudwatchLogsModel** | Lambda Event Source payload for Amazon CloudWatch Logs |
165+
| **DynamoDBStreamModel** | Lambda Event Source payload for Amazon DynamoDB Streams |
166+
| **EventBridgeModel** | Lambda Event Source payload for Amazon EventBridge |
167+
| **KafkaMskEventModel** | Lambda Event Source payload for AWS MSK payload |
168+
| **KafkaSelfManagedEventModel** | Lambda Event Source payload for self managed Kafka payload |
169+
| **KinesisDataStreamModel** | Lambda Event Source payload for Amazon Kinesis Data Streams |
170+
| **KinesisFirehoseModel** | Lambda Event Source payload for Amazon Kinesis Firehose |
171+
| **LambdaFunctionUrlModel** | Lambda Event Source payload for Lambda Function URL payload |
172+
| **S3EventNotificationEventBridgeModel** | Lambda Event Source payload for Amazon S3 Event Notification to EventBridge. |
173+
| **S3Model** | Lambda Event Source payload for Amazon S3 |
174+
| **S3ObjectLambdaEvent** | Lambda Event Source payload for Amazon S3 Object Lambda |
175+
| **S3SqsEventNotificationModel** | Lambda Event Source payload for S3 event notifications wrapped in SQS event (S3->SQS) |
176+
| **SesModel** | Lambda Event Source payload for Amazon Simple Email Service |
177+
| **SnsModel** | Lambda Event Source payload for Amazon Simple Notification Service |
178+
| **SqsModel** | Lambda Event Source payload for Amazon SQS |
178179

179180
#### Extending built-in models
180181

‎tests/events/s3SqsEvent.json

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
{
2+
"Records":[
3+
{
4+
"messageId":"ca3e7a89-c358-40e5-8aa0-5da01403c267",
5+
"receiptHandle":"AQEBE7XoI7IQRLF7SrpiW9W4BanmOWe8UtVDbv6/CEZYKf/OktSNIb4j689tQfR4k44V/LY20lZ5VpxYt2GTYCsSLKTcBalTJaRX9CKu/hVqy/23sSNiKxnP56D+VLSn+hU275+AP1h4pUL0d9gLdRB2haX8xiM+LcGfis5Jl8BBXtoxKRF60O87O9/NvCmmXLeqkJuexfyEZNyed0fFCRXFXSjbmThG0OIQgcrGI8glBRGPA8htns58VtXFsSaPYNoqP3p5n6+ewKKVLD0lfm+0DlnLKRa+mjvFBaSer9KK1ff+Aq6zJ6HynPwADj+aF70Hwimc2zImYe51SLEF/E2csYlMNZYI/2qXW0m9R7wJ/XDTV4g2+h+BMTxsKnJQ6NQd",
6+
"body":"{\"Records\":[{\"eventVersion\":\"2.1\",\"eventSource\":\"aws:s3\",\"awsRegion\":\"us-east-1\",\"eventTime\":\"2023-04-12T20:43:38.021Z\",\"eventName\":\"ObjectCreated:Put\",\"userIdentity\":{\"principalId\":\"A1YQ72UWCM96UF\"},\"requestParameters\":{\"sourceIPAddress\":\"93.108.161.96\"},\"responseElements\":{\"x-amz-request-id\":\"YMSSR8BZJ2Y99K6P\",\"x-amz-id-2\":\"6ASrUfj5xpn859fIq+6FXflOex/SKl/rjfiMd7wRzMg/zkHKR22PDpnh7KD3uq//cuOTbdX4DInN5eIs+cR0dY1z2Mc5NDP/\"},\"s3\":{\"s3SchemaVersion\":\"1.0\",\"configurationId\":\"SNS\",\"bucket\":{\"name\":\"xxx\",\"ownerIdentity\":{\"principalId\":\"A1YQ72UWCM96UF\"},\"arn\":\"arn:aws:s3:::xxx\"},\"object\":{\"key\":\"test.pdf\",\"size\":104681,\"eTag\":\"2e3ad1e983318bbd8e73b080e2997980\",\"versionId\":\"yd3d4HaWOT2zguDLvIQLU6ptDTwKBnQV\",\"sequencer\":\"00643717F9F8B85354\"}}}]}",
7+
"attributes":{
8+
"ApproximateReceiveCount":"1",
9+
"SentTimestamp":"1681332219270",
10+
"SenderId":"AIDAJHIPRHEMV73VRJEBU",
11+
"ApproximateFirstReceiveTimestamp":"1681332239270"
12+
},
13+
"messageAttributes":{
14+
15+
},
16+
"md5OfBody":"16f4460f4477d8d693a5abe94fdbbd73",
17+
"eventSource":"aws:sqs",
18+
"eventSourceARN":"arn:aws:sqs:us-east-1:123456789012:SQS",
19+
"awsRegion":"us-east-1"
20+
}
21+
]
22+
}

‎tests/functional/parser/test_s3.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,7 @@
66
from tests.functional.utils import load_event
77

88

9-
@event_parser(model=S3Model)
10-
def handle_s3(event: S3Model, _: LambdaContext):
9+
def assert_s3(event: S3Model):
1110
records = list(event.Records)
1211
assert len(records) == 1
1312
record: S3RecordModel = records[0]
@@ -41,6 +40,11 @@ def handle_s3(event: S3Model, _: LambdaContext):
4140
assert record.glacierEventData is None
4241

4342

43+
@event_parser(model=S3Model)
44+
def handle_s3(event: S3Model, _: LambdaContext):
45+
assert_s3(event)
46+
47+
4448
@event_parser(model=S3Model)
4549
def handle_s3_glacier(event: S3Model, _: LambdaContext):
4650
records = list(event.Records)

‎tests/unit/parser/test_s3.py

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,12 @@
1+
import json
12
from datetime import datetime
23

4+
import pytest
5+
6+
from aws_lambda_powertools.utilities.parser import ValidationError
37
from aws_lambda_powertools.utilities.parser.models import (
48
S3EventNotificationEventBridgeModel,
9+
S3SqsEventNotificationModel,
510
)
611
from tests.functional.utils import load_event
712

@@ -105,3 +110,36 @@ def test_s3_eventbridge_notification_object_restore_completed_event():
105110
assert model.detail.requester == raw_event["detail"]["requester"]
106111
assert model.detail.restore_expiry_time == raw_event["detail"]["restore-expiry-time"]
107112
assert model.detail.source_storage_class == raw_event["detail"]["source-storage-class"]
113+
114+
115+
def test_s3_sqs_event_notification():
116+
raw_event = load_event("s3SqsEvent.json")
117+
model = S3SqsEventNotificationModel(**raw_event)
118+
119+
body = json.loads(raw_event["Records"][0]["body"])
120+
121+
assert model.Records[0].body.Records[0].eventVersion == body["Records"][0]["eventVersion"]
122+
assert model.Records[0].body.Records[0].eventSource == body["Records"][0]["eventSource"]
123+
assert model.Records[0].body.Records[0].eventTime == datetime.fromisoformat(
124+
body["Records"][0]["eventTime"].replace("Z", "+00:00")
125+
)
126+
assert model.Records[0].body.Records[0].eventName == body["Records"][0]["eventName"]
127+
128+
129+
def test_s3_sqs_event_notification_body_invalid_json():
130+
raw_event = load_event("s3SqsEvent.json")
131+
132+
for record in raw_event["Records"]:
133+
record["body"] = "invalid body"
134+
135+
with pytest.raises(ValidationError):
136+
S3SqsEventNotificationModel(**raw_event)
137+
138+
139+
def test_s3_sqs_event_notification_body_containing_arbitrary_json():
140+
raw_event = load_event("s3SqsEvent.json")
141+
for record in raw_event["Records"]:
142+
record["body"] = {"foo": "bar"}
143+
144+
with pytest.raises(ValidationError):
145+
S3SqsEventNotificationModel(**raw_event)

0 commit comments

Comments
 (0)