diff --git a/.evergreen/config.yml b/.evergreen/config.yml index ffe6664439..345bb59eb4 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1303,6 +1303,33 @@ task_groups: tasks: - testazurekms-task + - name: test_aws_lambda_task_group + setup_group: + - func: fetch source + - func: prepare resources + - command: subprocess.exec + params: + working_dir: src + binary: bash + add_expansions_to_env: true + args: + - ${DRIVERS_TOOLS}/.evergreen/atlas/setup-atlas-cluster.sh + - command: expansions.update + params: + file: src/atlas-expansion.yml + teardown_group: + - command: subprocess.exec + params: + working_dir: src + binary: bash + add_expansions_to_env: true + args: + - ${DRIVERS_TOOLS}/.evergreen/atlas/teardown-atlas-cluster.sh + setup_group_can_fail_task: true + setup_group_timeout_secs: 1800 + tasks: + - test-aws-lambda-deployed + - name: test_atlas_task_group_search_indexes setup_group: - func: fetch source @@ -1785,6 +1812,23 @@ tasks: vars: TEST_DATA_LAKE: "true" + - name: "test-aws-lambda-deployed" + commands: + - func: "install dependencies" + - command: ec2.assume_role + params: + role_arn: ${LAMBDA_AWS_ROLE_ARN} + duration_seconds: 3600 + - command: subprocess.exec + params: + working_dir: src + binary: bash + add_expansions_to_env: true + args: + - .evergreen/run-deployed-lambda-aws-tests.sh + env: + TEST_LAMBDA_DIRECTORY: ${PROJECT_DIRECTORY}/test/lambda + - name: test-ocsp-rsa-valid-cert-server-staples tags: ["ocsp", "ocsp-rsa", "ocsp-staple"] commands: @@ -3358,6 +3402,12 @@ buildvariants: batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README - testazurekms-fail-task +- name: rhel8-test-lambda + display_name: AWS Lambda handler tests + run_on: rhel87-small + tasks: + - name: test_aws_lambda_task_group + - name: Release display_name: Release batchtime: 20160 # 14 days diff --git a/.evergreen/run-deployed-lambda-aws-tests.sh b/.evergreen/run-deployed-lambda-aws-tests.sh new file mode 100644 index 0000000000..aa16d62650 --- /dev/null +++ b/.evergreen/run-deployed-lambda-aws-tests.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -o errexit # Exit the script with error if any of the commands fail + +export PATH="/opt/python/3.9/bin:${PATH}" +python --version +pushd ./test/lambda + +. build.sh +popd +. ${DRIVERS_TOOLS}/.evergreen/aws_lambda/run-deployed-lambda-aws-tests.sh diff --git a/.gitignore b/.gitignore index 269a7e7081..3096d460ba 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,10 @@ mongocryptd.pid .idea/ .nova/ venv/ + +# Lambda temp files +test/lambda/.aws-sam +test/lambda/env.json +test/lambda/mongodb/pymongo/* +test/lambda/mongodb/gridfs/* +test/lambda/mongodb/bson/* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f19f15682c..44d9c78ba6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,6 +7,7 @@ repos: - id: check-case-conflict - id: check-toml - id: check-yaml + exclude: template.yaml - id: debug-statements - id: end-of-file-fixer exclude: WHEEL diff --git a/test/lambda/README.md b/test/lambda/README.md new file mode 100644 index 0000000000..2727a2cee9 --- /dev/null +++ b/test/lambda/README.md @@ -0,0 +1,17 @@ +AWS Lambda Testing +------------------ + +Running locally +=============== + +Prerequisites: + +- AWS SAM CLI +- Docker daemon running + +Usage +===== + +- Start a local mongodb instance on port 27017 +- Run ``build.sh`` +- Run ``test.sh`` diff --git a/test/lambda/build.sh b/test/lambda/build.sh new file mode 100755 index 0000000000..c7cc24eab2 --- /dev/null +++ b/test/lambda/build.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -o errexit # Exit the script with error if any of the commands fail +set -o xtrace + +rm -rf mongodb/pymongo +rm -rf mongodb/gridfs +rm -rf mongodb/bson + +pushd ../.. +rm -f pymongo/*.so +rm -f bson/*.so +image="quay.io/pypa/manylinux2014_x86_64:latest" + +DOCKER=$(command -v docker) || true +if [ -z "$DOCKER" ]; then + PODMAN=$(command -v podman) || true + if [ -z "$PODMAN" ]; then + echo "docker or podman are required!" + exit 1 + fi + DOCKER=podman +fi + +$DOCKER run --rm -v "`pwd`:/src" $image /src/test/lambda/build_internal.sh +cp -r pymongo ./test/lambda/mongodb/pymongo +cp -r bson ./test/lambda/mongodb/bson +cp -r gridfs ./test/lambda/mongodb/gridfs +popd diff --git a/test/lambda/build_internal.sh b/test/lambda/build_internal.sh new file mode 100755 index 0000000000..fec488d32c --- /dev/null +++ b/test/lambda/build_internal.sh @@ -0,0 +1,5 @@ +#!/bin/bash -ex + +cd /src +PYTHON=/opt/python/cp39-cp39/bin/python +$PYTHON -m pip install -v -e . diff --git a/test/lambda/events/event.json b/test/lambda/events/event.json new file mode 100644 index 0000000000..a6197dea6c --- /dev/null +++ b/test/lambda/events/event.json @@ -0,0 +1,62 @@ +{ + "body": "{\"message\": \"hello world\"}", + "resource": "/hello", + "path": "/hello", + "httpMethod": "GET", + "isBase64Encoded": false, + "queryStringParameters": { + "foo": "bar" + }, + "pathParameters": { + "proxy": "/path/to/resource" + }, + "stageVariables": { + "baz": "qux" + }, + "headers": { + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Accept-Encoding": "gzip, deflate, sdch", + "Accept-Language": "en-US,en;q=0.8", + "Cache-Control": "max-age=0", + "CloudFront-Forwarded-Proto": "https", + "CloudFront-Is-Desktop-Viewer": "true", + "CloudFront-Is-Mobile-Viewer": "false", + "CloudFront-Is-SmartTV-Viewer": "false", + "CloudFront-Is-Tablet-Viewer": "false", + "CloudFront-Viewer-Country": "US", + "Host": "1234567890.execute-api.us-east-1.amazonaws.com", + "Upgrade-Insecure-Requests": "1", + "User-Agent": "Custom User Agent String", + "Via": "1.1 08f323deadbeefa7af34d5feb414ce27.cloudfront.net (CloudFront)", + "X-Amz-Cf-Id": "cDehVQoZnx43VYQb9j2-nvCh-9z396Uhbp027Y2JvkCPNLmGJHqlaA==", + "X-Forwarded-For": "127.0.0.1, 127.0.0.2", + "X-Forwarded-Port": "443", + "X-Forwarded-Proto": "https" + }, + "requestContext": { + "accountId": "123456789012", + "resourceId": "123456", + "stage": "prod", + "requestId": "c6af9ac6-7b61-11e6-9a41-93e8deadbeef", + "requestTime": "09/Apr/2015:12:34:56 +0000", + "requestTimeEpoch": 1428582896000, + "identity": { + "cognitoIdentityPoolId": null, + "accountId": null, + "cognitoIdentityId": null, + "caller": null, + "accessKey": null, + "sourceIp": "127.0.0.1", + "cognitoAuthenticationType": null, + "cognitoAuthenticationProvider": null, + "userArn": null, + "userAgent": "Custom User Agent String", + "user": null + }, + "path": "/prod/hello", + "resourcePath": "/hello", + "httpMethod": "POST", + "apiId": "1234567890", + "protocol": "HTTP/1.1" + } +} diff --git a/test/lambda/mongodb/Makefile b/test/lambda/mongodb/Makefile new file mode 100644 index 0000000000..3632dfb161 --- /dev/null +++ b/test/lambda/mongodb/Makefile @@ -0,0 +1,4 @@ + +build-MongoDBFunction: + cp -r . $(ARTIFACTS_DIR) + python -m pip install -t $(ARTIFACTS_DIR) dnspython diff --git a/test/lambda/mongodb/__init__.py b/test/lambda/mongodb/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/lambda/mongodb/app.py b/test/lambda/mongodb/app.py new file mode 100644 index 0000000000..66c2164672 --- /dev/null +++ b/test/lambda/mongodb/app.py @@ -0,0 +1,149 @@ +""" +Lambda function for Python Driver testing + +Creates the client that is cached for all requests, subscribes to +relevant events, and forces the connection pool to get populated. +""" +import json +import os + +from bson import has_c as has_bson_c +from pymongo import MongoClient +from pymongo import has_c as has_pymongo_c +from pymongo.monitoring import ( + CommandListener, + ConnectionPoolListener, + ServerHeartbeatListener, +) + +open_connections = 0 +heartbeat_count = 0 +total_heartbeat_duration = 0 +total_commands = 0 +total_command_duration = 0 + +# Ensure we are using C extensions +assert has_bson_c() +assert has_pymongo_c() + + +class CommandHandler(CommandListener): + def started(self, event): + print("command started", event) + + def succeeded(self, event): + global total_commands, total_command_duration + total_commands += 1 + total_command_duration += event.duration_micros / 1e6 + print("command succeeded", event) + + def failed(self, event): + global total_commands, total_command_duration + total_commands += 1 + total_command_duration += event.duration_micros / 1e6 + print("command failed", event) + + +class ServerHeartbeatHandler(ServerHeartbeatListener): + def started(self, event): + print("server heartbeat started", event) + + def succeeded(self, event): + global heartbeat_count, total_heartbeat_duration + heartbeat_count += 1 + total_heartbeat_duration += event.duration + print("server heartbeat succeeded", event) + + def failed(self, event): + global heartbeat_count, total_heartbeat_duration + heartbeat_count += 1 + total_heartbeat_duration += event.duration + print("server heartbeat failed", event) + + +class ConnectionHandler(ConnectionPoolListener): + def connection_created(self, event): + global open_connections + open_connections += 1 + print("connection created") + + def connection_ready(self, event): + pass + + def connection_closed(self, event): + global open_connections + open_connections -= 1 + print("connection closed") + + def connection_check_out_started(self, event): + pass + + def connection_check_out_failed(self, event): + pass + + def connection_checked_out(self, event): + pass + + def connection_checked_in(self, event): + pass + + def pool_created(self, event): + pass + + def pool_ready(self, event): + pass + + def pool_cleared(self, event): + pass + + def pool_closed(self, event): + pass + + +listeners = [CommandHandler(), ServerHeartbeatHandler(), ConnectionHandler()] +print("Creating client") +client = MongoClient(os.environ["MONGODB_URI"], event_listeners=listeners) + + +# Populate the connection pool. +print("Connecting") +client.lambdaTest.list_collections() +print("Connected") + + +# Create the response to send back. +def create_response(): + return dict( + averageCommandDuration=total_command_duration / total_commands, + averageHeartbeatDuration=total_heartbeat_duration / heartbeat_count, + openConnections=open_connections, + heartbeatCount=heartbeat_count, + ) + + +# Reset the numbers. +def reset(): + global open_connections, heartbeat_count, total_heartbeat_duration, total_commands, total_command_duration + open_connections = 0 + heartbeat_count = 0 + total_heartbeat_duration = 0 + total_commands = 0 + total_command_duration = 0 + + +def lambda_handler(event, context): + """ + The handler function itself performs an insert/delete and returns the + id of the document in play. + """ + print("initializing") + db = client.lambdaTest + collection = db.test + result = collection.insert_one({"n": 1}) + collection.delete_one({"_id": result.inserted_id}) + # Create the response and then reset the numbers. + response = json.dumps(create_response()) + reset() + print("finished!") + + return dict(statusCode=200, body=response) diff --git a/test/lambda/run.sh b/test/lambda/run.sh new file mode 100755 index 0000000000..5f1980a5f9 --- /dev/null +++ b/test/lambda/run.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -o errexit # Exit the script with error if any of the commands fail + +sam build +sam local invoke --docker-network host --parameter-overrides "MongoDbUri=mongodb://host.docker.internal:27017" diff --git a/test/lambda/template.yaml b/test/lambda/template.yaml new file mode 100644 index 0000000000..651ac4a8f8 --- /dev/null +++ b/test/lambda/template.yaml @@ -0,0 +1,49 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: > + Python driver lambda function test + +# More info about Globals: https://github.com/awslabs/serverless-application-model/blob/master/docs/globals.rst +Globals: + Function: + Timeout: 30 + MemorySize: 128 + +Parameters: + MongoDbUri: + Type: String + Description: The MongoDB connection string. + +Resources: + MongoDBFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: mongodb/ + Environment: + Variables: + MONGODB_URI: !Ref MongoDbUri + Handler: app.lambda_handler + Runtime: python3.9 + Architectures: + - x86_64 + Events: + MongoDB: + Type: Api + Properties: + Path: /mongodb + Method: get + # Use a custom build method to make sure *.so files are copied. + # https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/building-custom-runtimes.html + Metadata: + BuildMethod: makefile + +Outputs: + MongoDBApi: + Description: "API Gateway endpoint URL for Prod stage for Python driver lambda function" + Value: !Sub "https://${ServerlessRestApi}.execute-api.${AWS::Region}.amazonaws.com/Prod/hello/" + MongoDBFunction: + Description: "Python driver lambda Function ARN" + Value: !GetAtt MongoDBFunction.Arn + MongoDBFunctionIamRole: + Description: "Implicit IAM Role created for Python driver lambda function" + Value: !GetAtt MongoDBFunctionRole.Arn