From e9edf986c3e5b417869fafb5d9083070126949ec Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Wed, 13 Jan 2021 12:03:35 +0100 Subject: [PATCH 1/7] tests: add benchmark on AWS Lambda --- benchmark/.gitignore | 1 + benchmark/benchmark.sh | 77 +++++++++++++++++++++ benchmark/src/instrumented/main.py | 17 +++++ benchmark/src/instrumented/requirements.txt | 1 + benchmark/src/reference/main.py | 4 ++ benchmark/src/reference/requirements.txt | 1 + benchmark/template.yaml | 48 +++++++++++++ 7 files changed, 149 insertions(+) create mode 100644 benchmark/.gitignore create mode 100755 benchmark/benchmark.sh create mode 100644 benchmark/src/instrumented/main.py create mode 100644 benchmark/src/instrumented/requirements.txt create mode 100644 benchmark/src/reference/main.py create mode 100644 benchmark/src/reference/requirements.txt create mode 100644 benchmark/template.yaml diff --git a/benchmark/.gitignore b/benchmark/.gitignore new file mode 100644 index 00000000000..592fb594cbf --- /dev/null +++ b/benchmark/.gitignore @@ -0,0 +1 @@ +.aws-sam \ No newline at end of file diff --git a/benchmark/benchmark.sh b/benchmark/benchmark.sh new file mode 100755 index 00000000000..9856b9390f3 --- /dev/null +++ b/benchmark/benchmark.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +set -e +trap cleanup EXIT + +export BENCHMARK_STACK_NAME=${BENCHMARK_STACK_NAME:-"powertools-benchmark"} + +function cleanup { + echo "Cleaning up stack..." + aws cloudformation delete-stack --stack-name $BENCHMARK_STACK_NAME +} + +function run_function { + # Update function to force a cold start + aws lambda update-function-configuration --function-name $1 --memory-size 256 >/dev/null + aws lambda update-function-configuration --function-name $1 --memory-size 128 >/dev/null + # Cold-start invoke + aws lambda invoke --function-name $1 --payload '{}' /dev/null >/dev/null && echo -n . || echo -n e +} + +# Retrieve statistics +function get_stats { + # Gather results from CloudWatch Logs Insights + query_id=$(aws logs start-query --log-group-name $1 --query-string 'filter @type = "REPORT" | stats avg(@initDuration) as init_duration, avg(@duration) as duration' --start-time $(expr $(date +%s) - 86400) --end-time $(expr $(date +%s) + 0) --query 'queryId' --output text) + while true; do + result=$(aws logs get-query-results --query-id $query_id --query 'status' --output text) + if [ $result == "Complete" ]; then + break + fi + sleep 1 + done + + # Check if greater than threshold and print result + init_duration=$(aws logs get-query-results --query-id $query_id --query 'results[0][?field==`init_duration`].value' --output text) + duration=$(aws logs get-query-results --query-id $query_id --query 'results[0][?field==`duration`].value' --output text) + echo "$init_duration,$duration" +} + +# Build and deploy the benchmark stack +echo "Building and deploying..." +sam build +sam deploy --stack-name $BENCHMARK_STACK_NAME --guided + +# Retrieve output values +echo "Retrieve values..." +export INSTRUMENTED_FUNCTION=$(aws cloudformation describe-stacks --stack-name $BENCHMARK_STACK_NAME --query 'Stacks[0].Outputs[?OutputKey==`InstrumentedFunction`].OutputValue' --output text) +export REFERENCE_FUNCTION=$(aws cloudformation describe-stacks --stack-name $BENCHMARK_STACK_NAME --query 'Stacks[0].Outputs[?OutputKey==`ReferenceFunction`].OutputValue' --output text) +export INSTRUMENTED_LOG_GROUP=$(aws cloudformation describe-stacks --stack-name $BENCHMARK_STACK_NAME --query 'Stacks[0].Outputs[?OutputKey==`InstrumentedLogGroup`].OutputValue' --output text) +export REFERENCE_LOG_GROUP=$(aws cloudformation describe-stacks --stack-name $BENCHMARK_STACK_NAME --query 'Stacks[0].Outputs[?OutputKey==`ReferenceLogGroup`].OutputValue' --output text) + +echo INSTRUMENTED_FUNCTION=$INSTRUMENTED_FUNCTION +echo REFERENCE_FUNCTION=$REFERENCE_FUNCTION +echo INSTRUMENTED_LOG_GROUP=$INSTRUMENTED_LOG_GROUP +echo REFERENCE_LOG_GROUP=$REFERENCE_LOG_GROUP + +# Running cold starts +echo "Running functions..." +for i in {0..20}; do + run_function $INSTRUMENTED_FUNCTION +done & +process_id=$! +for i in {0..20}; do + run_function $REFERENCE_FUNCTION +done & +wait $process_id +wait $! +echo + +# Gather statistics +sleep 150 +return_code=0 +echo -n "INSTRUMENTED=" +get_stats $INSTRUMENTED_LOG_GROUP +echo -n "REFERENCE=" +get_stats $REFERENCE_LOG_GROUP + +exit $return_code \ No newline at end of file diff --git a/benchmark/src/instrumented/main.py b/benchmark/src/instrumented/main.py new file mode 100644 index 00000000000..e26d9326c26 --- /dev/null +++ b/benchmark/src/instrumented/main.py @@ -0,0 +1,17 @@ +from aws_lambda_powertools import (Logger, Metrics, Tracer) + + +# Initialize core utilities +logger = Logger() +metrics = Metrics() +tracer = Tracer() + + +# Instrument Lambda function +@logger.inject_lambda_context +@metrics.log_metrics +@tracer.capture_lambda_handler +def handler(event, context): + return { + "message": "success" + } \ No newline at end of file diff --git a/benchmark/src/instrumented/requirements.txt b/benchmark/src/instrumented/requirements.txt new file mode 100644 index 00000000000..ed0b17144f4 --- /dev/null +++ b/benchmark/src/instrumented/requirements.txt @@ -0,0 +1 @@ +aws-lambda-powertools \ No newline at end of file diff --git a/benchmark/src/reference/main.py b/benchmark/src/reference/main.py new file mode 100644 index 00000000000..4b5fb3900a7 --- /dev/null +++ b/benchmark/src/reference/main.py @@ -0,0 +1,4 @@ +def handler(event, context): + return { + "message": "success" + } \ No newline at end of file diff --git a/benchmark/src/reference/requirements.txt b/benchmark/src/reference/requirements.txt new file mode 100644 index 00000000000..ed0b17144f4 --- /dev/null +++ b/benchmark/src/reference/requirements.txt @@ -0,0 +1 @@ +aws-lambda-powertools \ No newline at end of file diff --git a/benchmark/template.yaml b/benchmark/template.yaml new file mode 100644 index 00000000000..b88e1aa4b8e --- /dev/null +++ b/benchmark/template.yaml @@ -0,0 +1,48 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 + +Globals: + Function: + Handler: main.handler + Runtime: python3.8 + MemorySize: 128 + Tracing: Active + Environment: + Variables: + POWERTOOLS_SERVICE_NAME: benchmark + POWERTOOLS_METRICS_NAMESPACE: LambdaPowertools + POWERTOOLS_LOGGER_LOG_EVENT: "true" + LOG_LEVEL: INFO + +Resources: + InstrumentedFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: ./src/instrumented/ + + ReferenceFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: ./src/reference/ + + InstrumentedLogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: !Sub "/aws/lambda/${InstrumentedFunction}" + RetentionInDays: 7 + + ReferenceLogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: !Sub "/aws/lambda/${ReferenceFunction}" + RetentionInDays: 7 + +Outputs: + InstrumentedFunction: + Value: !Ref InstrumentedFunction + ReferenceFunction: + Value: !Ref ReferenceFunction + InstrumentedLogGroup: + Value: !Ref InstrumentedLogGroup + ReferenceLogGroup: + Value: !Ref ReferenceLogGroup \ No newline at end of file From 8c19b861817732fe929c3a78c84d9c951c4795d8 Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Wed, 13 Jan 2021 14:37:27 +0100 Subject: [PATCH 2/7] changing from average to p50 --- benchmark/benchmark.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/benchmark.sh b/benchmark/benchmark.sh index 9856b9390f3..ed7d809d9d8 100755 --- a/benchmark/benchmark.sh +++ b/benchmark/benchmark.sh @@ -21,7 +21,7 @@ function run_function { # Retrieve statistics function get_stats { # Gather results from CloudWatch Logs Insights - query_id=$(aws logs start-query --log-group-name $1 --query-string 'filter @type = "REPORT" | stats avg(@initDuration) as init_duration, avg(@duration) as duration' --start-time $(expr $(date +%s) - 86400) --end-time $(expr $(date +%s) + 0) --query 'queryId' --output text) + query_id=$(aws logs start-query --log-group-name $1 --query-string 'filter @type = "REPORT" | stats pct(@initDuration, 50) as init_duration, pct(@duration, 50) as duration' --start-time $(expr $(date +%s) - 86400) --end-time $(expr $(date +%s) + 0) --query 'queryId' --output text) while true; do result=$(aws logs get-query-results --query-id $query_id --query 'status' --output text) if [ $result == "Complete" ]; then From bf5aeb03bd6f7851b0ddc6c0579306984c6027e3 Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Fri, 15 Jan 2021 11:19:13 +0100 Subject: [PATCH 3/7] tests: add README to benchmark --- benchmark/README.md | 20 ++++++++++++++++++++ benchmark/benchmark.sh | 4 +++- 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 benchmark/README.md diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 00000000000..ac8d58b178f --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,20 @@ +# Cold Start Benchmark + +The [benchmark.sh script](./benchmark.sh) is a bash script to compare the cold-start time of using the AWS Lambda Powertools in a semi-automated way. It does so by deploying two Lambda functions which both have the aws-lambda-powertools module installed. One Lambda function will import and initialize the three main utilities (`Metrics`, `Logger`, `Tracer`), while the other one will not. + +## Usage + +To use the script, you should move into the benchmark folder and run the benchmark script: + +``` +cd benchmark +./benchmark.sh +``` + +This will: + +* Deploy a CloudFormation stack using guided SAM deployment (*you will need to answer a few questions*). +* Run loops to update the memory setting of the functions to force a cold start, then invoke them. This process is repeated a number of time to get more consistent results. +* Wait for the data to propagate from CloudWatch Logs to CloudWatch Logs Insights. +* Run a query on CloudWatch Logs insights, looking at the **REPORT** line from the logs. +* Delete the CloudFormation stack. diff --git a/benchmark/benchmark.sh b/benchmark/benchmark.sh index ed7d809d9d8..1a79a9ca9e4 100755 --- a/benchmark/benchmark.sh +++ b/benchmark/benchmark.sh @@ -67,6 +67,8 @@ wait $! echo # Gather statistics +# Waiting 2.5 minutes to make sure the data propagates from CloudWatch Logs +# into CloudWatch Logs Insights. sleep 150 return_code=0 echo -n "INSTRUMENTED=" @@ -74,4 +76,4 @@ get_stats $INSTRUMENTED_LOG_GROUP echo -n "REFERENCE=" get_stats $REFERENCE_LOG_GROUP -exit $return_code \ No newline at end of file +exit $return_code From 84555a119a849c1c68acf055a7c51df799a31d4b Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Fri, 22 Jan 2021 14:43:24 +0100 Subject: [PATCH 4/7] test: use S3_BUCKET environment variable for benchmark --- benchmark/README.md | 4 ++++ benchmark/benchmark.sh | 14 +++++++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/benchmark/README.md b/benchmark/README.md index ac8d58b178f..66c037a9d2d 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -2,11 +2,15 @@ The [benchmark.sh script](./benchmark.sh) is a bash script to compare the cold-start time of using the AWS Lambda Powertools in a semi-automated way. It does so by deploying two Lambda functions which both have the aws-lambda-powertools module installed. One Lambda function will import and initialize the three main utilities (`Metrics`, `Logger`, `Tracer`), while the other one will not. +Please note that this requires the [SAM CLI](https://github.com/aws/aws-sam-cli) version 1.2.0 or later. + ## Usage To use the script, you should move into the benchmark folder and run the benchmark script: ``` +export S3_BUCKET=code-artifact-s3-bucket + cd benchmark ./benchmark.sh ``` diff --git a/benchmark/benchmark.sh b/benchmark/benchmark.sh index 1a79a9ca9e4..7bcf182c185 100755 --- a/benchmark/benchmark.sh +++ b/benchmark/benchmark.sh @@ -3,6 +3,11 @@ set -e trap cleanup EXIT +if [ -z "S3_BUCKET" ]; then + echo "Missing S3_BUCKET environment variabe" + exit 1 +fi + export BENCHMARK_STACK_NAME=${BENCHMARK_STACK_NAME:-"powertools-benchmark"} function cleanup { @@ -39,7 +44,7 @@ function get_stats { # Build and deploy the benchmark stack echo "Building and deploying..." sam build -sam deploy --stack-name $BENCHMARK_STACK_NAME --guided +sam deploy --stack-name $BENCHMARK_STACK_NAME --s3-bucket $S3_BUCKET --capabilities CAPABILITY_IAM # Retrieve output values echo "Retrieve values..." @@ -69,11 +74,10 @@ echo # Gather statistics # Waiting 2.5 minutes to make sure the data propagates from CloudWatch Logs # into CloudWatch Logs Insights. +echo "Waiting for data to propagate in CloudWatch Logs Insights..." sleep 150 return_code=0 -echo -n "INSTRUMENTED=" -get_stats $INSTRUMENTED_LOG_GROUP -echo -n "REFERENCE=" -get_stats $REFERENCE_LOG_GROUP +echo "INSTRUMENTED=$(get_stats $INSTRUMENTED_LOG_GROUP)" +echo "REFERENCE=$(get_stats $REFERENCE_LOG_GROUP)" exit $return_code From 42ed507687c2dba58467eb46748cc600a0a6ae7c Mon Sep 17 00:00:00 2001 From: Heitor Lessa Date: Tue, 9 Feb 2021 09:58:17 +0100 Subject: [PATCH 5/7] docs: core utilities instead of main utilities --- benchmark/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/README.md b/benchmark/README.md index 66c037a9d2d..c9ae020b29a 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -1,6 +1,6 @@ # Cold Start Benchmark -The [benchmark.sh script](./benchmark.sh) is a bash script to compare the cold-start time of using the AWS Lambda Powertools in a semi-automated way. It does so by deploying two Lambda functions which both have the aws-lambda-powertools module installed. One Lambda function will import and initialize the three main utilities (`Metrics`, `Logger`, `Tracer`), while the other one will not. +The [benchmark.sh script](./benchmark.sh) is a bash script to compare the cold-start time of using the AWS Lambda Powertools in a semi-automated way. It does so by deploying two Lambda functions which both have the aws-lambda-powertools module installed. One Lambda function will import and initialize the three core utilities (`Metrics`, `Logger`, `Tracer`), while the other one will not. Please note that this requires the [SAM CLI](https://github.com/aws/aws-sam-cli) version 1.2.0 or later. From 1648b12cbc8148128a5f4a7653a33a8964df9a5e Mon Sep 17 00:00:00 2001 From: Heitor Lessa Date: Tue, 9 Feb 2021 09:58:48 +0100 Subject: [PATCH 6/7] docs: share fixed ETA on data propagation --- benchmark/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/README.md b/benchmark/README.md index c9ae020b29a..b97635859c8 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -19,6 +19,6 @@ This will: * Deploy a CloudFormation stack using guided SAM deployment (*you will need to answer a few questions*). * Run loops to update the memory setting of the functions to force a cold start, then invoke them. This process is repeated a number of time to get more consistent results. -* Wait for the data to propagate from CloudWatch Logs to CloudWatch Logs Insights. +* Wait 2.5 minutes to ensure data propagates from CloudWatch Logs to CloudWatch Logs Insights. * Run a query on CloudWatch Logs insights, looking at the **REPORT** line from the logs. * Delete the CloudFormation stack. From 88bcf96af7d17e81f793db57654c93e692dded2c Mon Sep 17 00:00:00 2001 From: Heitor Lessa Date: Tue, 9 Feb 2021 09:59:09 +0100 Subject: [PATCH 7/7] docs: note on systems we expect it to run plus charges --- benchmark/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/benchmark/README.md b/benchmark/README.md index b97635859c8..84b48129d94 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -6,6 +6,8 @@ Please note that this requires the [SAM CLI](https://github.com/aws/aws-sam-cli) ## Usage +> **NOTE**: This script is expected to run in Unix-based systems only, and can incur charges on your AWS account. + To use the script, you should move into the benchmark folder and run the benchmark script: ```