Add support for Python 3.9 (#164)

Author: nhinsch

.github/workflows/build.yml

@@ -13,7 +13,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.7
+          python-version: 3.9
 
       - name: Install dependencies
         run: |
@@ -37,7 +37,7 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        python-version: [2.7, 3.6, 3.7, 3.8]
+        python-version: [2.7, 3.6, 3.7, 3.8, 3.9]
 
     steps:
       - name: Checkout
@@ -64,7 +64,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        runtime-param: [2.7, 3.6, 3.7, 3.8]
+        runtime-param: [2.7, 3.6, 3.7, 3.8, 3.9]
     steps:
       - name: Checkout
         uses: actions/checkout@v2

README.md

@@ -6,7 +6,7 @@
 [![Slack](https://chat.datadoghq.com/badge.svg?bg=632CA6)](https://chat.datadoghq.com/)
 [![License](https://img.shields.io/badge/license-Apache--2.0-blue)](https://github.com/DataDog/datadog-lambda-python/blob/main/LICENSE)
 
-Datadog Lambda Library for Python (2.7, 3.6, 3.7 and 3.8) enables enhanced Lambda metrics, distributed tracing, and custom metric submission from AWS Lambda functions.  
+Datadog Lambda Library for Python (2.7, 3.6, 3.7, 3.8, and 3.9) enables enhanced Lambda metrics, distributed tracing, and custom metric submission from AWS Lambda functions.  
 
 **IMPORTANT NOTE:** AWS Lambda is expected to receive a [breaking change](https://aws.amazon.com/blogs/compute/upcoming-changes-to-the-python-sdk-in-aws-lambda/) on **December 1, 2021**. If you are using Datadog Python Lambda layer version 7 or below, please upgrade to the latest.
 

scripts/add_new_region.sh

@@ -12,8 +12,8 @@ set -e
 
 OLD_REGION='us-east-1'
 
-PYTHON_VERSIONS_FOR_AWS_CLI=("python2.7" "python3.6" "python3.7" "python3.8")
-LAYER_NAMES=("Datadog-Python27" "Datadog-Python36" "Datadog-Python37" "Datadog-Python38")
+PYTHON_VERSIONS_FOR_AWS_CLI=("python2.7" "python3.6" "python3.7" "python3.8" "python3.9")
+LAYER_NAMES=("Datadog-Python27" "Datadog-Python36" "Datadog-Python37" "Datadog-Python38" "Datadog-Python39")
 NEW_REGION=$1
 
 publish_layer() {

scripts/build_layers.sh

@@ -14,7 +14,7 @@ set -e
 
 LAYER_DIR=".layers"
 LAYER_FILES_PREFIX="datadog_lambda_py"
-AVAILABLE_PYTHON_VERSIONS=("2.7" "3.6" "3.7" "3.8")
+AVAILABLE_PYTHON_VERSIONS=("2.7" "3.6" "3.7" "3.8" "3.9")
 
 # Determine which Python versions to build layers for
 if [ -z "$PYTHON_VERSION" ]; then

scripts/check_layer_size.sh

@@ -14,7 +14,7 @@ MAX_LAYER_UNCOMPRESSED_SIZE_KB=$(expr 16 \* 1024)
 
 LAYER_FILES_PREFIX="datadog_lambda_py"
 LAYER_DIR=".layers"
-VERSIONS=("2.7" "3.6" "3.7" "3.8")
+VERSIONS=("2.7" "3.6" "3.7" "3.8" "3.9")
 
 for version in "${VERSIONS[@]}"
 do

scripts/list_layers.sh

@@ -10,7 +10,7 @@
 
 set -e
 
-LAYER_NAMES=("Datadog-Python27" "Datadog-Python36" "Datadog-Python37" "Datadog-Python38")
+LAYER_NAMES=("Datadog-Python27" "Datadog-Python36" "Datadog-Python37" "Datadog-Python38" "Datadog-Python39")
 AVAILABLE_REGIONS=$(aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName')
 LAYERS_MISSING_REGIONS=()
 

scripts/publish_layers.sh

@@ -13,9 +13,9 @@ set -e
 # Makes sure any subprocesses will be terminated with this process
 trap "pkill -P $$; exit 1;" INT
 
-PYTHON_VERSIONS_FOR_AWS_CLI=("python2.7" "python3.6" "python3.7" "python3.8")
-LAYER_PATHS=(".layers/datadog_lambda_py2.7.zip" ".layers/datadog_lambda_py3.6.zip" ".layers/datadog_lambda_py3.7.zip" ".layers/datadog_lambda_py3.8.zip")
-AVAILABLE_LAYERS=("Datadog-Python27" "Datadog-Python36" "Datadog-Python37" "Datadog-Python38")
+PYTHON_VERSIONS_FOR_AWS_CLI=("python2.7" "python3.6" "python3.7" "python3.8" "python3.9")
+LAYER_PATHS=(".layers/datadog_lambda_py2.7.zip" ".layers/datadog_lambda_py3.6.zip" ".layers/datadog_lambda_py3.7.zip" ".layers/datadog_lambda_py3.8.zip" ".layers/datadog_lambda_py3.9.zip")
+AVAILABLE_LAYERS=("Datadog-Python27" "Datadog-Python36" "Datadog-Python37" "Datadog-Python38" "Datadog-Python39")
 AVAILABLE_REGIONS=$(aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName')
 
 # Check that the layer files exist

scripts/run_integration_tests.sh

@@ -11,7 +11,7 @@ set -e
 # These values need to be in sync with serverless.yml, where there needs to be a function
 # defined for every handler_runtime combination
 LAMBDA_HANDLERS=("async-metrics" "sync-metrics")
-RUNTIMES=("python27" "python36" "python37" "python38")
+RUNTIMES=("python27" "python36" "python37" "python38" "python39")
 
 LOGS_WAIT_SECONDS=20
 
@@ -32,8 +32,9 @@ python27=("python2.7" "2.7" $(xxd -l 4 -c 4 -p < /dev/random))
 python36=("python3.6" "3.6" $(xxd -l 4 -c 4 -p < /dev/random))
 python37=("python3.7" "3.7" $(xxd -l 4 -c 4 -p < /dev/random))
 python38=("python3.8" "3.8" $(xxd -l 4 -c 4 -p < /dev/random))
+python39=("python3.9" "3.9" $(xxd -l 4 -c 4 -p < /dev/random))
 
-PARAMETERS_SETS=("python27" "python36" "python37" "python38")
+PARAMETERS_SETS=("python27" "python36" "python37" "python38" "python39")
 
 if [ -z "$RUNTIME_PARAM" ]; then
     echo "Python version not specified, running for all python versions."

scripts/run_tests.sh

@@ -8,7 +8,7 @@
 # Run unit tests in Docker
 set -e
 
-PYTHON_VERSIONS=("2.7" "3.6" "3.7" "3.8")
+PYTHON_VERSIONS=("2.7" "3.6" "3.7" "3.8" "3.9")
 
 for python_version in "${PYTHON_VERSIONS[@]}"
 do

scripts/sign_layers.sh

@@ -13,6 +13,7 @@ LAYER_FILES=(
     "datadog_lambda_py3.6.zip"
     "datadog_lambda_py3.7.zip"
     "datadog_lambda_py3.8.zip"
+    "datadog_lambda_py3.9.zip"
 )
 SIGNING_PROFILE_NAME="DatadogLambdaSigningProfile"
 

setup.py

@@ -23,6 +23,7 @@
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
     ],
     keywords="datadog aws lambda layer",
     packages=["datadog_lambda"],

tests/integration/serverless.yml

@@ -1,6 +1,12 @@
 # IAM permissions require service name to begin with 'integration-tests'
 service: integration-tests-python
 
+# As of mid-August 2021, the Serverless framework does not support Python 3.9
+# and complains about an invalid configuration when we deploy Python 3.9 functions.
+# This option suppresses the warning.
+# Remove this when the Serverless framework supports Python 3.9.
+configValidationMode: off
+
 provider:
   name: aws
   region: sa-east-1

tests/integration/snapshots/logs/async-metrics_python39.log

@@ -0,0 +1,27 @@
+START RequestId: XXXX Version: $LATEST
+{"m": "aws.lambda.enhanced.invocations", "v": 1, "e": XXXX, "t": ["region:sa-east-1", "account_id:XXXX", "functionname:integration-tests-python-XXXX-async-metrics_python39", "resource:integration-tests-python-XXXX-async-metrics_python39", "cold_start:true", "memorysize:1024", "runtime:python3.9", "datadog_lambda:vXX", "dd_lambda_layer:datadog-python39_X.X.X"]}
+{"m": "hello.dog", "v": 1, "e": XXXX, "t": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python39_X.X.X"]}
+{"m": "tests.integration.count", "v": 21, "e": XXXX, "t": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python39_X.X.X"]}
+HTTP GET https://httpstat.us/200/ Headers: ["Accept-Encoding:gzip, deflate", "Accept:*/*", "Connection:keep-alive", "User-Agent:python-requests/X.X.X", "x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:1", "x-datadog-trace-id:XXXX"] Data: {}
+HTTP GET https://httpstat.us/400/ Headers: ["Accept-Encoding:gzip, deflate", "Accept:*/*", "Connection:keep-alive", "User-Agent:python-requests/X.X.X", "x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:1", "x-datadog-trace-id:XXXX"] Data: {}
+{"traces": [[{"trace_id": "XXXX", "parent_id": "XXXX", "span_id": "XXXX", "service": "aws.lambda", "resource": "integration-tests-python-XXXX-async-metrics_python39", "name": "aws.lambda", "error": 0, "start": XXXX, "duration": XXXX, "meta": {"runtime-id": "XXXX", "_dd.origin": "lambda", "cold_start": "true", "function_arn": "arn:aws:lambda:sa-east-1:601427279990:function:integration-tests-python-XXXX-async-metrics_python39", "function_version": "$LATEST", "request_id": "XXXX", "resource_names": "integration-tests-python-XXXX-async-metrics_python39", "datadog_lambda": "X.X.X", "dd_trace": "X.X.X", "function_trigger.event_source": "api-gateway", "function_trigger.event_source_arn": "arn:aws:apigateway:sa-east-1::/restapis/wt6mne2s9k/stages/test", "http.method": "GET", "http.status_code": "200"}, "metrics": {"system.pid": XXXX, "_dd.agent_psr": 1.0, "_sampling_priority_v1": 1}, "type": "serverless"}, {"trace_id": "XXXX", "parent_id": "XXXX", "span_id": "XXXX", "service": "aws.lambda", "resource": "requests.request", "name": "requests.request", "error": 0, "start": XXXX, "duration": XXXX, "meta": {"_dd.origin": "lambda", "http.method": "GET", "http.url": "https://httpstat.us/200/", "http.status_code": "200"}, "metrics": {"_dd.measured": 1}, "type": "http"}, {"trace_id": "XXXX", "parent_id": "XXXX", "span_id": "XXXX", "service": "aws.lambda", "resource": "requests.request", "name": "requests.request", "error": 0, "start": XXXX, "duration": XXXX, "meta": {"_dd.origin": "lambda", "http.method": "GET", "http.url": "https://httpstat.us/400/", "http.status_code": "400"}, "metrics": {"_dd.measured": 1}, "type": "http"}]]}
+END RequestId: XXXX
+REPORT RequestId: XXXX	Duration: XXXX ms	Billed Duration: XXXX ms	Memory Size: 1024 MB	Max Memory Used: XXXX MB	Init Duration: XXXX ms	
+START RequestId: XXXX Version: $LATEST
+{"m": "aws.lambda.enhanced.invocations", "v": 1, "e": XXXX, "t": ["region:sa-east-1", "account_id:XXXX", "functionname:integration-tests-python-XXXX-async-metrics_python39", "resource:integration-tests-python-XXXX-async-metrics_python39", "cold_start:false", "memorysize:1024", "runtime:python3.9", "datadog_lambda:vXX", "dd_lambda_layer:datadog-python39_X.X.X"]}
+{"m": "hello.dog", "v": 1, "e": XXXX, "t": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python39_X.X.X"]}
+{"m": "tests.integration.count", "v": 21, "e": XXXX, "t": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python39_X.X.X"]}
+HTTP GET https://httpstat.us/200/ Headers: ["Accept-Encoding:gzip, deflate", "Accept:*/*", "Connection:keep-alive", "User-Agent:python-requests/X.X.X", "x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:1", "x-datadog-trace-id:XXXX"] Data: {}
+HTTP GET https://httpstat.us/400/ Headers: ["Accept-Encoding:gzip, deflate", "Accept:*/*", "Connection:keep-alive", "User-Agent:python-requests/X.X.X", "x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:1", "x-datadog-trace-id:XXXX"] Data: {}
+{"traces": [[{"trace_id": "XXXX", "parent_id": "XXXX", "span_id": "XXXX", "service": "aws.lambda", "resource": "integration-tests-python-XXXX-async-metrics_python39", "name": "aws.lambda", "error": 0, "start": XXXX, "duration": XXXX, "meta": {"runtime-id": "XXXX", "_dd.origin": "lambda", "cold_start": "false", "function_arn": "arn:aws:lambda:sa-east-1:601427279990:function:integration-tests-python-XXXX-async-metrics_python39", "function_version": "$LATEST", "request_id": "XXXX", "resource_names": "integration-tests-python-XXXX-async-metrics_python39", "datadog_lambda": "X.X.X", "dd_trace": "X.X.X", "function_trigger.event_source": "sns", "function_trigger.event_source_arn": "arn:aws:sns:us-east-2:123456789012:sns-lambda"}, "metrics": {"system.pid": XXXX, "_dd.agent_psr": 1.0, "_sampling_priority_v1": 1}, "type": "serverless"}, {"trace_id": "XXXX", "parent_id": "XXXX", "span_id": "XXXX", "service": "aws.lambda", "resource": "requests.request", "name": "requests.request", "error": 0, "start": XXXX, "duration": XXXX, "meta": {"_dd.origin": "lambda", "http.method": "GET", "http.url": "https://httpstat.us/200/", "http.status_code": "200"}, "metrics": {"_dd.measured": 1}, "type": "http"}, {"trace_id": "XXXX", "parent_id": "XXXX", "span_id": "XXXX", "service": "aws.lambda", "resource": "requests.request", "name": "requests.request", "error": 0, "start": XXXX, "duration": XXXX, "meta": {"_dd.origin": "lambda", "http.method": "GET", "http.url": "https://httpstat.us/400/", "http.status_code": "400"}, "metrics": {"_dd.measured": 1}, "type": "http"}]]}
+END RequestId: XXXX
+REPORT RequestId: XXXX	Duration: XXXX ms	Billed Duration: XXXX ms	Memory Size: 1024 MB	Max Memory Used: XXXX MB	
+START RequestId: XXXX Version: $LATEST
+{"m": "aws.lambda.enhanced.invocations", "v": 1, "e": XXXX, "t": ["region:sa-east-1", "account_id:XXXX", "functionname:integration-tests-python-XXXX-async-metrics_python39", "resource:integration-tests-python-XXXX-async-metrics_python39", "cold_start:false", "memorysize:1024", "runtime:python3.9", "datadog_lambda:vXX", "dd_lambda_layer:datadog-python39_X.X.X"]}
+{"m": "hello.dog", "v": 1, "e": XXXX, "t": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python39_X.X.X"]}
+{"m": "tests.integration.count", "v": 21, "e": XXXX, "t": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python39_X.X.X"]}
+HTTP GET https://httpstat.us/200/ Headers: ["Accept-Encoding:gzip, deflate", "Accept:*/*", "Connection:keep-alive", "User-Agent:python-requests/X.X.X", "x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:1", "x-datadog-trace-id:XXXX"] Data: {}
+HTTP GET https://httpstat.us/400/ Headers: ["Accept-Encoding:gzip, deflate", "Accept:*/*", "Connection:keep-alive", "User-Agent:python-requests/X.X.X", "x-datadog-parent-id:XXXX", "x-datadog-sampling-priority:1", "x-datadog-trace-id:XXXX"] Data: {}
+{"traces": [[{"trace_id": "XXXX", "parent_id": "XXXX", "span_id": "XXXX", "service": "aws.lambda", "resource": "integration-tests-python-XXXX-async-metrics_python39", "name": "aws.lambda", "error": 0, "start": XXXX, "duration": XXXX, "meta": {"_dd.origin": "lambda", "cold_start": "false", "function_arn": "arn:aws:lambda:sa-east-1:601427279990:function:integration-tests-python-XXXX-async-metrics_python39", "function_version": "$LATEST", "request_id": "XXXX", "resource_names": "integration-tests-python-XXXX-async-metrics_python39", "datadog_lambda": "X.X.X", "dd_trace": "X.X.X", "function_trigger.event_source": "sqs", "function_trigger.event_source_arn": "arn:aws:sqs:us-east-2:123456789012:my-queue"}, "metrics": {"_sampling_priority_v1": 1}, "type": "serverless"}, {"trace_id": "XXXX", "parent_id": "XXXX", "span_id": "XXXX", "service": "aws.lambda", "resource": "requests.request", "name": "requests.request", "error": 0, "start": XXXX, "duration": XXXX, "meta": {"_dd.origin": "lambda", "http.method": "GET", "http.url": "https://httpstat.us/200/", "http.status_code": "200"}, "metrics": {"_dd.measured": 1}, "type": "http"}, {"trace_id": "XXXX", "parent_id": "XXXX", "span_id": "XXXX", "service": "aws.lambda", "resource": "requests.request", "name": "requests.request", "error": 0, "start": XXXX, "duration": XXXX, "meta": {"_dd.origin": "lambda", "http.method": "GET", "http.url": "https://httpstat.us/400/", "http.status_code": "400"}, "metrics": {"_dd.measured": 1}, "type": "http"}]]}
+END RequestId: XXXX
+REPORT RequestId: XXXX	Duration: XXXX ms	Billed Duration: XXXX ms	Memory Size: 1024 MB	Max Memory Used: XXXX MB