feat: Automatic layer releases

.github/workflows/build.yml

@@ -63,74 +63,3 @@ jobs:
         run: |
           source venv/bin/activate
           pytest -vv
-
-  integration-test:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        runtime-param: ['3.8', '3.9', '3.10', '3.11', '3.12']
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Set up Node 14
-        uses: actions/setup-node@v3
-        with:
-          node-version: 14
-
-      - name: Cache Node modules
-        id: cache-node-modules
-        uses: actions/cache@v3
-        with:
-          path: "**/node_modules"
-          key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }}
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.9
-
-      - name: Install Python dependencies
-        run: |
-          pip install virtualenv
-          virtualenv venv
-          source venv/bin/activate
-          pip install .[dev]
-
-      - name: Install Serverless Framework
-        run: sudo yarn global add serverless@^3.7.0 --prefix /usr/local
-      - name: Install Crossbuild Deps
-        run: |
-          sudo apt-get update --allow-releaseinfo-change --fix-missing
-          sudo apt install -y qemu-user-static binfmt-support
-
-      - name: Install dependencies
-        if: steps.cache-node-modules.outputs.cache-hit != 'true'
-        working-directory: tests/integration
-        run: yarn install
-
-      - name: Run tests
-        env:
-          BUILD_LAYERS: true
-          DD_API_KEY: ${{ secrets.DD_API_KEY }}
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          RUNTIME_PARAM: ${{ matrix.runtime-param }}
-        run: ./scripts/run_integration_tests.sh
-
-      - name: Send success metric
-        env:
-          DD_API_KEY: ${{ secrets.DD_API_KEY }}
-        run: ./scripts/send_status_metric.sh 0 $DD_API_KEY
-
-  integration-test-failure:
-    runs-on: ubuntu-latest
-    needs: [integration-test]
-    if: always() && (needs.integration-test.result == 'failure')
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Send a failure metric
-        env:
-          DD_API_KEY: ${{ secrets.DD_API_KEY }}
-        run: ./scripts/send_status_metric.sh 1 $DD_API_KEY

.gitlab-ci.yml

@@ -0,0 +1,29 @@
+stages:
+  - pre 
+  - build
+
+.go-cache: &go-cache
+  key: datadog-lambda-python-go-cache
+  policy: pull
+
+generator:
+  stage: pre
+  image: registry.ddbuild.io/images/mirror/golang:alpine
+  tags: ["arch:amd64"]
+  cache: *go-cache
+  script:
+    - apk add --no-cache gomplate
+    - gomplate --config ci/config.yaml
+  artifacts:
+    paths:
+      - ci/*-pipeline.yaml
+
+build-layers:
+  stage: build
+  trigger:
+    include:
+      - artifact: ci/build-pipeline.yaml
+        job: generator
+    strategy: depend
+  rules:
+    - when: on_success

ci/config.yaml

@@ -0,0 +1,13 @@
+inputFiles:
+  - ci/input_files/build.yaml.tpl
+
+outputFiles:
+  - ci/build-pipeline.yaml
+
+datasources:
+  runtimes:
+    url: ci/datasources/runtimes.yaml
+  regions:
+    url: ci/datasources/regions.yaml
+  environments:
+    url: ci/datasources/environments.yaml

ci/datasources/environments.yaml

@@ -0,0 +1,9 @@
+environments:
+  - name: sandbox
+    external_id: sandbox-publish-externalid
+    role_to_assume: sandbox-layer-deployer
+    account: 425362996713
+  - name: prod
+    external_id: prod-publish-externalid
+    role_to_assume: dd-serverless-layer-deployer-role
+    account: 464622532012

ci/datasources/regions.yaml

@@ -0,0 +1,29 @@
+regions:
+  - code: "us-east-1"
+  - code: "us-east-2"
+  - code: "us-west-1"
+  - code: "us-west-2"
+  - code: "af-south-1"
+  - code: "ap-east-1"
+  - code: "ap-south-1"
+  - code: "ap-south-2"
+  - code: "ap-southeast-1"
+  - code: "ap-southeast-2"
+  - code: "ap-southeast-3"
+  - code: "ap-southeast-4"
+  - code: "ap-northeast-1"
+  - code: "ap-northeast-2"
+  - code: "ap-northeast-3"
+  - code: "ca-central-1"
+#  - code: "ca-west-1" we don't support it
+  - code: "eu-central-1"
+  - code: "eu-central-2"
+  - code: "eu-west-1"
+  - code: "eu-west-2"
+  - code: "eu-west-3"
+  - code: "eu-south-1"
+  - code: "eu-south-2"
+#  - code: "il-central-1" we don't support it
+  - code: "me-south-1"
+  - code: "me-central-1"
+  - code: "sa-east-1"

ci/datasources/runtimes.yaml

@@ -0,0 +1,41 @@
+runtimes:
+  - name: "python38"
+    python_version: "3.8"
+    arch: "amd64"
+    image: "3.8"
+  - name: "python38"
+    python_version: "3.8"
+    arch: "arm64"
+    image: "3.8"
+  - name: "python39"
+    python_version: "3.8"
+    arch: "amd64"
+    image: "3.9"
+  - name: "python39"
+    python_version: "3.9"
+    arch: "arm64"
+    image: "3.9"
+  - name: "python310"
+    python_version: "3.10"
+    arch: "amd64"
+    image: "3.10"
+  - name: "python310"
+    python_version: "3.10"  
+    arch: "arm64"
+    image: "3.10"
+  - name: "python311"
+    python_version: "3.11"
+    arch: "amd64"
+    image: "3.11.6"
+  - name: "python311"
+    python_version: "3.11" 
+    arch: "arm64"
+    image: "3.11.6"
+  - name: "python312"
+    python_version: "3.12"
+    arch: "amd64"
+    image: "3.12.0"
+  - name: "python312"
+    python_version: "3.12"
+    arch: "arm64"
+    image: "3.12.0"

ci/get_secrets.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Unless explicitly stated otherwise all files in this repository are licensed
+# under the Apache License Version 2.0.
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2023 Datadog, Inc.
+
+set -e
+
+if [ -z "$EXTERNAL_ID_NAME" ]; then
+    printf "[Error] No EXTERNAL_ID_NAME found.\n"
+    printf "Exiting script...\n"
+    exit 1
+fi
+
+if [ -z "$ROLE_TO_ASSUME" ]; then
+    printf "[Error] No ROLE_TO_ASSUME found.\n"
+    printf "Exiting script...\n"
+    exit 1
+fi
+
+printf "Getting AWS External ID...\n"
+
+EXTERNAL_ID=$(aws ssm get-parameter \
+    --region us-east-1 \
+    --name "ci.datadog-lambda-python.$EXTERNAL_ID_NAME" \
+    --with-decryption \
+    --query "Parameter.Value" \
+    --out text)
+
+printf "Getting DD API KEY...\n"
+
+export DD_API_KEY=$(aws ssm get-parameter \
+    --region us-east-1 \
+    --name ci.datadog-lambda-python.dd-api-key \
+    --with-decryption \
+    --query "Parameter.Value" \
+    --out text)
+
+printf "Assuming role...\n"
+
+export $(printf "AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s AWS_SESSION_TOKEN=%s" \
+    $(aws sts assume-role \
+    --role-arn "arn:aws:iam::$AWS_ACCOUNT:role/$ROLE_TO_ASSUME"  \
+    --role-session-name "ci.datadog-lambda-python-$CI_JOB_ID-$CI_JOB_STAGE" \
+    --query "Credentials.[AccessKeyId,SecretAccessKey,SessionToken]" \
+    --external-id $EXTERNAL_ID \
+    --output text))