From 2872eee455ac511d1fd9e360e10c086f2fae90c7 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Sun, 19 Nov 2023 17:03:57 -0500 Subject: [PATCH 1/2] feat: push layers in batches --- scripts/publish_layers.sh | 70 +++++++++++++++++++++++++-------------- 1 file changed, 46 insertions(+), 24 deletions(-) diff --git a/scripts/publish_layers.sh b/scripts/publish_layers.sh index d379cb67..5af1a1e0 100755 --- a/scripts/publish_layers.sh +++ b/scripts/publish_layers.sh @@ -19,6 +19,12 @@ AVAILABLE_LAYERS=("Datadog-Python37" "Datadog-Python38" "Datadog-Python38-ARM" " ARCHS=("amd64" "amd64" "amd64""amd64" "amd64" "arm64" "amd64" "arm64" "amd64" "arm64") AVAILABLE_REGIONS=$(aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName') +BATCH_SIZE=60 +PIDS=() + +# Makes sure any subprocesses will be terminated with this process +trap "pkill -P $$; exit 1;" INT + # Check that the layer files exist for layer_file in "${LAYER_PATHS[@]}" do @@ -102,41 +108,57 @@ publish_layer() { echo $version_nbr } -for region in $REGIONS -do - echo "Starting publishing layer for region $region..." - # Publish the layers for each version of python - for layer_name in "${LAYERS[@]}"; do - latest_version=$(aws lambda list-layer-versions --region $region --layer-name $layer_name --query 'LayerVersions[0].Version || `0`') - if [ $latest_version -ge $VERSION ]; then - echo "Layer $layer_name version $VERSION already exists in region $region, skipping..." - continue - elif [ $latest_version -lt $((VERSION-1)) ]; then - read -p "WARNING: The latest version of layer $layer_name in region $region is $latest_version, publish all the missing versions including $VERSION or EXIT the script (y/n)?" CONT - if [ "$CONT" != "y" ]; then - echo "Exiting" - exit 1 - fi - fi - - index=$(index_of_layer $layer_name) - aws_version_key="${PYTHON_VERSIONS_FOR_AWS_CLI[$index]}" - layer_path="${LAYER_PATHS[$index]}" +wait_for_processes() { + for pid in "${PIDS[@]}"; do + wait $pid + done + PIDS=() +} - while [ $latest_version -lt $VERSION ]; do +backfill_layers() { + region=$1 + layer_name=$2 + aws_version_key=$3 + layer_path=$4 + latest_version=$(aws lambda list-layer-versions --region $region --layer-name $layer_name --query 'LayerVersions[0].Version || `0`') + if [ $latest_version -ge $VERSION ]; then + echo "Layer $layer_name version $VERSION already exists in region $region, skipping..." + continue + elif [ $latest_version -lt $((VERSION-1)) ]; then + echo "WARNING: The latest version of layer $layer_name in region $region is $latest_version, this will publish all the missing versions including $VERSION" + fi + while [ $latest_version -lt $VERSION ]; do latest_version=$(publish_layer $region $layer_name $aws_version_key $layer_path) echo "Published version $latest_version for layer $layer_name in region $region" - # This shouldn't happen unless someone manually deleted the latest version, say 28 - # and then try to republish it again. The published version is actually be 29, because + # This shouldn't happen unless someone manually deleted the latest version, say 28, and + # then tries to republish 28 again. The published version would actually be 29, because # Lambda layers are immutable and AWS will skip deleted version and use the next number. if [ $latest_version -gt $VERSION ]; then echo "ERROR: Published version $latest_version is greater than the desired version $VERSION!" echo "Exiting" exit 1 fi - done + done +} + +for region in $REGIONS +do + echo "Starting publishing layer for region $region..." + # Publish the layers for each version of python + for layer_name in "${LAYERS[@]}"; do + + index=$(index_of_layer $layer_name) + aws_version_key="${PYTHON_VERSIONS_FOR_AWS_CLI[$index]}" + layer_path="${LAYER_PATHS[$index]}" + + backfill_layers $latest_version $region $layer_name $aws_version_key $layer_path & + PIDS+=($!) + if [ ${#PIDS[@]} -ge $BATCH_SIZE ]; then + wait_for_processes + fi done done +wait_for_processes echo "Done !" From c18d3aaf4a201354404148b6b20cfb35e56f111a Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Mon, 20 Nov 2023 10:23:19 -0500 Subject: [PATCH 2/2] feat: Check latest inside region loop to prevent early return on our backfill processes --- scripts/publish_layers.sh | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/scripts/publish_layers.sh b/scripts/publish_layers.sh index 5af1a1e0..bd8d3a7e 100755 --- a/scripts/publish_layers.sh +++ b/scripts/publish_layers.sh @@ -116,17 +116,12 @@ wait_for_processes() { } backfill_layers() { - region=$1 - layer_name=$2 - aws_version_key=$3 - layer_path=$4 - latest_version=$(aws lambda list-layer-versions --region $region --layer-name $layer_name --query 'LayerVersions[0].Version || `0`') - if [ $latest_version -ge $VERSION ]; then - echo "Layer $layer_name version $VERSION already exists in region $region, skipping..." - continue - elif [ $latest_version -lt $((VERSION-1)) ]; then - echo "WARNING: The latest version of layer $layer_name in region $region is $latest_version, this will publish all the missing versions including $VERSION" - fi + latest_version=$1 + region=$2 + layer_name=$3 + aws_version_key=$4 + layer_path=$5 + while [ $latest_version -lt $VERSION ]; do latest_version=$(publish_layer $region $layer_name $aws_version_key $layer_path) echo "Published version $latest_version for layer $layer_name in region $region" @@ -147,6 +142,13 @@ do echo "Starting publishing layer for region $region..." # Publish the layers for each version of python for layer_name in "${LAYERS[@]}"; do + latest_version=$(aws lambda list-layer-versions --region $region --layer-name $layer_name --query 'LayerVersions[0].Version || `0`') + if [ $latest_version -ge $VERSION ]; then + echo "Layer $layer_name version $VERSION already exists in region $region, skipping..." + continue + elif [ $latest_version -lt $((VERSION-1)) ]; then + echo "WARNING: The latest version of layer $layer_name in region $region is $latest_version, this will publish all the missing versions including $VERSION" + fi index=$(index_of_layer $layer_name) aws_version_key="${PYTHON_VERSIONS_FOR_AWS_CLI[$index]}"