From 79811fba833370122bf0656d68b3de5889ca122f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20Bra=C5=84ski?= Date: Thu, 18 Jun 2020 17:29:03 +0200 Subject: [PATCH 1/6] Add matching index with regex to avoid deleting all indexes for different patterns --- lambda/es-cleanup.py | 5 +++-- main.tf | 3 ++- variables.tf | 7 +++++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/lambda/es-cleanup.py b/lambda/es-cleanup.py index 0e7e67c..31ed63c 100755 --- a/lambda/es-cleanup.py +++ b/lambda/es-cleanup.py @@ -10,6 +10,7 @@ from __future__ import print_function import os import json +import re import time import boto3 import datetime @@ -62,6 +63,7 @@ def __init__(self, event, context): self.cfg["es_max_retry"] = int(self.get_parameter("es_max_retry", 3)) self.cfg["index_format"] = self.get_parameter( "index_format", "%Y.%m.%d") + self.cfg["index_regex"] = self.get_parameter("index_regex", "([^-]+)-(.*)") self.cfg["sns_arn"] = self.get_parameter("sns_arn", "") if not self.cfg["es_endpoint"]: @@ -194,8 +196,7 @@ def lambda_handler(event, context): print("Found Kibana index: %s - ignoring" % index["index"]) continue - idx_name = '-'.join(word for word in index["index"].split("-")[:-1]) - idx_date = index["index"].split("-")[-1] + idx_name, idx_date = re.match(es.cfg["index_regex"], index["index"]).groups() print("Found index: %s - %s" % (idx_name, idx_date)) if idx_name in es.cfg["index"] or "all" in es.cfg["index"]: diff --git a/main.tf b/main.tf index 2e132cc..869577d 100644 --- a/main.tf +++ b/main.tf @@ -116,9 +116,10 @@ resource "aws_lambda_function" "default" { environment { variables = { + delete_after = var.delete_after es_endpoint = var.es_endpoint index = var.index - delete_after = var.delete_after + index_regex = var.index_regex index_format = var.index_format sns_arn = var.sns_arn } diff --git a/variables.tf b/variables.tf index 6c9acba..348789f 100644 --- a/variables.tf +++ b/variables.tf @@ -95,6 +95,13 @@ variable "index_format" { description = "Combined with 'index' variable and is used to evaluate the index age" } + +variable "index_regex" { + type = string + default = "([^-]+)-(.*)" + description = "Determines regex that is used for matching index name and index date. By default it match two groups separated by hyphen." +} + variable "python_version" { type = string default = "2.7" From 933b4a71f531150dae2af23e46b01e1d76421a0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20Bra=C5=84ski?= Date: Thu, 18 Jun 2020 17:39:08 +0200 Subject: [PATCH 2/6] Readme --- README.md | 71 +++++++++++++++++++++++++++++++---------------- docs/terraform.md | 71 +++++++++++++++++++++++++++++++---------------- variables.tf | 1 - 3 files changed, 94 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index b40fa98..74cf85e 100644 --- a/README.md +++ b/README.md @@ -127,38 +127,61 @@ Available targets: lint Lint terraform code ``` +## Module: cloudposse/terraform-aws-lambda-elasticsearch-cleanup + +This module creates a scheduled Lambda function which will delete old +Elasticsearch indexes using SigV4Auth authentication. The lambda +function can optionally send output to an SNS topic if the topic ARN +is given + +## Requirements + +| Name | Version | +|------|---------| +| terraform | ~> 0.12.0 | +| aws | ~> 2.0 | +| null | ~> 2.0 | +| template | ~> 2.0 | + +## Providers + +| Name | Version | +|------|---------| +| aws | ~> 2.0 | + ## Inputs | Name | Description | Type | Default | Required | -|------|-------------|:----:|:-----:|:-----:| -| artifact_url | URL template for the remote artifact | string | `https://artifacts.cloudposse.com/$$${module_name}/$$${git_ref}/$$${filename}` | no | -| attributes | Additional attributes (e.g. `1`) | list(string) | `` | no | -| delete_after | Number of days to preserve | number | `15` | no | -| delimiter | Delimiter to be used between `namespace`, `stage`, `name` and `attributes` | string | `-` | no | -| enabled | This module will not create any resources unless enabled is set to "true" | bool | `true` | no | -| es_domain_arn | The Elasticsearch domain ARN | string | - | yes | -| es_endpoint | The Elasticsearch endpoint for the Lambda function to connect to | string | - | yes | -| es_security_group_id | The Elasticsearch cluster security group ID | string | - | yes | -| index | Index/indices to process. Use a comma-separated list. Specify `all` to match every index except for `.kibana` or `.kibana_1` | string | `all` | no | -| index_format | Combined with 'index' variable and is used to evaluate the index age | string | `%Y.%m.%d` | no | -| name | Solution name, e.g. 'app' or 'cluster' | string | `app` | no | -| namespace | Namespace, which could be your organization name, e.g. 'eg' or 'cp' | string | `` | no | -| python_version | The Python version to use | string | `2.7` | no | -| schedule | CloudWatch Events rule schedule using cron or rate expression | string | `cron(0 3 * * ? *)` | no | -| sns_arn | SNS ARN to publish alerts | string | `` | no | -| stage | Stage, e.g. 'prod', 'staging', 'dev', or 'test' | string | `` | no | -| subnet_ids | Subnet IDs | list(string) | - | yes | -| tags | Additional tags (e.g. `map('BusinessUnit','XYZ')` | map(string) | `` | no | -| timeout | Timeout for Lambda function in seconds | number | `300` | no | -| vpc_id | The VPC ID for the Lambda function | string | - | yes | +|------|-------------|------|---------|:--------:| +| artifact\_url | URL template for the remote artifact | `string` | `"https://artifacts.cloudposse.com/$${module_name}/$${git_ref}/$${filename}"` | no | +| attributes | Additional attributes (e.g. `1`) | `list(string)` | `[]` | no | +| delete\_after | Number of days to preserve | `number` | `15` | no | +| delimiter | Delimiter to be used between `namespace`, `stage`, `name` and `attributes` | `string` | `"-"` | no | +| enabled | This module will not create any resources unless enabled is set to "true" | `bool` | `true` | no | +| es\_domain\_arn | The Elasticsearch domain ARN | `string` | n/a | yes | +| es\_endpoint | The Elasticsearch endpoint for the Lambda function to connect to | `string` | n/a | yes | +| es\_security\_group\_id | The Elasticsearch cluster security group ID | `string` | n/a | yes | +| index | Index/indices to process. Use a comma-separated list. Specify `all` to match every index except for `.kibana` or `.kibana_1` | `string` | `"all"` | no | +| index\_format | Combined with 'index' variable and is used to evaluate the index age | `string` | `"%Y.%m.%d"` | no | +| index\_regex | Determines regex that is used for matching index name and index date. By default it match two groups separated by hyphen. | `string` | `"([^-]+)-(.*)"` | no | +| name | Solution name, e.g. 'app' or 'cluster' | `string` | `"app"` | no | +| namespace | Namespace, which could be your organization name, e.g. 'eg' or 'cp' | `string` | `""` | no | +| python\_version | The Python version to use | `string` | `"2.7"` | no | +| schedule | CloudWatch Events rule schedule using cron or rate expression | `string` | `"cron(0 3 * * ? *)"` | no | +| sns\_arn | SNS ARN to publish alerts | `string` | `""` | no | +| stage | Stage, e.g. 'prod', 'staging', 'dev', or 'test' | `string` | `""` | no | +| subnet\_ids | Subnet IDs | `list(string)` | n/a | yes | +| tags | Additional tags (e.g. `map('BusinessUnit','XYZ')` | `map(string)` | `{}` | no | +| timeout | Timeout for Lambda function in seconds | `number` | `300` | no | +| vpc\_id | The VPC ID for the Lambda function | `string` | n/a | yes | ## Outputs | Name | Description | |------|-------------| -| lambda_function_arn | ARN of the Lambda Function | -| lambda_function_source_code_size | The size in bytes of the function .zip file | -| security_group_id | Security Group ID of the Lambda Function | +| lambda\_function\_arn | ARN of the Lambda Function | +| lambda\_function\_source\_code\_size | The size in bytes of the function .zip file | +| security\_group\_id | Security Group ID of the Lambda Function | diff --git a/docs/terraform.md b/docs/terraform.md index 9c481a0..3d07f71 100644 --- a/docs/terraform.md +++ b/docs/terraform.md @@ -1,33 +1,56 @@ +## Module: cloudposse/terraform-aws-lambda-elasticsearch-cleanup + +This module creates a scheduled Lambda function which will delete old +Elasticsearch indexes using SigV4Auth authentication. The lambda +function can optionally send output to an SNS topic if the topic ARN +is given + +## Requirements + +| Name | Version | +|------|---------| +| terraform | ~> 0.12.0 | +| aws | ~> 2.0 | +| null | ~> 2.0 | +| template | ~> 2.0 | + +## Providers + +| Name | Version | +|------|---------| +| aws | ~> 2.0 | + ## Inputs | Name | Description | Type | Default | Required | -|------|-------------|:----:|:-----:|:-----:| -| artifact_url | URL template for the remote artifact | string | `https://artifacts.cloudposse.com/$$${module_name}/$$${git_ref}/$$${filename}` | no | -| attributes | Additional attributes (e.g. `1`) | list(string) | `` | no | -| delete_after | Number of days to preserve | number | `15` | no | -| delimiter | Delimiter to be used between `namespace`, `stage`, `name` and `attributes` | string | `-` | no | -| enabled | This module will not create any resources unless enabled is set to "true" | bool | `true` | no | -| es_domain_arn | The Elasticsearch domain ARN | string | - | yes | -| es_endpoint | The Elasticsearch endpoint for the Lambda function to connect to | string | - | yes | -| es_security_group_id | The Elasticsearch cluster security group ID | string | - | yes | -| index | Index/indices to process. Use a comma-separated list. Specify `all` to match every index except for `.kibana` or `.kibana_1` | string | `all` | no | -| index_format | Combined with 'index' variable and is used to evaluate the index age | string | `%Y.%m.%d` | no | -| name | Solution name, e.g. 'app' or 'cluster' | string | `app` | no | -| namespace | Namespace, which could be your organization name, e.g. 'eg' or 'cp' | string | `` | no | -| python_version | The Python version to use | string | `2.7` | no | -| schedule | CloudWatch Events rule schedule using cron or rate expression | string | `cron(0 3 * * ? *)` | no | -| sns_arn | SNS ARN to publish alerts | string | `` | no | -| stage | Stage, e.g. 'prod', 'staging', 'dev', or 'test' | string | `` | no | -| subnet_ids | Subnet IDs | list(string) | - | yes | -| tags | Additional tags (e.g. `map('BusinessUnit','XYZ')` | map(string) | `` | no | -| timeout | Timeout for Lambda function in seconds | number | `300` | no | -| vpc_id | The VPC ID for the Lambda function | string | - | yes | +|------|-------------|------|---------|:--------:| +| artifact\_url | URL template for the remote artifact | `string` | `"https://artifacts.cloudposse.com/$${module_name}/$${git_ref}/$${filename}"` | no | +| attributes | Additional attributes (e.g. `1`) | `list(string)` | `[]` | no | +| delete\_after | Number of days to preserve | `number` | `15` | no | +| delimiter | Delimiter to be used between `namespace`, `stage`, `name` and `attributes` | `string` | `"-"` | no | +| enabled | This module will not create any resources unless enabled is set to "true" | `bool` | `true` | no | +| es\_domain\_arn | The Elasticsearch domain ARN | `string` | n/a | yes | +| es\_endpoint | The Elasticsearch endpoint for the Lambda function to connect to | `string` | n/a | yes | +| es\_security\_group\_id | The Elasticsearch cluster security group ID | `string` | n/a | yes | +| index | Index/indices to process. Use a comma-separated list. Specify `all` to match every index except for `.kibana` or `.kibana_1` | `string` | `"all"` | no | +| index\_format | Combined with 'index' variable and is used to evaluate the index age | `string` | `"%Y.%m.%d"` | no | +| index\_regex | Determines regex that is used for matching index name and index date. By default it match two groups separated by hyphen. | `string` | `"([^-]+)-(.*)"` | no | +| name | Solution name, e.g. 'app' or 'cluster' | `string` | `"app"` | no | +| namespace | Namespace, which could be your organization name, e.g. 'eg' or 'cp' | `string` | `""` | no | +| python\_version | The Python version to use | `string` | `"2.7"` | no | +| schedule | CloudWatch Events rule schedule using cron or rate expression | `string` | `"cron(0 3 * * ? *)"` | no | +| sns\_arn | SNS ARN to publish alerts | `string` | `""` | no | +| stage | Stage, e.g. 'prod', 'staging', 'dev', or 'test' | `string` | `""` | no | +| subnet\_ids | Subnet IDs | `list(string)` | n/a | yes | +| tags | Additional tags (e.g. `map('BusinessUnit','XYZ')` | `map(string)` | `{}` | no | +| timeout | Timeout for Lambda function in seconds | `number` | `300` | no | +| vpc\_id | The VPC ID for the Lambda function | `string` | n/a | yes | ## Outputs | Name | Description | |------|-------------| -| lambda_function_arn | ARN of the Lambda Function | -| lambda_function_source_code_size | The size in bytes of the function .zip file | -| security_group_id | Security Group ID of the Lambda Function | +| lambda\_function\_arn | ARN of the Lambda Function | +| lambda\_function\_source\_code\_size | The size in bytes of the function .zip file | +| security\_group\_id | Security Group ID of the Lambda Function | diff --git a/variables.tf b/variables.tf index 348789f..433e3aa 100644 --- a/variables.tf +++ b/variables.tf @@ -95,7 +95,6 @@ variable "index_format" { description = "Combined with 'index' variable and is used to evaluate the index age" } - variable "index_regex" { type = string default = "([^-]+)-(.*)" From 3705e370fd2b26a8a7e0133498341ba491d65dd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20Bra=C5=84ski?= Date: Thu, 18 Jun 2020 17:49:15 +0200 Subject: [PATCH 3/6] Add me to contributors --- README.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.yaml b/README.yaml index ec66ddb..07b643c 100644 --- a/README.yaml +++ b/README.yaml @@ -58,3 +58,5 @@ contributors: github: aknysh - name: Igor Rodionov github: goruha +- name: Marcin Brański + github: 3h4x From a6d28b91f5515e2644d7cc8eb2d87bdff8be601f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20Bra=C5=84ski?= Date: Mon, 29 Jun 2020 17:00:39 +0200 Subject: [PATCH 4/6] Rebase + readme --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 74cf85e..2e401f6 100644 --- a/README.md +++ b/README.md @@ -329,8 +329,8 @@ Check out [our other projects][github], [follow us on twitter][twitter], [apply ### Contributors -| [![Josh Myers][joshmyers_avatar]][joshmyers_homepage]
[Josh Myers][joshmyers_homepage] | [![Erik Osterman][osterman_avatar]][osterman_homepage]
[Erik Osterman][osterman_homepage] | [![Andriy Knysh][aknysh_avatar]][aknysh_homepage]
[Andriy Knysh][aknysh_homepage] | [![Igor Rodionov][goruha_avatar]][goruha_homepage]
[Igor Rodionov][goruha_homepage] | -|---|---|---|---| +| [![Josh Myers][joshmyers_avatar]][joshmyers_homepage]
[Josh Myers][joshmyers_homepage] | [![Erik Osterman][osterman_avatar]][osterman_homepage]
[Erik Osterman][osterman_homepage] | [![Andriy Knysh][aknysh_avatar]][aknysh_homepage]
[Andriy Knysh][aknysh_homepage] | [![Igor Rodionov][goruha_avatar]][goruha_homepage]
[Igor Rodionov][goruha_homepage] | [![Marcin Brański][3h4x_avatar]][3h4x_homepage]
[Marcin Brański][3h4x_homepage] | +|---|---|---|---|---| [joshmyers_homepage]: https://github.com/joshmyers [joshmyers_avatar]: https://img.cloudposse.com/150x150/https://github.com/joshmyers.png @@ -340,6 +340,8 @@ Check out [our other projects][github], [follow us on twitter][twitter], [apply [aknysh_avatar]: https://img.cloudposse.com/150x150/https://github.com/aknysh.png [goruha_homepage]: https://github.com/goruha [goruha_avatar]: https://img.cloudposse.com/150x150/https://github.com/goruha.png + [3h4x_homepage]: https://github.com/3h4x + [3h4x_avatar]: https://img.cloudposse.com/150x150/https://github.com/3h4x.png [![README Footer][readme_footer_img]][readme_footer_link] [![Beacon][beacon]][website] From d4a18c8a50883089e6f07f29c28aecb870497455 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20Bra=C5=84ski?= Date: Mon, 29 Jun 2020 20:13:20 +0200 Subject: [PATCH 5/6] Fix tests --- test/src/examples_complete_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/src/examples_complete_test.go b/test/src/examples_complete_test.go index 4f60f55..88e75db 100644 --- a/test/src/examples_complete_test.go +++ b/test/src/examples_complete_test.go @@ -33,12 +33,12 @@ func TestExamplesComplete(t *testing.T) { // Run `terraform output` to get the value of an output variable privateSubnetCidrs := terraform.OutputList(t, terraformOptions, "private_subnet_cidrs") // Verify we're getting back the outputs we expect - assert.Equal(t, []string{"172.16.0.0/18", "172.16.64.0/18"}, privateSubnetCidrs) + assert.Equal(t, []string{"172.16.0.0/19", "172.16.32.0/19"}, privateSubnetCidrs) // Run `terraform output` to get the value of an output variable publicSubnetCidrs := terraform.OutputList(t, terraformOptions, "public_subnet_cidrs") // Verify we're getting back the outputs we expect - assert.Equal(t, []string{"172.16.128.0/18", "172.16.192.0/18"}, publicSubnetCidrs) + assert.Equal(t, []string{"172.16.128.0/19", "172.16.96.0/19"}, publicSubnetCidrs) // Run `terraform output` to get the value of an output variable domainHostname := terraform.Output(t, terraformOptions, "domain_hostname") From f7e5667cf9bea247ddb50488fc4a7f98d38abad5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20Bra=C5=84ski?= Date: Mon, 29 Jun 2020 22:00:11 +0200 Subject: [PATCH 6/6] Fix test assertion order --- test/src/examples_complete_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/src/examples_complete_test.go b/test/src/examples_complete_test.go index 88e75db..3a70b00 100644 --- a/test/src/examples_complete_test.go +++ b/test/src/examples_complete_test.go @@ -38,7 +38,7 @@ func TestExamplesComplete(t *testing.T) { // Run `terraform output` to get the value of an output variable publicSubnetCidrs := terraform.OutputList(t, terraformOptions, "public_subnet_cidrs") // Verify we're getting back the outputs we expect - assert.Equal(t, []string{"172.16.128.0/19", "172.16.96.0/19"}, publicSubnetCidrs) + assert.Equal(t, []string{"172.16.96.0/19", "172.16.128.0/19"}, publicSubnetCidrs) // Run `terraform output` to get the value of an output variable domainHostname := terraform.Output(t, terraformOptions, "domain_hostname")