Matching index with regex pattern

README.md

@@ -127,38 +127,61 @@ Available targets:
   lint                                Lint terraform code
 
 ```
+## Module: cloudposse/terraform-aws-lambda-elasticsearch-cleanup
+
+This module creates a scheduled Lambda function which will delete old  
+Elasticsearch indexes using SigV4Auth authentication. The lambda  
+function can optionally send output to an SNS topic if the topic ARN  
+is given
+
+## Requirements
+
+| Name | Version |
+|------|---------|
+| terraform | ~> 0.12.0 |
+| aws | ~> 2.0 |
+| null | ~> 2.0 |
+| template | ~> 2.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| aws | ~> 2.0 |
+
 ## Inputs
 
 | Name | Description | Type | Default | Required |
-|------|-------------|:----:|:-----:|:-----:|
-| artifact_url | URL template for the remote artifact | string | `https://artifacts.cloudposse.com/$$${module_name}/$$${git_ref}/$$${filename}` | no |
-| attributes | Additional attributes (e.g. `1`) | list(string) | `<list>` | no |
-| delete_after | Number of days to preserve | number | `15` | no |
-| delimiter | Delimiter to be used between `namespace`, `stage`, `name` and `attributes` | string | `-` | no |
-| enabled | This module will not create any resources unless enabled is set to "true" | bool | `true` | no |
-| es_domain_arn | The Elasticsearch domain ARN | string | - | yes |
-| es_endpoint | The Elasticsearch endpoint for the Lambda function to connect to | string | - | yes |
-| es_security_group_id | The Elasticsearch cluster security group ID | string | - | yes |
-| index | Index/indices to process. Use a comma-separated list. Specify `all` to match every index except for `.kibana` or `.kibana_1` | string | `all` | no |
-| index_format | Combined with 'index' variable and is used to evaluate the index age | string | `%Y.%m.%d` | no |
-| name | Solution name, e.g. 'app' or 'cluster' | string | `app` | no |
-| namespace | Namespace, which could be your organization name, e.g. 'eg' or 'cp' | string | `` | no |
-| python_version | The Python version to use | string | `2.7` | no |
-| schedule | CloudWatch Events rule schedule using cron or rate expression | string | `cron(0 3 * * ? *)` | no |
-| sns_arn | SNS ARN to publish alerts | string | `` | no |
-| stage | Stage, e.g. 'prod', 'staging', 'dev', or 'test' | string | `` | no |
-| subnet_ids | Subnet IDs | list(string) | - | yes |
-| tags | Additional tags (e.g. `map('BusinessUnit','XYZ')` | map(string) | `<map>` | no |
-| timeout | Timeout for Lambda function in seconds | number | `300` | no |
-| vpc_id | The VPC ID for the Lambda function | string | - | yes |
+|------|-------------|------|---------|:--------:|
+| artifact\_url | URL template for the remote artifact | `string` | `"https://artifacts.cloudposse.com/$${module_name}/$${git_ref}/$${filename}"` | no |
+| attributes | Additional attributes (e.g. `1`) | `list(string)` | `[]` | no |
+| delete\_after | Number of days to preserve | `number` | `15` | no |
+| delimiter | Delimiter to be used between `namespace`, `stage`, `name` and `attributes` | `string` | `"-"` | no |
+| enabled | This module will not create any resources unless enabled is set to "true" | `bool` | `true` | no |
+| es\_domain\_arn | The Elasticsearch domain ARN | `string` | n/a | yes |
+| es\_endpoint | The Elasticsearch endpoint for the Lambda function to connect to | `string` | n/a | yes |
+| es\_security\_group\_id | The Elasticsearch cluster security group ID | `string` | n/a | yes |
+| index | Index/indices to process. Use a comma-separated list. Specify `all` to match every index except for `.kibana` or `.kibana_1` | `string` | `"all"` | no |
+| index\_format | Combined with 'index' variable and is used to evaluate the index age | `string` | `"%Y.%m.%d"` | no |
+| index\_regex | Determines regex that is used for matching index name and index date. By default it match two groups separated by hyphen. | `string` | `"([^-]+)-(.*)"` | no |
+| name | Solution name, e.g. 'app' or 'cluster' | `string` | `"app"` | no |
+| namespace | Namespace, which could be your organization name, e.g. 'eg' or 'cp' | `string` | `""` | no |
+| python\_version | The Python version to use | `string` | `"2.7"` | no |
+| schedule | CloudWatch Events rule schedule using cron or rate expression | `string` | `"cron(0 3 * * ? *)"` | no |
+| sns\_arn | SNS ARN to publish alerts | `string` | `""` | no |
+| stage | Stage, e.g. 'prod', 'staging', 'dev', or 'test' | `string` | `""` | no |
+| subnet\_ids | Subnet IDs | `list(string)` | n/a | yes |
+| tags | Additional tags (e.g. `map('BusinessUnit','XYZ')` | `map(string)` | `{}` | no |
+| timeout | Timeout for Lambda function in seconds | `number` | `300` | no |
+| vpc\_id | The VPC ID for the Lambda function | `string` | n/a | yes |
 
 ## Outputs
 
 | Name | Description |
 |------|-------------|
-| lambda_function_arn | ARN of the Lambda Function |
-| lambda_function_source_code_size | The size in bytes of the function .zip file |
-| security_group_id | Security Group ID of the Lambda Function |
+| lambda\_function\_arn | ARN of the Lambda Function |
+| lambda\_function\_source\_code\_size | The size in bytes of the function .zip file |
+| security\_group\_id | Security Group ID of the Lambda Function |
 
 
 
@@ -306,8 +329,8 @@ Check out [our other projects][github], [follow us on twitter][twitter], [apply
 
 ### Contributors
 
-|  [![Josh Myers][joshmyers_avatar]][joshmyers_homepage]<br/>[Josh Myers][joshmyers_homepage] | [![Erik Osterman][osterman_avatar]][osterman_homepage]<br/>[Erik Osterman][osterman_homepage] | [![Andriy Knysh][aknysh_avatar]][aknysh_homepage]<br/>[Andriy Knysh][aknysh_homepage] | [![Igor Rodionov][goruha_avatar]][goruha_homepage]<br/>[Igor Rodionov][goruha_homepage] |
-|---|---|---|---|
+|  [![Josh Myers][joshmyers_avatar]][joshmyers_homepage]<br/>[Josh Myers][joshmyers_homepage] | [![Erik Osterman][osterman_avatar]][osterman_homepage]<br/>[Erik Osterman][osterman_homepage] | [![Andriy Knysh][aknysh_avatar]][aknysh_homepage]<br/>[Andriy Knysh][aknysh_homepage] | [![Igor Rodionov][goruha_avatar]][goruha_homepage]<br/>[Igor Rodionov][goruha_homepage] | [![Marcin Brański][3h4x_avatar]][3h4x_homepage]<br/>[Marcin Brański][3h4x_homepage] |
+|---|---|---|---|---|
 
   [joshmyers_homepage]: https://github.com/joshmyers
   [joshmyers_avatar]: https://img.cloudposse.com/150x150/https://github.com/joshmyers.png
@@ -317,6 +340,8 @@ Check out [our other projects][github], [follow us on twitter][twitter], [apply
   [aknysh_avatar]: https://img.cloudposse.com/150x150/https://github.com/aknysh.png
   [goruha_homepage]: https://github.com/goruha
   [goruha_avatar]: https://img.cloudposse.com/150x150/https://github.com/goruha.png
+  [3h4x_homepage]: https://github.com/3h4x
+  [3h4x_avatar]: https://img.cloudposse.com/150x150/https://github.com/3h4x.png
 
 [![README Footer][readme_footer_img]][readme_footer_link]
 [![Beacon][beacon]][website]

README.yaml

@@ -58,3 +58,5 @@ contributors:
   github: aknysh
 - name: Igor Rodionov
   github: goruha
+- name: Marcin Brański
+  github: 3h4x

docs/terraform.md

@@ -1,33 +1,56 @@
+## Module: cloudposse/terraform-aws-lambda-elasticsearch-cleanup
+
+This module creates a scheduled Lambda function which will delete old  
+Elasticsearch indexes using SigV4Auth authentication. The lambda  
+function can optionally send output to an SNS topic if the topic ARN  
+is given
+
+## Requirements
+
+| Name | Version |
+|------|---------|
+| terraform | ~> 0.12.0 |
+| aws | ~> 2.0 |
+| null | ~> 2.0 |
+| template | ~> 2.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| aws | ~> 2.0 |
+
 ## Inputs
 
 | Name | Description | Type | Default | Required |
-|------|-------------|:----:|:-----:|:-----:|
-| artifact_url | URL template for the remote artifact | string | `https://artifacts.cloudposse.com/$$${module_name}/$$${git_ref}/$$${filename}` | no |
-| attributes | Additional attributes (e.g. `1`) | list(string) | `<list>` | no |
-| delete_after | Number of days to preserve | number | `15` | no |
-| delimiter | Delimiter to be used between `namespace`, `stage`, `name` and `attributes` | string | `-` | no |
-| enabled | This module will not create any resources unless enabled is set to "true" | bool | `true` | no |
-| es_domain_arn | The Elasticsearch domain ARN | string | - | yes |
-| es_endpoint | The Elasticsearch endpoint for the Lambda function to connect to | string | - | yes |
-| es_security_group_id | The Elasticsearch cluster security group ID | string | - | yes |
-| index | Index/indices to process. Use a comma-separated list. Specify `all` to match every index except for `.kibana` or `.kibana_1` | string | `all` | no |
-| index_format | Combined with 'index' variable and is used to evaluate the index age | string | `%Y.%m.%d` | no |
-| name | Solution name, e.g. 'app' or 'cluster' | string | `app` | no |
-| namespace | Namespace, which could be your organization name, e.g. 'eg' or 'cp' | string | `` | no |
-| python_version | The Python version to use | string | `2.7` | no |
-| schedule | CloudWatch Events rule schedule using cron or rate expression | string | `cron(0 3 * * ? *)` | no |
-| sns_arn | SNS ARN to publish alerts | string | `` | no |
-| stage | Stage, e.g. 'prod', 'staging', 'dev', or 'test' | string | `` | no |
-| subnet_ids | Subnet IDs | list(string) | - | yes |
-| tags | Additional tags (e.g. `map('BusinessUnit','XYZ')` | map(string) | `<map>` | no |
-| timeout | Timeout for Lambda function in seconds | number | `300` | no |
-| vpc_id | The VPC ID for the Lambda function | string | - | yes |
+|------|-------------|------|---------|:--------:|
+| artifact\_url | URL template for the remote artifact | `string` | `"https://artifacts.cloudposse.com/$${module_name}/$${git_ref}/$${filename}"` | no |
+| attributes | Additional attributes (e.g. `1`) | `list(string)` | `[]` | no |
+| delete\_after | Number of days to preserve | `number` | `15` | no |
+| delimiter | Delimiter to be used between `namespace`, `stage`, `name` and `attributes` | `string` | `"-"` | no |
+| enabled | This module will not create any resources unless enabled is set to "true" | `bool` | `true` | no |
+| es\_domain\_arn | The Elasticsearch domain ARN | `string` | n/a | yes |
+| es\_endpoint | The Elasticsearch endpoint for the Lambda function to connect to | `string` | n/a | yes |
+| es\_security\_group\_id | The Elasticsearch cluster security group ID | `string` | n/a | yes |
+| index | Index/indices to process. Use a comma-separated list. Specify `all` to match every index except for `.kibana` or `.kibana_1` | `string` | `"all"` | no |
+| index\_format | Combined with 'index' variable and is used to evaluate the index age | `string` | `"%Y.%m.%d"` | no |
+| index\_regex | Determines regex that is used for matching index name and index date. By default it match two groups separated by hyphen. | `string` | `"([^-]+)-(.*)"` | no |
+| name | Solution name, e.g. 'app' or 'cluster' | `string` | `"app"` | no |
+| namespace | Namespace, which could be your organization name, e.g. 'eg' or 'cp' | `string` | `""` | no |
+| python\_version | The Python version to use | `string` | `"2.7"` | no |
+| schedule | CloudWatch Events rule schedule using cron or rate expression | `string` | `"cron(0 3 * * ? *)"` | no |
+| sns\_arn | SNS ARN to publish alerts | `string` | `""` | no |
+| stage | Stage, e.g. 'prod', 'staging', 'dev', or 'test' | `string` | `""` | no |
+| subnet\_ids | Subnet IDs | `list(string)` | n/a | yes |
+| tags | Additional tags (e.g. `map('BusinessUnit','XYZ')` | `map(string)` | `{}` | no |
+| timeout | Timeout for Lambda function in seconds | `number` | `300` | no |
+| vpc\_id | The VPC ID for the Lambda function | `string` | n/a | yes |
 
 ## Outputs
 
 | Name | Description |
 |------|-------------|
-| lambda_function_arn | ARN of the Lambda Function |
-| lambda_function_source_code_size | The size in bytes of the function .zip file |
-| security_group_id | Security Group ID of the Lambda Function |
+| lambda\_function\_arn | ARN of the Lambda Function |
+| lambda\_function\_source\_code\_size | The size in bytes of the function .zip file |
+| security\_group\_id | Security Group ID of the Lambda Function |
 

lambda/es-cleanup.py

@@ -10,6 +10,7 @@
 from __future__ import print_function
 import os
 import json
+import re
 import time
 import boto3
 import datetime
@@ -62,6 +63,7 @@ def __init__(self, event, context):
         self.cfg["es_max_retry"] = int(self.get_parameter("es_max_retry", 3))
         self.cfg["index_format"] = self.get_parameter(
             "index_format", "%Y.%m.%d")
+        self.cfg["index_regex"] = self.get_parameter("index_regex", "([^-]+)-(.*)")
         self.cfg["sns_arn"] = self.get_parameter("sns_arn", "")
 
         if not self.cfg["es_endpoint"]:
@@ -194,8 +196,7 @@ def lambda_handler(event, context):
                 print("Found Kibana index: %s - ignoring" % index["index"])
                 continue
 
-            idx_name = '-'.join(word for word in index["index"].split("-")[:-1])
-            idx_date = index["index"].split("-")[-1]
+            idx_name, idx_date = re.match(es.cfg["index_regex"], index["index"]).groups()
             print("Found index: %s - %s" % (idx_name, idx_date))
             if idx_name in es.cfg["index"] or "all" in es.cfg["index"]:
 

main.tf

@@ -116,9 +116,10 @@ resource "aws_lambda_function" "default" {
 
   environment {
     variables = {
+      delete_after = var.delete_after
       es_endpoint  = var.es_endpoint
       index        = var.index
-      delete_after = var.delete_after
+      index_regex  = var.index_regex
       index_format = var.index_format
       sns_arn      = var.sns_arn
     }

test/src/examples_complete_test.go

@@ -33,12 +33,12 @@ func TestExamplesComplete(t *testing.T) {
 	// Run `terraform output` to get the value of an output variable
 	privateSubnetCidrs := terraform.OutputList(t, terraformOptions, "private_subnet_cidrs")
 	// Verify we're getting back the outputs we expect
-	assert.Equal(t, []string{"172.16.0.0/18", "172.16.64.0/18"}, privateSubnetCidrs)
+	assert.Equal(t, []string{"172.16.0.0/19", "172.16.32.0/19"}, privateSubnetCidrs)
 
 	// Run `terraform output` to get the value of an output variable
 	publicSubnetCidrs := terraform.OutputList(t, terraformOptions, "public_subnet_cidrs")
 	// Verify we're getting back the outputs we expect
-	assert.Equal(t, []string{"172.16.128.0/18", "172.16.192.0/18"}, publicSubnetCidrs)
+	assert.Equal(t, []string{"172.16.96.0/19", "172.16.128.0/19"}, publicSubnetCidrs)
 
 	// Run `terraform output` to get the value of an output variable
 	domainHostname := terraform.Output(t, terraformOptions, "domain_hostname")

variables.tf

@@ -95,6 +95,12 @@ variable "index_format" {
   description = "Combined with 'index' variable and is used to evaluate the index age"
 }
 
+variable "index_regex" {
+  type        = string
+  default     = "([^-]+)-(.*)"
+  description = "Determines regex that is used for matching index name and index date. By default it match two groups separated by hyphen."
+}
+
 variable "python_version" {
   type        = string
   default     = "2.7"