From bc59b7e6b9d659d01a86add872c2d495aa6099b5 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sat, 1 Oct 2022 17:17:53 +0100 Subject: [PATCH 01/19] chore(parameters): refactoring examples --- docs/utilities/parameters.md | 61 ++++++++----------- .../src/getting_started_appconfig.py | 18 ++++++ ...getting_started_recursive_ssm_parameter.py | 23 +++++++ .../parameters/src/getting_started_secret.py | 21 +++++++ .../getting_started_single_ssm_parameter.py | 17 ++++++ 5 files changed, 106 insertions(+), 34 deletions(-) create mode 100644 examples/parameters/src/getting_started_appconfig.py create mode 100644 examples/parameters/src/getting_started_recursive_ssm_parameter.py create mode 100644 examples/parameters/src/getting_started_secret.py create mode 100644 examples/parameters/src/getting_started_single_ssm_parameter.py diff --git a/docs/utilities/parameters.md b/docs/utilities/parameters.md index 2559044b632..89898e74cb9 100644 --- a/docs/utilities/parameters.md +++ b/docs/utilities/parameters.md @@ -15,6 +15,9 @@ The parameters utility provides high-level functions to retrieve one or multiple ## Getting started +???+ tip + All examples shared in this documentation are available within the [project repository](https://github.com/awslabs/aws-lambda-powertools-python/tree/develop/examples){target="_blank"}. + By default, we fetch parameters from System Manager Parameter Store, secrets from Secrets Manager, and application configuration from AppConfig. ### IAM Permissions @@ -24,14 +27,14 @@ This utility requires additional permissions to work as expected. ???+ note Different parameter providers require different permissions. -| Provider | Function/Method | IAM Permission | -| ------------------- | ---------------------------------------------------- | ------------------------------- | -| SSM Parameter Store | `get_parameter`, `SSMProvider.get` | `ssm:GetParameter` | -| SSM Parameter Store | `get_parameters`, `SSMProvider.get_multiple` | `ssm:GetParametersByPath` | -| Secrets Manager | `get_secret`, `SecretsManager.get` | `secretsmanager:GetSecretValue` | -| DynamoDB | `DynamoDBProvider.get` | `dynamodb:GetItem` | -| DynamoDB | `DynamoDBProvider.get_multiple` | `dynamodb:Query` | -| App Config | `AppConfigProvider.get_app_config`, `get_app_config` | `appconfig:GetConfiguration` | +| Provider | Function/Method | IAM Permission | +| ------------------- | ---------------------------------------------------- | --------------------------------------------------------------------------------------- | +| SSM Parameter Store | `get_parameter`, `SSMProvider.get` | `ssm:GetParameter` | +| SSM Parameter Store | `get_parameters`, `SSMProvider.get_multiple` | `ssm:GetParametersByPath` | +| Secrets Manager | `get_secret`, `SecretsManager.get` | `secretsmanager:GetSecretValue` | +| DynamoDB | `DynamoDBProvider.get` | `dynamodb:GetItem` | +| DynamoDB | `DynamoDBProvider.get_multiple` | `dynamodb:Query` | +| App Config | `AppConfigProvider.get_app_config`, `get_app_config` | `appconfig:GetLatestConfiguration` and `appconfig:StartConfigurationSession` | ### Fetching parameters @@ -39,31 +42,24 @@ You can retrieve a single parameter using `get_parameter` high-level function. For multiple parameters, you can use `get_parameters` and pass a path to retrieve them recursively. -```python hl_lines="1 5 9" title="Fetching multiple parameters recursively" -from aws_lambda_powertools.utilities import parameters - -def handler(event, context): - # Retrieve a single parameter - value = parameters.get_parameter("/my/parameter") +=== "getting_started_single_ssm_parameter.py" + ```python hl_lines="3 10 16" + --8<-- "examples/parameters/src/getting_started_single_ssm_parameter.py" + ``` - # Retrieve multiple parameters from a path prefix recursively - # This returns a dict with the parameter name as key - values = parameters.get_parameters("/my/path/prefix") - for k, v in values.items(): - print(f"{k}: {v}") -``` +=== "getting_started_recursive_ssm_parameter.py" + ```python hl_lines="3 10 13 22" + --8<-- "examples/parameters/src/getting_started_recursive_ssm_parameter.py" + ``` ### Fetching secrets You can fetch secrets stored in Secrets Manager using `get_secrets`. -```python hl_lines="1 5" title="Fetching secrets" -from aws_lambda_powertools.utilities import parameters - -def handler(event, context): - # Retrieve a single secret - value = parameters.get_secret("my-secret") -``` +=== "getting_started_secret.py" + ```python hl_lines="3 13 20" + --8<-- "examples/parameters/src/getting_started_secret.py" + ``` ### Fetching app configurations @@ -71,13 +67,10 @@ You can fetch application configurations in AWS AppConfig using `get_app_config` The following will retrieve the latest version and store it in the cache. -```python hl_lines="1 5" title="Fetching latest config from AppConfig" -from aws_lambda_powertools.utilities import parameters - -def handler(event, context): - # Retrieve a single configuration, latest version - value: bytes = parameters.get_app_config(name="my_configuration", environment="my_env", application="my_app") -``` +=== "getting_started_appconfig.py" + ```python hl_lines="3 10 17" + --8<-- "examples/parameters/src/getting_started_appconfig.py" + ``` ## Advanced diff --git a/examples/parameters/src/getting_started_appconfig.py b/examples/parameters/src/getting_started_appconfig.py new file mode 100644 index 00000000000..e281cff4889 --- /dev/null +++ b/examples/parameters/src/getting_started_appconfig.py @@ -0,0 +1,18 @@ +import requests + +from aws_lambda_powertools.utilities import parameters +from aws_lambda_powertools.utilities.typing import LambdaContext + + +def lambda_handler(event: dict, context: LambdaContext): + try: + # Retrieve a single parameter + endpoint_comments: bytes = parameters.get_app_config(name="config", environment="dev", application="comments") + print(endpoint_comments) + + # the value of this parameter is https://jsonplaceholder.typicode.com/comments/ + comments: requests.Response = requests.get(endpoint_comments) + + return {"comments": comments.json()[:10], "statusCode": 200} + except parameters.exceptions.GetParameterError as error: + return {"comments": None, "message": str(error), "statusCode": 400} diff --git a/examples/parameters/src/getting_started_recursive_ssm_parameter.py b/examples/parameters/src/getting_started_recursive_ssm_parameter.py new file mode 100644 index 00000000000..af3d9cc15a8 --- /dev/null +++ b/examples/parameters/src/getting_started_recursive_ssm_parameter.py @@ -0,0 +1,23 @@ +import requests + +from aws_lambda_powertools.utilities import parameters +from aws_lambda_powertools.utilities.typing import LambdaContext + + +def lambda_handler(event: dict, context: LambdaContext): + try: + # Retrieve a single parameter + all_parameters: dict = parameters.get_parameters("/lambda-powertools/") + endpoint_comments = "https://jsonplaceholder.typicode.com/noexists/" + + for parameter, value in all_parameters.items(): + + if parameter == "endpoint_comments": + endpoint_comments = value + + # the value of parameter is https://jsonplaceholder.typicode.com/comments/ + comments: requests.Response = requests.get(endpoint_comments) + + return {"comments": comments.json()[:10]} + except parameters.exceptions.GetParameterError as error: + return {"comments": None, "message": str(error), "statusCode": 400} diff --git a/examples/parameters/src/getting_started_secret.py b/examples/parameters/src/getting_started_secret.py new file mode 100644 index 00000000000..40a6700e28a --- /dev/null +++ b/examples/parameters/src/getting_started_secret.py @@ -0,0 +1,21 @@ +import requests + +from aws_lambda_powertools.utilities import parameters +from aws_lambda_powertools.utilities.typing import LambdaContext + + +def lambda_handler(event: dict, context: LambdaContext): + + try: + # Usually an endpoint is not sensitive data, so we store it in SSM Parameters + endpoint_comments: str = parameters.get_parameter("/lambda-powertools/endpoint_comments") + # An API-KEY is a sensitive data and should be stored in SecretsManager + api_key: str = parameters.get_secret("/lambda-powertools/api-key") + + headers: dict = {"X-API-Key": api_key} + + comments: requests.Response = requests.get(endpoint_comments, headers=headers) + + return {"comments": comments.json()[:10], "statusCode": 200} + except parameters.exceptions.GetParameterError as error: + return {"comments": None, "message": str(error), "statusCode": 400} diff --git a/examples/parameters/src/getting_started_single_ssm_parameter.py b/examples/parameters/src/getting_started_single_ssm_parameter.py new file mode 100644 index 00000000000..ef3aa4a8751 --- /dev/null +++ b/examples/parameters/src/getting_started_single_ssm_parameter.py @@ -0,0 +1,17 @@ +import requests + +from aws_lambda_powertools.utilities import parameters +from aws_lambda_powertools.utilities.typing import LambdaContext + + +def lambda_handler(event: dict, context: LambdaContext): + try: + # Retrieve a single parameter + endpoint_comments: str = parameters.get_parameter("/lambda-powertools/endpoint_comments") + + # the value of this parameter is https://jsonplaceholder.typicode.com/comments/ + comments: requests.Response = requests.get(endpoint_comments) + + return {"comments": comments.json()[:10], "statusCode": 200} + except parameters.exceptions.GetParameterError as error: + return {"comments": None, "message": str(error), "statusCode": 400} From 6e16fe36e652e12273d71e150cc22254ac81493c Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sat, 1 Oct 2022 17:37:15 +0100 Subject: [PATCH 02/19] chore(parameters): adding cache examples --- docs/utilities/parameters.md | 35 ++++++++++--------- .../parameters/src/appconfig_with_cache.py | 19 ++++++++++ .../src/getting_started_appconfig.py | 1 - ...getting_started_recursive_ssm_parameter.py | 4 +-- .../src/recursive_ssm_parameter_with_cache.py | 23 ++++++++++++ examples/parameters/src/secret_with_cache.py | 21 +++++++++++ .../src/single_ssm_parameter_with_cache.py | 17 +++++++++ 7 files changed, 101 insertions(+), 19 deletions(-) create mode 100644 examples/parameters/src/appconfig_with_cache.py create mode 100644 examples/parameters/src/recursive_ssm_parameter_with_cache.py create mode 100644 examples/parameters/src/secret_with_cache.py create mode 100644 examples/parameters/src/single_ssm_parameter_with_cache.py diff --git a/docs/utilities/parameters.md b/docs/utilities/parameters.md index 89898e74cb9..a22a8b54ac7 100644 --- a/docs/utilities/parameters.md +++ b/docs/utilities/parameters.md @@ -68,7 +68,7 @@ You can fetch application configurations in AWS AppConfig using `get_app_config` The following will retrieve the latest version and store it in the cache. === "getting_started_appconfig.py" - ```python hl_lines="3 10 17" + ```python hl_lines="3 10 16" --8<-- "examples/parameters/src/getting_started_appconfig.py" ``` @@ -77,28 +77,31 @@ The following will retrieve the latest version and store it in the cache. ### Adjusting cache TTL ???+ tip - `max_age` parameter is also available in high level functions like `get_parameter`, `get_secret`, etc. + `max_age` parameter is also available in underlying provider functions like `get()`, `get_multiple()`, etc. By default, we cache parameters retrieved in-memory for 5 seconds. -You can adjust how long we should keep values in cache by using the param `max_age`, when using `get()` or `get_multiple()` methods across all providers. +You can adjust how long we should keep values in cache by using the param `max_age`, when using `get_parameter()`, `get_parameters()` and `get_secret()` methods across all providers. -```python hl_lines="9" title="Caching parameter(s) value in memory for longer than 5 seconds" -from aws_lambda_powertools.utilities import parameters -from botocore.config import Config +=== "single_ssm_parameter_with_cache.py" + ```python hl_lines="3 10 16" + --8<-- "examples/parameters/src/single_ssm_parameter_with_cache.py" + ``` -config = Config(region_name="us-west-1") -ssm_provider = parameters.SSMProvider(config=config) +=== "recursive_ssm_parameter_with_cache.py" + ```python hl_lines="3 10 13 22" + --8<-- "examples/parameters/src/recursive_ssm_parameter_with_cache.py" + ``` -def handler(event, context): - # Retrieve a single parameter - value = ssm_provider.get("/my/parameter", max_age=60) # 1 minute +=== "secret_with_cache.py" + ```python hl_lines="3 13 20" + --8<-- "examples/parameters/src/secret_with_cache.py" + ``` - # Retrieve multiple parameters from a path prefix - values = ssm_provider.get_multiple("/my/path/prefix", max_age=60) - for k, v in values.items(): - print(f"{k}: {v}") -``` +=== "appconfig_with_cache.py" + ```python hl_lines="3 10 11 18" + --8<-- "examples/parameters/src/appconfig_with_cache.py" + ``` ### Always fetching the latest diff --git a/examples/parameters/src/appconfig_with_cache.py b/examples/parameters/src/appconfig_with_cache.py new file mode 100644 index 00000000000..ebcbf834964 --- /dev/null +++ b/examples/parameters/src/appconfig_with_cache.py @@ -0,0 +1,19 @@ +import requests + +from aws_lambda_powertools.utilities import parameters +from aws_lambda_powertools.utilities.typing import LambdaContext + + +def lambda_handler(event: dict, context: LambdaContext): + try: + # Retrieve a single parameter + endpoint_comments: bytes = parameters.get_app_config( + name="config", environment="dev", application="comments", max_age=20 + ) + + # the value of this parameter is https://jsonplaceholder.typicode.com/comments/ + comments: requests.Response = requests.get(endpoint_comments) + + return {"comments": comments.json()[:10], "statusCode": 200} + except parameters.exceptions.GetParameterError as error: + return {"comments": None, "message": str(error), "statusCode": 400} diff --git a/examples/parameters/src/getting_started_appconfig.py b/examples/parameters/src/getting_started_appconfig.py index e281cff4889..01fc0adec3b 100644 --- a/examples/parameters/src/getting_started_appconfig.py +++ b/examples/parameters/src/getting_started_appconfig.py @@ -8,7 +8,6 @@ def lambda_handler(event: dict, context: LambdaContext): try: # Retrieve a single parameter endpoint_comments: bytes = parameters.get_app_config(name="config", environment="dev", application="comments") - print(endpoint_comments) # the value of this parameter is https://jsonplaceholder.typicode.com/comments/ comments: requests.Response = requests.get(endpoint_comments) diff --git a/examples/parameters/src/getting_started_recursive_ssm_parameter.py b/examples/parameters/src/getting_started_recursive_ssm_parameter.py index af3d9cc15a8..5c7acbc0eab 100644 --- a/examples/parameters/src/getting_started_recursive_ssm_parameter.py +++ b/examples/parameters/src/getting_started_recursive_ssm_parameter.py @@ -6,8 +6,8 @@ def lambda_handler(event: dict, context: LambdaContext): try: - # Retrieve a single parameter - all_parameters: dict = parameters.get_parameters("/lambda-powertools/") + # Retrieve multiple parameters from a path prefix + all_parameters: dict = parameters.get_parameters("/lambda-powertools/", max_age=20) endpoint_comments = "https://jsonplaceholder.typicode.com/noexists/" for parameter, value in all_parameters.items(): diff --git a/examples/parameters/src/recursive_ssm_parameter_with_cache.py b/examples/parameters/src/recursive_ssm_parameter_with_cache.py new file mode 100644 index 00000000000..5c7acbc0eab --- /dev/null +++ b/examples/parameters/src/recursive_ssm_parameter_with_cache.py @@ -0,0 +1,23 @@ +import requests + +from aws_lambda_powertools.utilities import parameters +from aws_lambda_powertools.utilities.typing import LambdaContext + + +def lambda_handler(event: dict, context: LambdaContext): + try: + # Retrieve multiple parameters from a path prefix + all_parameters: dict = parameters.get_parameters("/lambda-powertools/", max_age=20) + endpoint_comments = "https://jsonplaceholder.typicode.com/noexists/" + + for parameter, value in all_parameters.items(): + + if parameter == "endpoint_comments": + endpoint_comments = value + + # the value of parameter is https://jsonplaceholder.typicode.com/comments/ + comments: requests.Response = requests.get(endpoint_comments) + + return {"comments": comments.json()[:10]} + except parameters.exceptions.GetParameterError as error: + return {"comments": None, "message": str(error), "statusCode": 400} diff --git a/examples/parameters/src/secret_with_cache.py b/examples/parameters/src/secret_with_cache.py new file mode 100644 index 00000000000..651d949f3bc --- /dev/null +++ b/examples/parameters/src/secret_with_cache.py @@ -0,0 +1,21 @@ +import requests + +from aws_lambda_powertools.utilities import parameters +from aws_lambda_powertools.utilities.typing import LambdaContext + + +def lambda_handler(event: dict, context: LambdaContext): + + try: + # Usually an endpoint is not sensitive data, so we store it in SSM Parameters + endpoint_comments: str = parameters.get_parameter("/lambda-powertools/endpoint_comments") + # An API-KEY is a sensitive data and should be stored in SecretsManager + api_key: str = parameters.get_secret("/lambda-powertools/api-key", max_age=20) + + headers: dict = {"X-API-Key": api_key} + + comments: requests.Response = requests.get(endpoint_comments, headers=headers) + + return {"comments": comments.json()[:10], "statusCode": 200} + except parameters.exceptions.GetParameterError as error: + return {"comments": None, "message": str(error), "statusCode": 400} diff --git a/examples/parameters/src/single_ssm_parameter_with_cache.py b/examples/parameters/src/single_ssm_parameter_with_cache.py new file mode 100644 index 00000000000..39a21a37789 --- /dev/null +++ b/examples/parameters/src/single_ssm_parameter_with_cache.py @@ -0,0 +1,17 @@ +import requests + +from aws_lambda_powertools.utilities import parameters +from aws_lambda_powertools.utilities.typing import LambdaContext + + +def lambda_handler(event: dict, context: LambdaContext): + try: + # Retrieve a single parameter with 20s cache + endpoint_comments: str = parameters.get_parameter("/lambda-powertools/endpoint_comments", max_age=20) + + # the value of this parameter is https://jsonplaceholder.typicode.com/comments/ + comments: requests.Response = requests.get(endpoint_comments) + + return {"comments": comments.json()[:10], "statusCode": 200} + except parameters.exceptions.GetParameterError as error: + return {"comments": None, "message": str(error), "statusCode": 400} From ba513154c4d39487a5a51ede32eb56dd5d00d6c1 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sun, 2 Oct 2022 00:33:22 +0100 Subject: [PATCH 03/19] chore(parameters): adding builtin examples and fix mypy errors --- docs/utilities/parameters.md | 82 ++++++++++--------- .../parameters/src/appconfig_force_fetch.py | 21 +++++ .../parameters/src/appconfig_with_cache.py | 4 +- ...uiltin_provider_ssm_recursive_parameter.py | 44 ++++++++++ .../builtin_provider_ssm_single_parameter.py | 25 ++++++ .../src/builtin_provider_ssm_with_decrypt.py | 28 +++++++ .../builtin_provider_ssm_with_no_recursive.py | 39 +++++++++ .../src/getting_started_appconfig.py | 4 +- ...getting_started_recursive_ssm_parameter.py | 4 +- .../parameters/src/getting_started_secret.py | 6 +- .../getting_started_single_ssm_parameter.py | 6 +- .../recursive_ssm_parameter_force_fetch.py | 25 ++++++ .../src/recursive_ssm_parameter_with_cache.py | 4 +- examples/parameters/src/secret_force_fetch.py | 23 ++++++ examples/parameters/src/secret_with_cache.py | 6 +- .../src/single_ssm_parameter_force_fetch.py | 19 +++++ .../src/single_ssm_parameter_with_cache.py | 4 +- 17 files changed, 295 insertions(+), 49 deletions(-) create mode 100644 examples/parameters/src/appconfig_force_fetch.py create mode 100644 examples/parameters/src/builtin_provider_ssm_recursive_parameter.py create mode 100644 examples/parameters/src/builtin_provider_ssm_single_parameter.py create mode 100644 examples/parameters/src/builtin_provider_ssm_with_decrypt.py create mode 100644 examples/parameters/src/builtin_provider_ssm_with_no_recursive.py create mode 100644 examples/parameters/src/recursive_ssm_parameter_force_fetch.py create mode 100644 examples/parameters/src/secret_force_fetch.py create mode 100644 examples/parameters/src/single_ssm_parameter_force_fetch.py diff --git a/docs/utilities/parameters.md b/docs/utilities/parameters.md index a22a8b54ac7..cd54ae99bbc 100644 --- a/docs/utilities/parameters.md +++ b/docs/utilities/parameters.md @@ -34,7 +34,7 @@ This utility requires additional permissions to work as expected. | Secrets Manager | `get_secret`, `SecretsManager.get` | `secretsmanager:GetSecretValue` | | DynamoDB | `DynamoDBProvider.get` | `dynamodb:GetItem` | | DynamoDB | `DynamoDBProvider.get_multiple` | `dynamodb:Query` | -| App Config | `AppConfigProvider.get_app_config`, `get_app_config` | `appconfig:GetLatestConfiguration` and `appconfig:StartConfigurationSession` | +| App Config | `get_app_config`, `AppConfigProvider.get_app_config` | `appconfig:GetLatestConfiguration` and `appconfig:StartConfigurationSession` | ### Fetching parameters @@ -107,39 +107,44 @@ You can adjust how long we should keep values in cache by using the param `max_a If you'd like to always ensure you fetch the latest parameter from the store regardless if already available in cache, use `force_fetch` param. -```python hl_lines="5" title="Forcefully fetching the latest parameter whether TTL has expired or not" -from aws_lambda_powertools.utilities import parameters +=== "single_ssm_parameter_force_fetch.py" + ```python hl_lines="3 10 16" + --8<-- "examples/parameters/src/single_ssm_parameter_force_fetch.py" + ``` -def handler(event, context): - # Retrieve a single parameter - value = parameters.get_parameter("/my/parameter", force_fetch=True) -``` +=== "recursive_ssm_parameter_force_fetch.py" + ```python hl_lines="3 10 13 22" + --8<-- "examples/parameters/src/recursive_ssm_parameter_force_fetch.py" + ``` + +=== "secret_force_fetch.py" + ```python hl_lines="3 13 20" + --8<-- "examples/parameters/src/secret_force_fetch.py" + ``` + +=== "appconfig_force_fetch.py" + ```python hl_lines="3 10 11 18" + --8<-- "examples/parameters/src/appconfig_force_fetch.py" + ``` ### Built-in provider class For greater flexibility such as configuring the underlying SDK client used by built-in providers, you can use their respective Provider Classes directly. ???+ tip - This can be used to retrieve values from other regions, change the retry behavior, etc. + This is useful when you need to customize parameters for the SDK client, such as region, credentials, retries and others. For more information, read [botocore.config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html) and [boto3.session]( .com/v1/documentation/api/latest/reference/core/session.html#module-boto3.session). #### SSMProvider -```python hl_lines="5 9 12" title="Example with SSMProvider for further extensibility" -from aws_lambda_powertools.utilities import parameters -from botocore.config import Config - -config = Config(region_name="us-west-1") -ssm_provider = parameters.SSMProvider(config=config) # or boto3_session=boto3.Session() - -def handler(event, context): - # Retrieve a single parameter - value = ssm_provider.get("/my/parameter") +=== "builtin_provider_ssm_single_parameter.py" + ```python hl_lines="3 5 9 10 15" + --8<-- "examples/parameters/src/builtin_provider_ssm_single_parameter.py" + ``` - # Retrieve multiple parameters from a path prefix - values = ssm_provider.get_multiple("/my/path/prefix") - for k, v in values.items(): - print(f"{k}: {v}") -``` +=== "builtin_provider_ssm_recursive_parameter.py" + ```python hl_lines="3 5 9 10 15" + --8<-- "examples/parameters/src/builtin_provider_ssm_recursive_parameter.py" + ``` The AWS Systems Manager Parameter Store provider supports two additional arguments for the `get()` and `get_multiple()` methods: @@ -148,16 +153,17 @@ The AWS Systems Manager Parameter Store provider supports two additional argumen | **decrypt** | `False` | Will automatically decrypt the parameter. | | **recursive** | `True` | For `get_multiple()` only, will fetch all parameter values recursively based on a path prefix. | -```python hl_lines="6 8" title="Example with get() and get_multiple()" -from aws_lambda_powertools.utilities import parameters - -ssm_provider = parameters.SSMProvider() +You can create secure string parameters, which are parameters that have a plaintext parameter name and an encrypted parameter value. Read [here](https://docs.aws.amazon.com/kms/latest/developerguide/services-parameter-store.html) about best practices using KMS to secure your parameters. -def handler(event, context): - decrypted_value = ssm_provider.get("/my/encrypted/parameter", decrypt=True) +=== "builtin_provider_ssm_with_decrypt.py" + ```python hl_lines="3 5 9 10 15" + --8<-- "examples/parameters/src/builtin_provider_ssm_with_decrypt.py" + ``` - no_recursive_values = ssm_provider.get_multiple("/my/path/prefix", recursive=False) -``` +=== "builtin_provider_ssm_with_no_recursive.py" + ```python hl_lines="3 5 9 10 15" + --8<-- "examples/parameters/src/builtin_provider_ssm_with_no_recursive.py" + ``` #### SecretsProvider @@ -473,18 +479,18 @@ Here is the mapping between this utility's functions and methods and the underly | Secrets Manager | `SecretsManager.get` | `secretsmanager` | [get_secret_value](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/secretsmanager.html#SecretsManager.Client.get_secret_value) | | DynamoDB | `DynamoDBProvider.get` | `dynamodb` | ([Table resource](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#table)) | [get_item](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#DynamoDB.Table.get_item) | | DynamoDB | `DynamoDBProvider.get_multiple` | `dynamodb` | ([Table resource](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#table)) | [query](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#DynamoDB.Table.query) | -| App Config | `get_app_config` | `appconfig` | [get_configuration](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/appconfig.html#AppConfig.Client.get_configuration) | +| App Config | `get_app_config` | `appconfigdata` | [start_configuration_session](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/appconfigdata.html#AppConfigData.Client.start_configuration_session) and [get_latest_configuration](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/appconfigdata.html#AppConfigData.Client.get_latest_configuration) | ### Bring your own boto client You can use `boto3_client` parameter via any of the available [Provider Classes](#built-in-provider-class). Some providers expect a low level boto3 client while others expect a high level boto3 client, here is the mapping for each of them: -| Provider | Type | Boto client construction | -| --------------------------------------- | ---------- | ---------------------------- | -| [SSMProvider](#ssmprovider) | low level | `boto3.client("ssm")` | -| [SecretsProvider](#secretsprovider) | low level | `boto3.client("secrets")` | -| [AppConfigProvider](#appconfigprovider) | low level | `boto3.client("appconfig")` | -| [DynamoDBProvider](#dynamodbprovider) | high level | `boto3.resource("dynamodb")` | +| Provider | Type | Boto client construction | +| --------------------------------------- | ---------- | -------------------------------- | +| [SSMProvider](#ssmprovider) | low level | `boto3.client("ssm")` | +| [SecretsProvider](#secretsprovider) | low level | `boto3.client("secrets")` | +| [AppConfigProvider](#appconfigprovider) | low level | `boto3.client("appconfigdata")` | +| [DynamoDBProvider](#dynamodbprovider) | high level | `boto3.resource("dynamodb")` | Bringing them together in a single code snippet would look like this: diff --git a/examples/parameters/src/appconfig_force_fetch.py b/examples/parameters/src/appconfig_force_fetch.py new file mode 100644 index 00000000000..3cf5cbda6fd --- /dev/null +++ b/examples/parameters/src/appconfig_force_fetch.py @@ -0,0 +1,21 @@ +from typing import Any + +import requests + +from aws_lambda_powertools.utilities import parameters +from aws_lambda_powertools.utilities.typing import LambdaContext + + +def lambda_handler(event: dict, context: LambdaContext): + try: + # Retrieve a single parameter + endpoint_comments: Any = parameters.get_app_config( + name="config", environment="dev", application="comments", force_fetch=True + ) + + # the value of this parameter is https://jsonplaceholder.typicode.com/comments/ + comments: requests.Response = requests.get(endpoint_comments) + + return {"comments": comments.json()[:10], "statusCode": 200} + except parameters.exceptions.GetParameterError as error: + return {"comments": None, "message": str(error), "statusCode": 400} diff --git a/examples/parameters/src/appconfig_with_cache.py b/examples/parameters/src/appconfig_with_cache.py index ebcbf834964..55514e12ea7 100644 --- a/examples/parameters/src/appconfig_with_cache.py +++ b/examples/parameters/src/appconfig_with_cache.py @@ -1,3 +1,5 @@ +from typing import Any + import requests from aws_lambda_powertools.utilities import parameters @@ -7,7 +9,7 @@ def lambda_handler(event: dict, context: LambdaContext): try: # Retrieve a single parameter - endpoint_comments: bytes = parameters.get_app_config( + endpoint_comments: Any = parameters.get_app_config( name="config", environment="dev", application="comments", max_age=20 ) diff --git a/examples/parameters/src/builtin_provider_ssm_recursive_parameter.py b/examples/parameters/src/builtin_provider_ssm_recursive_parameter.py new file mode 100644 index 00000000000..ae6c6a29a87 --- /dev/null +++ b/examples/parameters/src/builtin_provider_ssm_recursive_parameter.py @@ -0,0 +1,44 @@ +from typing import Any + +import boto3 +import requests + +from aws_lambda_powertools.utilities import parameters +from aws_lambda_powertools.utilities.typing import LambdaContext + +# assuming role from another account to get parameter there +# see: https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html +sts_client = boto3.client("sts") +assumed_role_object = sts_client.assume_role( + RoleArn="arn:aws:iam::account-of-role-to-assume:role/name-of-role", RoleSessionName="RoleAssume1" +) +credentials = assumed_role_object["Credentials"] + +# using temporary credentials in your SSMProvider provider +# see: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html#module-boto3.session +boto3_session = boto3.session.Session( + region_name="us-east-1", + aws_access_key_id=credentials["AccessKeyId"], + aws_secret_access_key=credentials["SecretAccessKey"], + aws_session_token=credentials["SessionToken"], +) +ssm_provider = parameters.SSMProvider(boto3_session=boto3_session) + + +def lambda_handler(event: dict, context: LambdaContext): + try: + # Retrieve multiple parameters from a path prefix + all_parameters: Any = ssm_provider.get_multiple("/lambda-powertools/") + endpoint_comments = "https://jsonplaceholder.typicode.com/noexists/" + + for parameter, value in all_parameters.items(): + + if parameter == "endpoint_comments": + endpoint_comments = value + + # the value of parameter is https://jsonplaceholder.typicode.com/comments/ + comments: requests.Response = requests.get(endpoint_comments) + + return {"comments": comments.json()[:10]} + except parameters.exceptions.GetParameterError as error: + return {"comments": None, "message": str(error), "statusCode": 400} diff --git a/examples/parameters/src/builtin_provider_ssm_single_parameter.py b/examples/parameters/src/builtin_provider_ssm_single_parameter.py new file mode 100644 index 00000000000..28217b4deb2 --- /dev/null +++ b/examples/parameters/src/builtin_provider_ssm_single_parameter.py @@ -0,0 +1,25 @@ +from typing import Any + +import requests +from botocore.config import Config + +from aws_lambda_powertools.utilities import parameters +from aws_lambda_powertools.utilities.typing import LambdaContext + +# changing region_name, connect_timeout and retrie configurations +# see: https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html +config = Config(region_name="sa-east-1", connect_timeout=1, retries={"total_max_attempts": 2, "max_attempts": 5}) +ssm_provider = parameters.SSMProvider(config=config) + + +def lambda_handler(event: dict, context: LambdaContext): + try: + # Retrieve a single parameter + endpoint_comments: Any = ssm_provider.get("/lambda-powertools/endpoint_comments") + + # the value of this parameter is https://jsonplaceholder.typicode.com/comments/ + comments: requests.Response = requests.get(endpoint_comments) + + return {"comments": comments.json()[:10], "statusCode": 200} + except parameters.exceptions.GetParameterError as error: + return {"comments": None, "message": str(error), "statusCode": 400} diff --git a/examples/parameters/src/builtin_provider_ssm_with_decrypt.py b/examples/parameters/src/builtin_provider_ssm_with_decrypt.py new file mode 100644 index 00000000000..1bb4444726a --- /dev/null +++ b/examples/parameters/src/builtin_provider_ssm_with_decrypt.py @@ -0,0 +1,28 @@ +from typing import Any +from uuid import uuid4 + +import boto3 + +from aws_lambda_powertools.utilities import parameters +from aws_lambda_powertools.utilities.typing import LambdaContext + +ec2 = boto3.resource("ec2") +ssm_provider = parameters.SSMProvider() + + +def lambda_handler(event: dict, context: LambdaContext): + try: + # Retrieve the key pair from secure string parameter + ec2_pem: Any = ssm_provider.get("/lambda-powertools/ec2_pem", decrypt=True) + + name_key_pair = f"kp_{uuid4()}" + + ec2.import_key_pair(KeyName=name_key_pair, PublicKeyMaterial=ec2_pem) + + ec2.create_instances( + ImageId="ami-026b57f3c383c2eec", InstanceType="t2.micro", MinCount=1, MaxCount=1, KeyName=name_key_pair + ) + + return {"message": "EC2 created", "success": True} + except parameters.exceptions.GetParameterError as error: + return {"message": f"Error creating EC2 => {str(error)}", "success": False} diff --git a/examples/parameters/src/builtin_provider_ssm_with_no_recursive.py b/examples/parameters/src/builtin_provider_ssm_with_no_recursive.py new file mode 100644 index 00000000000..cf9cbf2020e --- /dev/null +++ b/examples/parameters/src/builtin_provider_ssm_with_no_recursive.py @@ -0,0 +1,39 @@ +from typing import Any + +import requests + +from aws_lambda_powertools.utilities import parameters +from aws_lambda_powertools.utilities.typing import LambdaContext + +ssm_provider = parameters.SSMProvider() + + +class ConfigNotFound(Exception): + ... + + +def lambda_handler(event: dict, context: LambdaContext): + try: + # Retrieve multiple parameters from a path prefix + # /config = root + # /config/endpoint = url + # /config/endpoint/query = querystring + all_parameters: Any = ssm_provider.get_multiple("/config", recursive=False) + endpoint_comments = "https://jsonplaceholder.typicode.com/comments/" + + for parameter, value in all_parameters.items(): + + # scheme is http or https + if "query" in parameter: + endpoint_comments = f"{endpoint_comments}{value}" + break + else: + # scheme config was not found because get_multiple is not recursive + raise ConfigNotFound("URL query was not found") + + # the value of parameter is https://jsonplaceholder.typicode.com/comments/ + comments: requests.Response = requests.get(endpoint_comments) + + return {"comments": comments.json()} + except parameters.exceptions.GetParameterError as error: + return {"comments": None, "message": str(error), "statusCode": 400} diff --git a/examples/parameters/src/getting_started_appconfig.py b/examples/parameters/src/getting_started_appconfig.py index 01fc0adec3b..fdb0c91d631 100644 --- a/examples/parameters/src/getting_started_appconfig.py +++ b/examples/parameters/src/getting_started_appconfig.py @@ -1,3 +1,5 @@ +from typing import Any + import requests from aws_lambda_powertools.utilities import parameters @@ -7,7 +9,7 @@ def lambda_handler(event: dict, context: LambdaContext): try: # Retrieve a single parameter - endpoint_comments: bytes = parameters.get_app_config(name="config", environment="dev", application="comments") + endpoint_comments: Any = parameters.get_app_config(name="config", environment="dev", application="comments") # the value of this parameter is https://jsonplaceholder.typicode.com/comments/ comments: requests.Response = requests.get(endpoint_comments) diff --git a/examples/parameters/src/getting_started_recursive_ssm_parameter.py b/examples/parameters/src/getting_started_recursive_ssm_parameter.py index 5c7acbc0eab..9cf48b39dde 100644 --- a/examples/parameters/src/getting_started_recursive_ssm_parameter.py +++ b/examples/parameters/src/getting_started_recursive_ssm_parameter.py @@ -1,3 +1,5 @@ +from typing import Any + import requests from aws_lambda_powertools.utilities import parameters @@ -7,7 +9,7 @@ def lambda_handler(event: dict, context: LambdaContext): try: # Retrieve multiple parameters from a path prefix - all_parameters: dict = parameters.get_parameters("/lambda-powertools/", max_age=20) + all_parameters: Any = parameters.get_parameters("/lambda-powertools/", max_age=20) endpoint_comments = "https://jsonplaceholder.typicode.com/noexists/" for parameter, value in all_parameters.items(): diff --git a/examples/parameters/src/getting_started_secret.py b/examples/parameters/src/getting_started_secret.py index 40a6700e28a..1f10394e834 100644 --- a/examples/parameters/src/getting_started_secret.py +++ b/examples/parameters/src/getting_started_secret.py @@ -1,3 +1,5 @@ +from typing import Any + import requests from aws_lambda_powertools.utilities import parameters @@ -8,9 +10,9 @@ def lambda_handler(event: dict, context: LambdaContext): try: # Usually an endpoint is not sensitive data, so we store it in SSM Parameters - endpoint_comments: str = parameters.get_parameter("/lambda-powertools/endpoint_comments") + endpoint_comments: Any = parameters.get_parameter("/lambda-powertools/endpoint_comments") # An API-KEY is a sensitive data and should be stored in SecretsManager - api_key: str = parameters.get_secret("/lambda-powertools/api-key") + api_key: Any = parameters.get_secret("/lambda-powertools/api-key") headers: dict = {"X-API-Key": api_key} diff --git a/examples/parameters/src/getting_started_single_ssm_parameter.py b/examples/parameters/src/getting_started_single_ssm_parameter.py index ef3aa4a8751..199b402cc4a 100644 --- a/examples/parameters/src/getting_started_single_ssm_parameter.py +++ b/examples/parameters/src/getting_started_single_ssm_parameter.py @@ -1,13 +1,15 @@ +from typing import Any + import requests from aws_lambda_powertools.utilities import parameters from aws_lambda_powertools.utilities.typing import LambdaContext -def lambda_handler(event: dict, context: LambdaContext): +def lambda_handler(event: dict, context: LambdaContext) -> dict: try: # Retrieve a single parameter - endpoint_comments: str = parameters.get_parameter("/lambda-powertools/endpoint_comments") + endpoint_comments: Any = parameters.get_parameter("/lambda-powertools/endpoint_comments") # the value of this parameter is https://jsonplaceholder.typicode.com/comments/ comments: requests.Response = requests.get(endpoint_comments) diff --git a/examples/parameters/src/recursive_ssm_parameter_force_fetch.py b/examples/parameters/src/recursive_ssm_parameter_force_fetch.py new file mode 100644 index 00000000000..6082a0173d4 --- /dev/null +++ b/examples/parameters/src/recursive_ssm_parameter_force_fetch.py @@ -0,0 +1,25 @@ +from typing import Any + +import requests + +from aws_lambda_powertools.utilities import parameters +from aws_lambda_powertools.utilities.typing import LambdaContext + + +def lambda_handler(event: dict, context: LambdaContext): + try: + # Retrieve multiple parameters from a path prefix + all_parameters: Any = parameters.get_parameters("/lambda-powertools/", force_fetch=True) + endpoint_comments = "https://jsonplaceholder.typicode.com/noexists/" + + for parameter, value in all_parameters.items(): + + if parameter == "endpoint_comments": + endpoint_comments = value + + # the value of parameter is https://jsonplaceholder.typicode.com/comments/ + comments: requests.Response = requests.get(endpoint_comments) + + return {"comments": comments.json()[:10]} + except parameters.exceptions.GetParameterError as error: + return {"comments": None, "message": str(error), "statusCode": 400} diff --git a/examples/parameters/src/recursive_ssm_parameter_with_cache.py b/examples/parameters/src/recursive_ssm_parameter_with_cache.py index 5c7acbc0eab..9cf48b39dde 100644 --- a/examples/parameters/src/recursive_ssm_parameter_with_cache.py +++ b/examples/parameters/src/recursive_ssm_parameter_with_cache.py @@ -1,3 +1,5 @@ +from typing import Any + import requests from aws_lambda_powertools.utilities import parameters @@ -7,7 +9,7 @@ def lambda_handler(event: dict, context: LambdaContext): try: # Retrieve multiple parameters from a path prefix - all_parameters: dict = parameters.get_parameters("/lambda-powertools/", max_age=20) + all_parameters: Any = parameters.get_parameters("/lambda-powertools/", max_age=20) endpoint_comments = "https://jsonplaceholder.typicode.com/noexists/" for parameter, value in all_parameters.items(): diff --git a/examples/parameters/src/secret_force_fetch.py b/examples/parameters/src/secret_force_fetch.py new file mode 100644 index 00000000000..121d9f57bfb --- /dev/null +++ b/examples/parameters/src/secret_force_fetch.py @@ -0,0 +1,23 @@ +from typing import Any + +import requests + +from aws_lambda_powertools.utilities import parameters +from aws_lambda_powertools.utilities.typing import LambdaContext + + +def lambda_handler(event: dict, context: LambdaContext): + + try: + # Usually an endpoint is not sensitive data, so we store it in SSM Parameters + endpoint_comments: Any = parameters.get_parameter("/lambda-powertools/endpoint_comments") + # An API-KEY is a sensitive data and should be stored in SecretsManager + api_key: Any = parameters.get_secret("/lambda-powertools/api-key", force_fetch=True) + + headers: dict = {"X-API-Key": api_key} + + comments: requests.Response = requests.get(endpoint_comments, headers=headers) + + return {"comments": comments.json()[:10], "statusCode": 200} + except parameters.exceptions.GetParameterError as error: + return {"comments": None, "message": str(error), "statusCode": 400} diff --git a/examples/parameters/src/secret_with_cache.py b/examples/parameters/src/secret_with_cache.py index 651d949f3bc..8d3ed927107 100644 --- a/examples/parameters/src/secret_with_cache.py +++ b/examples/parameters/src/secret_with_cache.py @@ -1,3 +1,5 @@ +from typing import Any + import requests from aws_lambda_powertools.utilities import parameters @@ -8,9 +10,9 @@ def lambda_handler(event: dict, context: LambdaContext): try: # Usually an endpoint is not sensitive data, so we store it in SSM Parameters - endpoint_comments: str = parameters.get_parameter("/lambda-powertools/endpoint_comments") + endpoint_comments: Any = parameters.get_parameter("/lambda-powertools/endpoint_comments") # An API-KEY is a sensitive data and should be stored in SecretsManager - api_key: str = parameters.get_secret("/lambda-powertools/api-key", max_age=20) + api_key: Any = parameters.get_secret("/lambda-powertools/api-key", max_age=20) headers: dict = {"X-API-Key": api_key} diff --git a/examples/parameters/src/single_ssm_parameter_force_fetch.py b/examples/parameters/src/single_ssm_parameter_force_fetch.py new file mode 100644 index 00000000000..7b45c1be216 --- /dev/null +++ b/examples/parameters/src/single_ssm_parameter_force_fetch.py @@ -0,0 +1,19 @@ +from typing import Any + +import requests + +from aws_lambda_powertools.utilities import parameters +from aws_lambda_powertools.utilities.typing import LambdaContext + + +def lambda_handler(event: dict, context: LambdaContext): + try: + # Retrieve a single parameter with 20s cache + endpoint_comments: Any = parameters.get_parameter("/lambda-powertools/endpoint_comments", force_fetch=True) + + # the value of this parameter is https://jsonplaceholder.typicode.com/comments/ + comments: requests.Response = requests.get(endpoint_comments) + + return {"comments": comments.json()[:10], "statusCode": 200} + except parameters.exceptions.GetParameterError as error: + return {"comments": None, "message": str(error), "statusCode": 400} diff --git a/examples/parameters/src/single_ssm_parameter_with_cache.py b/examples/parameters/src/single_ssm_parameter_with_cache.py index 39a21a37789..1be8632bc00 100644 --- a/examples/parameters/src/single_ssm_parameter_with_cache.py +++ b/examples/parameters/src/single_ssm_parameter_with_cache.py @@ -1,3 +1,5 @@ +from typing import Any + import requests from aws_lambda_powertools.utilities import parameters @@ -7,7 +9,7 @@ def lambda_handler(event: dict, context: LambdaContext): try: # Retrieve a single parameter with 20s cache - endpoint_comments: str = parameters.get_parameter("/lambda-powertools/endpoint_comments", max_age=20) + endpoint_comments: Any = parameters.get_parameter("/lambda-powertools/endpoint_comments", max_age=20) # the value of this parameter is https://jsonplaceholder.typicode.com/comments/ comments: requests.Response = requests.get(endpoint_comments) From 7119d93498e2351dd00bde69566d407c008a3248 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sun, 2 Oct 2022 00:53:32 +0100 Subject: [PATCH 04/19] chore(parameters): adding all providers with custom config --- docs/utilities/parameters.md | 43 ++++++------------- .../src/custom_boto3_all_providers.py | 17 ++++++++ 2 files changed, 31 insertions(+), 29 deletions(-) create mode 100644 examples/parameters/src/custom_boto3_all_providers.py diff --git a/docs/utilities/parameters.md b/docs/utilities/parameters.md index cd54ae99bbc..172584db07f 100644 --- a/docs/utilities/parameters.md +++ b/docs/utilities/parameters.md @@ -27,14 +27,15 @@ This utility requires additional permissions to work as expected. ???+ note Different parameter providers require different permissions. -| Provider | Function/Method | IAM Permission | -| ------------------- | ---------------------------------------------------- | --------------------------------------------------------------------------------------- | -| SSM Parameter Store | `get_parameter`, `SSMProvider.get` | `ssm:GetParameter` | -| SSM Parameter Store | `get_parameters`, `SSMProvider.get_multiple` | `ssm:GetParametersByPath` | -| Secrets Manager | `get_secret`, `SecretsManager.get` | `secretsmanager:GetSecretValue` | -| DynamoDB | `DynamoDBProvider.get` | `dynamodb:GetItem` | -| DynamoDB | `DynamoDBProvider.get_multiple` | `dynamodb:Query` | -| App Config | `get_app_config`, `AppConfigProvider.get_app_config` | `appconfig:GetLatestConfiguration` and `appconfig:StartConfigurationSession` | +| Provider | Function/Method | IAM Permission | +| ------------------- | -----------------------------------------------------------------| -----------------------------------------------------------------------------| +| SSM Parameter Store | `get_parameter`, `SSMProvider.get` | `ssm:GetParameter` | +| SSM Parameter Store | `get_parameters`, `SSMProvider.get_multiple` | `ssm:GetParametersByPath` | +| SSM Parameter Store | If using `decrypt=True` | You must add an additional permission `kms:Decrypt` | +| Secrets Manager | `get_secret`, `SecretsManager.get` | `secretsmanager:GetSecretValue` | +| DynamoDB | `DynamoDBProvider.get` | `dynamodb:GetItem` | +| DynamoDB | `DynamoDBProvider.get_multiple` | `dynamodb:Query` | +| App Config | `get_app_config`, `AppConfigProvider.get_app_config` | `appconfig:GetLatestConfiguration` and `appconfig:StartConfigurationSession` | ### Fetching parameters @@ -153,7 +154,7 @@ The AWS Systems Manager Parameter Store provider supports two additional argumen | **decrypt** | `False` | Will automatically decrypt the parameter. | | **recursive** | `True` | For `get_multiple()` only, will fetch all parameter values recursively based on a path prefix. | -You can create secure string parameters, which are parameters that have a plaintext parameter name and an encrypted parameter value. Read [here](https://docs.aws.amazon.com/kms/latest/developerguide/services-parameter-store.html) about best practices using KMS to secure your parameters. +You can create `SecureString` parameters, which are parameters that have a plaintext parameter name and an encrypted parameter value. If you don't use the `decrypt` argument, you will get an encrypted value. Read [here](https://docs.aws.amazon.com/kms/latest/developerguide/services-parameter-store.html) about best practices using KMS to secure your parameters. === "builtin_provider_ssm_with_decrypt.py" ```python hl_lines="3 5 9 10 15" @@ -494,26 +495,10 @@ You can use `boto3_client` parameter via any of the available [Provider Classes] Bringing them together in a single code snippet would look like this: -```python title="Example: passing a custom boto3 client for each provider" -import boto3 -from botocore.config import Config - -from aws_lambda_powertools.utilities import parameters - -config = Config(region_name="us-west-1") - -# construct boto clients with any custom configuration -ssm = boto3.client("ssm", config=config) -secrets = boto3.client("secrets", config=config) -appconfig = boto3.client("appconfig", config=config) -dynamodb = boto3.resource("dynamodb", config=config) - -ssm_provider = parameters.SSMProvider(boto3_client=ssm) -secrets_provider = parameters.SecretsProvider(boto3_client=secrets) -appconf_provider = parameters.AppConfigProvider(boto3_client=appconfig, environment="my_env", application="my_app") -dynamodb_provider = parameters.DynamoDBProvider(boto3_client=dynamodb, table_name="my-table") - -``` +=== "custom_boto3_all_providers.py" + ```python + --8<-- "examples/parameters/src/custom_boto3_all_providers.py" + ``` ???+ question "When is this useful?" Injecting a custom boto3 client can make unit/snapshot testing easier, including SDK customizations. diff --git a/examples/parameters/src/custom_boto3_all_providers.py b/examples/parameters/src/custom_boto3_all_providers.py new file mode 100644 index 00000000000..629eba6e5c6 --- /dev/null +++ b/examples/parameters/src/custom_boto3_all_providers.py @@ -0,0 +1,17 @@ +import boto3 +from botocore.config import Config + +from aws_lambda_powertools.utilities import parameters + +config = Config(region_name="us-west-1") + +# construct boto clients with any custom configuration +ssm = boto3.client("ssm", config=config) +secrets = boto3.client("secrets", config=config) +appconfig = boto3.client("appconfigdata", config=config) +dynamodb = boto3.resource("dynamodb", config=config) + +ssm_provider = parameters.SSMProvider(boto3_client=ssm) +secrets_provider = parameters.SecretsProvider(boto3_client=secrets) +appconf_provider = parameters.AppConfigProvider(boto3_client=appconfig, environment="my_env", application="my_app") +dynamodb_provider = parameters.DynamoDBProvider(boto3_client=dynamodb, table_name="my-table") From fd69844056fd3da0883aab7eee63af12d2145fe3 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sun, 2 Oct 2022 15:11:25 +0100 Subject: [PATCH 05/19] chore(parameters): adding custom provider --- docs/utilities/parameters.md | 67 +++++-------------- examples/parameters/src/custom_provider_s3.py | 52 ++++++++++++++ .../parameters/src/custom_provider_vault.py | 36 ++++++++++ .../src/working_with_own_provider_s3.py | 30 +++++++++ .../src/working_with_own_provider_vault.py | 35 ++++++++++ mypy.ini | 3 + poetry.lock | 29 +++++++- pyproject.toml | 1 + 8 files changed, 202 insertions(+), 51 deletions(-) create mode 100644 examples/parameters/src/custom_provider_s3.py create mode 100644 examples/parameters/src/custom_provider_vault.py create mode 100644 examples/parameters/src/working_with_own_provider_s3.py create mode 100644 examples/parameters/src/working_with_own_provider_vault.py diff --git a/docs/utilities/parameters.md b/docs/utilities/parameters.md index 172584db07f..cc2ccb66671 100644 --- a/docs/utilities/parameters.md +++ b/docs/utilities/parameters.md @@ -305,60 +305,27 @@ You can create your own custom parameter store provider by inheriting the `BaseP All transformation and caching logic is handled by the `get()` and `get_multiple()` methods from the base provider class. -Here is an example implementation using S3 as a custom parameter store: +Here are two examples of implementing a custom parameter store. One using an external service like [Hashicorp Vault](https://www.vaultproject.io/), a widely popular key-value and secret storage. -```python hl_lines="3 6 17 27" title="Creating a S3 Provider to fetch parameters" -import copy - -from aws_lambda_powertools.utilities import BaseProvider -import boto3 - -class S3Provider(BaseProvider): - bucket_name = None - client = None - - def __init__(self, bucket_name: str): - # Initialize the client to your custom parameter store - # E.g.: - - self.bucket_name = bucket_name - self.client = boto3.client("s3") - - def _get(self, name: str, **sdk_options) -> str: - # Retrieve a single value - # E.g.: - - sdk_options["Bucket"] = self.bucket_name - sdk_options["Key"] = name - - response = self.client.get_object(**sdk_options) - return - - def _get_multiple(self, path: str, **sdk_options) -> Dict[str, str]: - # Retrieve multiple values - # E.g.: - - list_sdk_options = copy.deepcopy(sdk_options) - - list_sdk_options["Bucket"] = self.bucket_name - list_sdk_options["Prefix"] = path - - list_response = self.client.list_objects_v2(**list_sdk_options) - - parameters = {} - - for obj in list_response.get("Contents", []): - get_sdk_options = copy.deepcopy(sdk_options) - - get_sdk_options["Bucket"] = self.bucket_name - get_sdk_options["Key"] = obj["Key"] +=== "working_with_own_provider_vault.py" + ```python hl_lines="3 5 9 10 15" + --8<-- "examples/parameters/src/working_with_own_provider_vault.py" + ``` - get_response = self.client.get_object(**get_sdk_options) +=== "custom_provider_vault.py" + ```python hl_lines="3 5 9 10 15" + --8<-- "examples/parameters/src/custom_provider_vault.py" + ``` - parameters[obj["Key"]] = get_response["Body"].read().decode() +=== "working_with_own_provider_s3.py" + ```python hl_lines="3 5 9 10 15" + --8<-- "examples/parameters/src/working_with_own_provider_s3.py" + ``` - return parameters -``` +=== "custom_provider_s3.py" + ```python hl_lines="3 5 9 10 15" + --8<-- "examples/parameters/src/custom_provider_s3.py" + ``` ### Deserializing values with transform parameter diff --git a/examples/parameters/src/custom_provider_s3.py b/examples/parameters/src/custom_provider_s3.py new file mode 100644 index 00000000000..7233ac0b307 --- /dev/null +++ b/examples/parameters/src/custom_provider_s3.py @@ -0,0 +1,52 @@ +import copy +from typing import Dict + +import boto3 + +from aws_lambda_powertools.utilities.parameters import BaseProvider + + +class S3Provider(BaseProvider): + def __init__(self, bucket_name: str): + # Initialize the client to your custom parameter store + # E.g.: + + super().__init__() + + self.bucket_name = bucket_name + self.client = boto3.client("s3") + + def _get(self, name: str, **sdk_options) -> str: + # Retrieve a single value + # E.g.: + + sdk_options["Bucket"] = self.bucket_name + sdk_options["Key"] = name + + response = self.client.get_object(**sdk_options) + return response["Body"].read().decode() + + def _get_multiple(self, path: str, **sdk_options) -> Dict[str, str]: + # Retrieve multiple values + # E.g.: + + list_sdk_options = copy.deepcopy(sdk_options) + + list_sdk_options["Bucket"] = self.bucket_name + list_sdk_options["Prefix"] = path + + list_response = self.client.list_objects_v2(**list_sdk_options) + + parameters = {} + + for obj in list_response.get("Contents", []): + get_sdk_options = copy.deepcopy(sdk_options) + + get_sdk_options["Bucket"] = self.bucket_name + get_sdk_options["Key"] = obj["Key"] + + get_response = self.client.get_object(**get_sdk_options) + + parameters[obj["Key"]] = get_response["Body"].read().decode() + + return parameters diff --git a/examples/parameters/src/custom_provider_vault.py b/examples/parameters/src/custom_provider_vault.py new file mode 100644 index 00000000000..06d0a929fff --- /dev/null +++ b/examples/parameters/src/custom_provider_vault.py @@ -0,0 +1,36 @@ +import json +from typing import Dict + +from hvac import Client + +from aws_lambda_powertools.utilities.parameters import BaseProvider + + +class VaultProvider(BaseProvider): + def __init__(self, vault_url: str, vault_token: str) -> None: + + super().__init__() + + self.vault_client = Client(url=vault_url, verify=False, timeout=10) + self.vault_client.token = vault_token + + def _get(self, name: str, **sdk_options) -> str: + + # for example proposal, the mountpoint is always /secret + kv_configuration = self.vault_client.secrets.kv.v2.read_secret(path=name) + + return json.dumps(kv_configuration["data"]["data"]) + + def _get_multiple(self, path: str, **sdk_options) -> Dict[str, str]: + + list_secrets = {} + all_secrets = self.vault_client.secrets.kv.v2.list_secrets(path=path) + + # for example proposal, the mountpoint is always /secret + for secret in all_secrets["data"]["keys"]: + kv_configuration = self.vault_client.secrets.kv.v2.read_secret(path=secret) + + for key, value in kv_configuration["data"]["data"].items(): + list_secrets[key] = value + + return list_secrets diff --git a/examples/parameters/src/working_with_own_provider_s3.py b/examples/parameters/src/working_with_own_provider_s3.py new file mode 100644 index 00000000000..d4f011a9e23 --- /dev/null +++ b/examples/parameters/src/working_with_own_provider_s3.py @@ -0,0 +1,30 @@ +from typing import Any + +import requests +from custom_provider_s3 import S3Provider + +from aws_lambda_powertools import Logger +from aws_lambda_powertools.utilities.typing import LambdaContext + +logger = Logger() + +s3_provider = S3Provider(bucket_name="bucket_name") + + +def lambda_handler(event: dict, context: LambdaContext): + + try: + # Retrieve a single parameter using key + endpoint_comments: Any = s3_provider.get("comments_endpoint") + # you can get all parameters using get_multiple and specifying a bucket prefix + # # for testing purposes we will not use it + all_parameters: Any = s3_provider.get_multiple("/") + logger.info(all_parameters) + + # the value of this parameter is https://jsonplaceholder.typicode.com/comments/ + comments: requests.Response = requests.get(endpoint_comments) + + return {"comments": comments.json()[:10], "statusCode": 200} + # general exception + except Exception as error: + return {"comments": None, "message": str(error), "statusCode": 400} diff --git a/examples/parameters/src/working_with_own_provider_vault.py b/examples/parameters/src/working_with_own_provider_vault.py new file mode 100644 index 00000000000..7be9ea60242 --- /dev/null +++ b/examples/parameters/src/working_with_own_provider_vault.py @@ -0,0 +1,35 @@ +from typing import Any + +import hvac +import requests +from custom_provider_vault import VaultProvider + +from aws_lambda_powertools import Logger +from aws_lambda_powertools.utilities.typing import LambdaContext + +logger = Logger() + +# In production you must use Vault over HTTPS and certificates. +vault_provider = VaultProvider(vault_url="http://192.168.68.105:8200/", vault_token="YOUR_TOKEN") + + +def lambda_handler(event: dict, context: LambdaContext): + + try: + # Retrieve a single parameter + endpoint_comments: Any = vault_provider.get("comments_endpoint", transform="json") + + # you can get all parameters using get_multiple and specifying vault mount point + # # for testing purposes we will not use it + all_parameters: Any = vault_provider.get_multiple("/") + logger.info(all_parameters) + + # the value of this parameter is https://jsonplaceholder.typicode.com/comments/ + comments: requests.Response = requests.get(endpoint_comments["url"]) + + return {"comments": comments.json()[:10], "statusCode": 200} + except hvac.exceptions.InvalidPath as error: + return {"comments": None, "message": str(error), "statusCode": 400} + # general exception + except Exception as error: + return {"comments": None, "message": str(error), "statusCode": 400} diff --git a/mypy.ini b/mypy.ini index 4da15d3898a..7f983cce05b 100644 --- a/mypy.ini +++ b/mypy.ini @@ -46,3 +46,6 @@ ignore_missing_imports = True [mypy-snappy] ignore_missing_imports = True + +[mypy-hvac] +ignore_missing_imports = True diff --git a/poetry.lock b/poetry.lock index 7ea9941e137..389b72fd324 100644 --- a/poetry.lock +++ b/poetry.lock @@ -465,6 +465,18 @@ python-versions = ">=3.6" gitdb = ">=4.0.1,<5" typing-extensions = {version = ">=3.7.4.3", markers = "python_version < \"3.10\""} +[[package]] +name = "hvac" +version = "1.0.2" +description = "HashiCorp Vault API client" +category = "dev" +optional = false +python-versions = ">=3.6.2,<4.0.0" + +[package.dependencies] +pyhcl = ">=0.4.4,<0.5.0" +requests = ">=2.27.1,<3.0.0" + [[package]] name = "idna" version = "3.3" @@ -985,6 +997,14 @@ python-versions = ">=3.6" [package.extras] plugins = ["importlib-metadata"] +[[package]] +name = "pyhcl" +version = "0.4.4" +description = "HCL configuration parser for python" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "pymdown-extensions" version = "9.5" @@ -1383,7 +1403,7 @@ pydantic = ["pydantic", "email-validator"] [metadata] lock-version = "1.1" python-versions = "^3.6.2" -content-hash = "38255d108f6ff5c2539c3df5d1924fc8ec5eb1baa74a249734caf36e48f9e111" +content-hash = "03ade04294dc25828c2ed9ecf5bd49d5a893405b5068e4787401509dd1a1d177" [metadata.files] atomicwrites = [ @@ -1599,6 +1619,10 @@ gitpython = [ {file = "GitPython-3.1.20-py3-none-any.whl", hash = "sha256:b1e1c269deab1b08ce65403cf14e10d2ef1f6c89e33ea7c5e5bb0222ea593b8a"}, {file = "GitPython-3.1.20.tar.gz", hash = "sha256:df0e072a200703a65387b0cfdf0466e3bab729c0458cf6b7349d0e9877636519"}, ] +hvac = [ + {file = "hvac-1.0.2-py3-none-any.whl", hash = "sha256:e8256343de2576b18bc8d49f09a04c728f2a8f3a866825bb413aa4f9ebab1fea"}, + {file = "hvac-1.0.2.tar.gz", hash = "sha256:e4028c5c0ecc7b7fcf6a54d290f99240e5abcdb9ffe442d1c14f061310d4c61c"}, +] idna = [ {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, @@ -1894,6 +1918,9 @@ pygments = [ {file = "Pygments-2.13.0-py3-none-any.whl", hash = "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"}, {file = "Pygments-2.13.0.tar.gz", hash = "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1"}, ] +pyhcl = [ + {file = "pyhcl-0.4.4.tar.gz", hash = "sha256:2d9b9dcdf1023d812bfed561ba72c99104c5b3f52e558d595130a44ce081b003"}, +] pymdown-extensions = [ {file = "pymdown_extensions-9.5-py3-none-any.whl", hash = "sha256:ec141c0f4983755349f0c8710416348d1a13753976c028186ed14f190c8061c4"}, {file = "pymdown_extensions-9.5.tar.gz", hash = "sha256:3ef2d998c0d5fa7eb09291926d90d69391283561cf6306f85cd588a5eb5befa0"}, diff --git a/pyproject.toml b/pyproject.toml index 0e62a392ee4..6f65d211c84 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,6 +73,7 @@ mkdocs-material = { version = "^8.5.3", python = ">=3.7" } filelock = { version = "^3.8.0", python = ">=3.7" } # Maintenance: 2022-09-19 pinned mako to fix vulnerability as a pdoc3 dependency. Remove once we drop python 3.6. Mako = {version = "1.2.3", python = ">=3.7"} +hvac = "^1.0.2" [tool.poetry.extras] pydantic = ["pydantic", "email-validator"] From 0ce8f426410ab9883c832f847653ac728d75dc6c Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sun, 2 Oct 2022 15:16:08 +0100 Subject: [PATCH 06/19] chore(parameters): adding custom provider --- docs/utilities/parameters.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/utilities/parameters.md b/docs/utilities/parameters.md index cc2ccb66671..f8f80e73919 100644 --- a/docs/utilities/parameters.md +++ b/docs/utilities/parameters.md @@ -305,7 +305,7 @@ You can create your own custom parameter store provider by inheriting the `BaseP All transformation and caching logic is handled by the `get()` and `get_multiple()` methods from the base provider class. -Here are two examples of implementing a custom parameter store. One using an external service like [Hashicorp Vault](https://www.vaultproject.io/), a widely popular key-value and secret storage. +Here are two examples of implementing a custom parameter store. One using an external service like [Hashicorp Vault](https://www.vaultproject.io/), a widely popular key-value and secret storage and the other one using [Amazon S3](https://aws.amazon.com/s3/?nc1=h_ls), a popular an object storage. === "working_with_own_provider_vault.py" ```python hl_lines="3 5 9 10 15" From cd85cfbf98dfd73a62000ad4f10d1528fe9b801b Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sun, 2 Oct 2022 15:21:39 +0100 Subject: [PATCH 07/19] chore(parameters): adding custom provider --- docs/utilities/parameters.md | 15 +++-------- .../parameters/src/builtin_provider_secret.py | 27 +++++++++++++++++++ .../builtin_provider_ssm_with_no_recursive.py | 4 +-- 3 files changed, 33 insertions(+), 13 deletions(-) create mode 100644 examples/parameters/src/builtin_provider_secret.py diff --git a/docs/utilities/parameters.md b/docs/utilities/parameters.md index f8f80e73919..a784fcc5783 100644 --- a/docs/utilities/parameters.md +++ b/docs/utilities/parameters.md @@ -168,17 +168,10 @@ You can create `SecureString` parameters, which are parameters that have a plain #### SecretsProvider -```python hl_lines="5 9" title="Example with SecretsProvider for further extensibility" -from aws_lambda_powertools.utilities import parameters -from botocore.config import Config - -config = Config(region_name="us-west-1") -secrets_provider = parameters.SecretsProvider(config=config) - -def handler(event, context): - # Retrieve a single secret - value = secrets_provider.get("my-secret") -``` +=== "builtin_provider_secret.py" + ```python hl_lines="3 5 9 10 15" + --8<-- "examples/parameters/src/builtin_provider_secret.py" + ``` #### DynamoDBProvider diff --git a/examples/parameters/src/builtin_provider_secret.py b/examples/parameters/src/builtin_provider_secret.py new file mode 100644 index 00000000000..449664c1863 --- /dev/null +++ b/examples/parameters/src/builtin_provider_secret.py @@ -0,0 +1,27 @@ +from typing import Any + +import requests +from botocore.config import Config + +from aws_lambda_powertools.utilities import parameters +from aws_lambda_powertools.utilities.typing import LambdaContext + +config = Config(region_name="sa-east-1", connect_timeout=1, retries={"total_max_attempts": 2, "max_attempts": 5}) +ssm_provider = parameters.SecretsProvider(config=config) + + +def lambda_handler(event: dict, context: LambdaContext): + + try: + # Usually an endpoint is not sensitive data, so we store it in SSM Parameters + endpoint_comments: Any = parameters.get_parameter("/lambda-powertools/endpoint_comments") + # An API-KEY is a sensitive data and should be stored in SecretsManager + api_key: Any = ssm_provider.get("/lambda-powertools/api-key") + + headers: dict = {"X-API-Key": api_key} + + comments: requests.Response = requests.get(endpoint_comments, headers=headers) + + return {"comments": comments.json()[:10], "statusCode": 200} + except parameters.exceptions.GetParameterError as error: + return {"comments": None, "message": str(error), "statusCode": 400} diff --git a/examples/parameters/src/builtin_provider_ssm_with_no_recursive.py b/examples/parameters/src/builtin_provider_ssm_with_no_recursive.py index cf9cbf2020e..0f92d27bfbc 100644 --- a/examples/parameters/src/builtin_provider_ssm_with_no_recursive.py +++ b/examples/parameters/src/builtin_provider_ssm_with_no_recursive.py @@ -23,13 +23,13 @@ def lambda_handler(event: dict, context: LambdaContext): for parameter, value in all_parameters.items(): - # scheme is http or https + # query parameter is used to query endpoint if "query" in parameter: endpoint_comments = f"{endpoint_comments}{value}" break else: # scheme config was not found because get_multiple is not recursive - raise ConfigNotFound("URL query was not found") + raise ConfigNotFound("URL query parameter was not found") # the value of parameter is https://jsonplaceholder.typicode.com/comments/ comments: requests.Response = requests.get(endpoint_comments) From abe8d72ec3e4e6a3cbd282013e9aa57ff752f373 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sun, 2 Oct 2022 16:45:34 +0100 Subject: [PATCH 08/19] chore(parameters): adding DynamoDB provider --- docs/utilities/parameters.md | 94 +++++++------------ .../sam/sam_dynamodb_custom_fields.yaml | 22 +++++ .../sam/sam_dynamodb_table_recursive.yaml | 22 +++++ .../sam/sam_dynamodb_table_single.yaml | 18 ++++ ...iltin_provider_dynamodb_custom_endpoint.py | 22 +++++ ...builtin_provider_dynamodb_custom_fields.py | 24 +++++ ...n_provider_dynamodb_recursive_parameter.py | 33 +++++++ ...ltin_provider_dynamodb_single_parameter.py | 22 +++++ 8 files changed, 198 insertions(+), 59 deletions(-) create mode 100644 examples/parameters/sam/sam_dynamodb_custom_fields.yaml create mode 100644 examples/parameters/sam/sam_dynamodb_table_recursive.yaml create mode 100644 examples/parameters/sam/sam_dynamodb_table_single.yaml create mode 100644 examples/parameters/src/builtin_provider_dynamodb_custom_endpoint.py create mode 100644 examples/parameters/src/builtin_provider_dynamodb_custom_fields.py create mode 100644 examples/parameters/src/builtin_provider_dynamodb_recursive_parameter.py create mode 100644 examples/parameters/src/builtin_provider_dynamodb_single_parameter.py diff --git a/docs/utilities/parameters.md b/docs/utilities/parameters.md index a784fcc5783..ef0f2ba1b10 100644 --- a/docs/utilities/parameters.md +++ b/docs/utilities/parameters.md @@ -191,25 +191,22 @@ For single parameters, you must use `id` as the [partition key](https://docs.aws With this table, `dynamodb_provider.get("my-param")` will return `my-value`. -=== "app.py" - ```python hl_lines="3 7" - from aws_lambda_powertools.utilities import parameters - - dynamodb_provider = parameters.DynamoDBProvider(table_name="my-table") - - def handler(event, context): - # Retrieve a value from DynamoDB - value = dynamodb_provider.get("my-parameter") - ``` +=== "builtin_provider_dynamodb_single_parameter.py" + ```python hl_lines="3 5 9 10 15" + --8<-- "examples/parameters/src/builtin_provider_dynamodb_single_parameter.py" + ``` -=== "DynamoDB Local example" - You can initialize the DynamoDB provider pointing to [DynamoDB Local](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.html) using `endpoint_url` parameter: +=== "sam_dynamodb_table_single.yaml" + ```yaml hl_lines="3 5 9 10 15" + --8<-- "examples/parameters/sam/sam_dynamodb_table_single.yaml" + ``` - ```python hl_lines="3" - from aws_lambda_powertools.utilities import parameters +You can initialize the DynamoDB provider pointing to [DynamoDB Local](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.html) using `endpoint_url` parameter: - dynamodb_provider = parameters.DynamoDBProvider(table_name="my-table", endpoint_url="http://localhost:8000") - ``` +=== "builtin_provider_dynamodb_custom_endpoint.py" + ```python hl_lines="3 5 9 10 15" + --8<-- "examples/parameters/src/builtin_provider_dynamodb_custom_endpoint.py" + ``` **DynamoDB table structure for multiple values parameters** @@ -219,39 +216,22 @@ You can retrieve multiple parameters sharing the same `id` by having a sort key DynamoDB table with `id` primary key, `sk` as sort key` and `value` as attribute - | id | sk | value | - | ----------- | ------- | ---------- | - | my-hash-key | param-a | my-value-a | - | my-hash-key | param-b | my-value-b | - | my-hash-key | param-c | my-value-c | - -With this table, `dynamodb_provider.get_multiple("my-hash-key")` will return a dictionary response in the shape of `sk:value`. + | id | sk | value | + | -------| ----------------- | ---------------------------------------------- | + | config | endpoint_comments | | + | config | limit | 10 | -=== "app.py" - ```python hl_lines="3 8" - from aws_lambda_powertools.utilities import parameters +With this table, `dynamodb_provider.get_multiple("config")` will return a dictionary response in the shape of `sk:value`. - dynamodb_provider = parameters.DynamoDBProvider(table_name="my-table") - - def handler(event, context): - # Retrieve multiple values by performing a Query on the DynamoDB table - # This returns a dict with the sort key attribute as dict key. - parameters = dynamodb_provider.get_multiple("my-hash-key") - for k, v in parameters.items(): - # k: param-a - # v: "my-value-a" - print(f"{k}: {v}") - ``` - -=== "parameters dict response" +=== "builtin_provider_dynamodb_recursive_parameter.py" + ```python hl_lines="3 5 9 10 15" + --8<-- "examples/parameters/src/builtin_provider_dynamodb_recursive_parameter.py" + ``` - ```json - { - "param-a": "my-value-a", - "param-b": "my-value-b", - "param-c": "my-value-c" - } - ``` +=== "sam_dynamodb_table_recursive.yaml" + ```yaml hl_lines="3 5 9 10 15" + --8<-- "examples/parameters/sam/sam_dynamodb_table_recursive.yaml" + ``` **Customizing DynamoDBProvider** @@ -264,19 +244,15 @@ DynamoDB provider can be customized at initialization to match your table struct | **sort_attr** | No | `sk` | Range key for the DynamoDB table. You don't need to set this if you don't use the `get_multiple()` method. | | **value_attr** | No | `value` | Name of the attribute containing the parameter value. | -```python hl_lines="3-8" title="Customizing DynamoDBProvider to suit your table design" -from aws_lambda_powertools.utilities import parameters - -dynamodb_provider = parameters.DynamoDBProvider( - table_name="my-table", - key_attr="MyKeyAttr", - sort_attr="MySortAttr", - value_attr="MyvalueAttr" -) +=== "builtin_provider_dynamodb_custom_fields.py" + ```python hl_lines="3 5 9 10 15" + --8<-- "examples/parameters/src/builtin_provider_dynamodb_custom_fields.py" + ``` -def handler(event, context): - value = dynamodb_provider.get("my-parameter") -``` +=== "sam_dynamodb_custom_fields.yaml" + ```yaml hl_lines="3 5 9 10 15" + --8<-- "examples/parameters/sam/sam_dynamodb_custom_fields.yaml" + ``` #### AppConfigProvider @@ -298,7 +274,7 @@ You can create your own custom parameter store provider by inheriting the `BaseP All transformation and caching logic is handled by the `get()` and `get_multiple()` methods from the base provider class. -Here are two examples of implementing a custom parameter store. One using an external service like [Hashicorp Vault](https://www.vaultproject.io/), a widely popular key-value and secret storage and the other one using [Amazon S3](https://aws.amazon.com/s3/?nc1=h_ls), a popular an object storage. +Here are two examples of implementing a custom parameter store. One using an external service like [Hashicorp Vault](https://www.vaultproject.io/), a widely popular key-value and secret storage and the other one using [Amazon S3](https://aws.amazon.com/s3/?nc1=h_ls), a popular object storage. === "working_with_own_provider_vault.py" ```python hl_lines="3 5 9 10 15" diff --git a/examples/parameters/sam/sam_dynamodb_custom_fields.yaml b/examples/parameters/sam/sam_dynamodb_custom_fields.yaml new file mode 100644 index 00000000000..1c864a0e747 --- /dev/null +++ b/examples/parameters/sam/sam_dynamodb_custom_fields.yaml @@ -0,0 +1,22 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: 'DynamoDB Table example' +Resources: + ParameterTable: + Type: AWS::DynamoDB::Table + Properties: + TableName: ParameterTable + AttributeDefinitions: + - AttributeName: IdKeyAttr + AttributeType: S + - AttributeName: SkKeyAttr + AttributeType: S + KeySchema: + - AttributeName: IdKeyAttr + KeyType: HASH + - AttributeName: SkKeyAttr + KeyType: RANGE + TimeToLiveSpecification: + AttributeName: expiration + Enabled: true + BillingMode: PAY_PER_REQUEST diff --git a/examples/parameters/sam/sam_dynamodb_table_recursive.yaml b/examples/parameters/sam/sam_dynamodb_table_recursive.yaml new file mode 100644 index 00000000000..6e02c2f8f6d --- /dev/null +++ b/examples/parameters/sam/sam_dynamodb_table_recursive.yaml @@ -0,0 +1,22 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: 'DynamoDB Table example' +Resources: + ParameterTable: + Type: AWS::DynamoDB::Table + Properties: + TableName: ParameterTable + AttributeDefinitions: + - AttributeName: id + AttributeType: S + - AttributeName: sk + AttributeType: S + KeySchema: + - AttributeName: id + KeyType: HASH + - AttributeName: sk + KeyType: RANGE + TimeToLiveSpecification: + AttributeName: expiration + Enabled: true + BillingMode: PAY_PER_REQUEST diff --git a/examples/parameters/sam/sam_dynamodb_table_single.yaml b/examples/parameters/sam/sam_dynamodb_table_single.yaml new file mode 100644 index 00000000000..17d25553fa7 --- /dev/null +++ b/examples/parameters/sam/sam_dynamodb_table_single.yaml @@ -0,0 +1,18 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: 'DynamoDB Table example' +Resources: + ParameterTable: + Type: AWS::DynamoDB::Table + Properties: + TableName: ParameterTable + AttributeDefinitions: + - AttributeName: id + AttributeType: S + KeySchema: + - AttributeName: id + KeyType: HASH + TimeToLiveSpecification: + AttributeName: expiration + Enabled: true + BillingMode: PAY_PER_REQUEST diff --git a/examples/parameters/src/builtin_provider_dynamodb_custom_endpoint.py b/examples/parameters/src/builtin_provider_dynamodb_custom_endpoint.py new file mode 100644 index 00000000000..e77506f27d7 --- /dev/null +++ b/examples/parameters/src/builtin_provider_dynamodb_custom_endpoint.py @@ -0,0 +1,22 @@ +from typing import Any + +import requests + +from aws_lambda_powertools.utilities import parameters +from aws_lambda_powertools.utilities.typing import LambdaContext + +dynamodb_provider = parameters.DynamoDBProvider(table_name="ParameterTable", endpoint_url="http://localhost:8000") + + +def lambda_handler(event: dict, context: LambdaContext): + + try: + # Usually an endpoint is not sensitive data, so we store it in DynamoDB Table + endpoint_comments: Any = dynamodb_provider.get("comments_endpoint") + + comments: requests.Response = requests.get(endpoint_comments) + + return {"comments": comments.json()[:10], "statusCode": 200} + # general exception + except Exception as error: + return {"comments": None, "message": str(error), "statusCode": 400} diff --git a/examples/parameters/src/builtin_provider_dynamodb_custom_fields.py b/examples/parameters/src/builtin_provider_dynamodb_custom_fields.py new file mode 100644 index 00000000000..6936f7f0a19 --- /dev/null +++ b/examples/parameters/src/builtin_provider_dynamodb_custom_fields.py @@ -0,0 +1,24 @@ +from typing import Any + +import requests + +from aws_lambda_powertools.utilities import parameters +from aws_lambda_powertools.utilities.typing import LambdaContext + +dynamodb_provider = parameters.DynamoDBProvider( + table_name="ParameterTable", key_attr="IdKeyAttr", sort_attr="SkKeyAttr", value_attr="ValueAttr" +) + + +def lambda_handler(event: dict, context: LambdaContext): + + try: + # Usually an endpoint is not sensitive data, so we store it in DynamoDB Table + endpoint_comments: Any = dynamodb_provider.get("comments_endpoint") + + comments: requests.Response = requests.get(endpoint_comments) + + return {"comments": comments.json()[:10], "statusCode": 200} + # general exception + except Exception as error: + return {"comments": None, "message": str(error), "statusCode": 400} diff --git a/examples/parameters/src/builtin_provider_dynamodb_recursive_parameter.py b/examples/parameters/src/builtin_provider_dynamodb_recursive_parameter.py new file mode 100644 index 00000000000..7db0d4d913a --- /dev/null +++ b/examples/parameters/src/builtin_provider_dynamodb_recursive_parameter.py @@ -0,0 +1,33 @@ +from typing import Any + +import requests + +from aws_lambda_powertools.utilities import parameters +from aws_lambda_powertools.utilities.typing import LambdaContext + +dynamodb_provider = parameters.DynamoDBProvider(table_name="ParameterTable") + + +def lambda_handler(event: dict, context: LambdaContext): + + try: + # Retrieve multiple parameters using HASH KEY + all_parameters: Any = dynamodb_provider.get_multiple("config") + endpoint_comments = "https://jsonplaceholder.typicode.com/noexists/" + limit = 2 + + for parameter, value in all_parameters.items(): + + if parameter == "endpoint_comments": + endpoint_comments = value + + if parameter == "limit": + limit = int(value) + + # the value of parameter is https://jsonplaceholder.typicode.com/comments/ + comments: requests.Response = requests.get(endpoint_comments) + + return {"comments": comments.json()[limit]} + # general exception + except Exception as error: + return {"comments": None, "message": str(error), "statusCode": 400} diff --git a/examples/parameters/src/builtin_provider_dynamodb_single_parameter.py b/examples/parameters/src/builtin_provider_dynamodb_single_parameter.py new file mode 100644 index 00000000000..036058f2b33 --- /dev/null +++ b/examples/parameters/src/builtin_provider_dynamodb_single_parameter.py @@ -0,0 +1,22 @@ +from typing import Any + +import requests + +from aws_lambda_powertools.utilities import parameters +from aws_lambda_powertools.utilities.typing import LambdaContext + +dynamodb_provider = parameters.DynamoDBProvider(table_name="ParameterTable") + + +def lambda_handler(event: dict, context: LambdaContext): + + try: + # Usually an endpoint is not sensitive data, so we store it in DynamoDB Table + endpoint_comments: Any = dynamodb_provider.get("comments_endpoint") + + comments: requests.Response = requests.get(endpoint_comments) + + return {"comments": comments.json()[:10], "statusCode": 200} + # general exception + except Exception as error: + return {"comments": None, "message": str(error), "statusCode": 400} From f7ba42c98cb8ca13be1d7076978234f80891474e Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sun, 2 Oct 2022 17:05:02 +0100 Subject: [PATCH 09/19] chore(parameters): adding tests - mock --- docs/utilities/parameters.md | 83 +++++-------------- examples/parameters/src/custom_boto_client.py | 13 +++ examples/parameters/src/custom_boto_config.py | 13 +++ .../parameters/src/custom_boto_session.py | 13 +++ examples/parameters/tests/src/__init__.py | 0 examples/parameters/tests/src/single_mock.py | 7 ++ examples/parameters/tests/test_single_mock.py | 10 +++ 7 files changed, 76 insertions(+), 63 deletions(-) create mode 100644 examples/parameters/src/custom_boto_client.py create mode 100644 examples/parameters/src/custom_boto_config.py create mode 100644 examples/parameters/src/custom_boto_session.py create mode 100644 examples/parameters/tests/src/__init__.py create mode 100644 examples/parameters/tests/src/single_mock.py create mode 100644 examples/parameters/tests/test_single_mock.py diff --git a/docs/utilities/parameters.md b/docs/utilities/parameters.md index ef0f2ba1b10..e4c8f3af7c8 100644 --- a/docs/utilities/parameters.md +++ b/docs/utilities/parameters.md @@ -448,49 +448,20 @@ The **`config`** , **`boto3_session`**, and **`boto3_client`** parameters enabl When using VPC private endpoints, you can pass a custom client altogether. It's also useful for testing when injecting fake instances. -=== "Custom session" - - ```python hl_lines="2 4 5" - from aws_lambda_powertools.utilities import parameters - import boto3 - - boto3_session = boto3.session.Session() - ssm_provider = parameters.SSMProvider(boto3_session=boto3_session) - - def handler(event, context): - # Retrieve a single parameter - value = ssm_provider.get("/my/parameter") - ... - ``` -=== "Custom config" - - ```python hl_lines="2 4 5" - from aws_lambda_powertools.utilities import parameters - from botocore.config import Config - - boto_config = Config() - ssm_provider = parameters.SSMProvider(config=boto_config) - - def handler(event, context): - # Retrieve a single parameter - value = ssm_provider.get("/my/parameter") - ... - ``` - -=== "Custom client" - - ```python hl_lines="2 4 5" - from aws_lambda_powertools.utilities import parameters - import boto3 +=== "custom_boto_session.py" + ```python + --8<-- "examples/parameters/src/custom_boto_session.py" + ``` - boto3_client= boto3.client("ssm") - ssm_provider = parameters.SSMProvider(boto3_client=boto3_client) +=== "custom_boto_config.py" + ```python + --8<-- "examples/parameters/src/custom_boto_config.py" + ``` - def handler(event, context): - # Retrieve a single parameter - value = ssm_provider.get("/my/parameter") - ... - ``` +=== "custom_boto_client.py" + ```python + --8<-- "examples/parameters/src/custom_boto_client.py" + ``` ## Testing your code @@ -498,29 +469,15 @@ The **`config`** , **`boto3_session`**, and **`boto3_client`** parameters enabl For unit testing your applications, you can mock the calls to the parameters utility to avoid calling AWS APIs. This can be achieved in a number of ways - in this example, we use the [pytest monkeypatch fixture](https://docs.pytest.org/en/latest/how-to/monkeypatch.html) to patch the `parameters.get_parameter` method: -=== "tests.py" - ```python - from src import index - - def test_handler(monkeypatch): - - def mockreturn(name): - return "mock_value" - - monkeypatch.setattr(index.parameters, "get_parameter", mockreturn) - return_val = index.handler({}, {}) - assert return_val.get('message') == 'mock_value' - ``` - -=== "src/index.py" - ```python - from aws_lambda_powertools.utilities import parameters +=== "test_single_mock.py" + ```python + --8<-- "examples/parameters/tests/test_single_mock.py" + ``` - def handler(event, context): - # Retrieve a single parameter - value = parameters.get_parameter("my-parameter-name") - return {"message": value} - ``` +=== "single_mock.py" + ```python + --8<-- "examples/parameters/tests/src/single_mock.py" + ``` If we need to use this pattern across multiple tests, we can avoid repetition by refactoring to use our own pytest fixture: diff --git a/examples/parameters/src/custom_boto_client.py b/examples/parameters/src/custom_boto_client.py new file mode 100644 index 00000000000..be81bb574a3 --- /dev/null +++ b/examples/parameters/src/custom_boto_client.py @@ -0,0 +1,13 @@ +import boto3 + +from aws_lambda_powertools.utilities import parameters + +boto3_client = boto3.client("ssm") +ssm_provider = parameters.SSMProvider(boto3_client=boto3_client) + + +def handler(event, context): + # Retrieve a single parameter + value = ssm_provider.get("/my/parameter") + + return value diff --git a/examples/parameters/src/custom_boto_config.py b/examples/parameters/src/custom_boto_config.py new file mode 100644 index 00000000000..8401a9bab89 --- /dev/null +++ b/examples/parameters/src/custom_boto_config.py @@ -0,0 +1,13 @@ +from botocore.config import Config + +from aws_lambda_powertools.utilities import parameters + +boto_config = Config() +ssm_provider = parameters.SSMProvider(config=boto_config) + + +def handler(event, context): + # Retrieve a single parameter + value = ssm_provider.get("/my/parameter") + + return value diff --git a/examples/parameters/src/custom_boto_session.py b/examples/parameters/src/custom_boto_session.py new file mode 100644 index 00000000000..c65481aa305 --- /dev/null +++ b/examples/parameters/src/custom_boto_session.py @@ -0,0 +1,13 @@ +import boto3 + +from aws_lambda_powertools.utilities import parameters + +boto3_session = boto3.session.Session() +ssm_provider = parameters.SSMProvider(boto3_session=boto3_session) + + +def handler(event, context): + # Retrieve a single parameter + value = ssm_provider.get("/my/parameter") + + return value diff --git a/examples/parameters/tests/src/__init__.py b/examples/parameters/tests/src/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/examples/parameters/tests/src/single_mock.py b/examples/parameters/tests/src/single_mock.py new file mode 100644 index 00000000000..ef94df54f25 --- /dev/null +++ b/examples/parameters/tests/src/single_mock.py @@ -0,0 +1,7 @@ +from aws_lambda_powertools.utilities import parameters + + +def handler(event, context): + # Retrieve a single parameter + value = parameters.get_parameter("my-parameter-name") + return {"message": value} diff --git a/examples/parameters/tests/test_single_mock.py b/examples/parameters/tests/test_single_mock.py new file mode 100644 index 00000000000..c4d045be499 --- /dev/null +++ b/examples/parameters/tests/test_single_mock.py @@ -0,0 +1,10 @@ +import src.single_mock as single_mock + + +def test_handler(monkeypatch): + def mockreturn(name): + return "mock_value" + + monkeypatch.setattr(single_mock.parameters, "get_parameter", mockreturn) + return_val = single_mock.handler({}, {}) + assert return_val.get("message") == "mock_value" From 0231759ae32c938b981907e01a16e8e66af0828e Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sun, 2 Oct 2022 17:33:31 +0100 Subject: [PATCH 10/19] chore(parameters): adding more tests --- docs/utilities/parameters.md | 113 +++--------------- examples/parameters/tests/src/app.py | 10 ++ .../tests/test_clear_cache_global.py | 24 ++++ .../tests/test_clear_cache_method.py | 22 ++++ .../parameters/tests/test_with_fixture.py | 16 +++ .../parameters/tests/test_with_monkeypatch.py | 13 ++ 6 files changed, 104 insertions(+), 94 deletions(-) create mode 100644 examples/parameters/tests/src/app.py create mode 100644 examples/parameters/tests/test_clear_cache_global.py create mode 100644 examples/parameters/tests/test_clear_cache_method.py create mode 100644 examples/parameters/tests/test_with_fixture.py create mode 100644 examples/parameters/tests/test_with_monkeypatch.py diff --git a/docs/utilities/parameters.md b/docs/utilities/parameters.md index e4c8f3af7c8..8fa9251bce2 100644 --- a/docs/utilities/parameters.md +++ b/docs/utilities/parameters.md @@ -481,46 +481,20 @@ For unit testing your applications, you can mock the calls to the parameters uti If we need to use this pattern across multiple tests, we can avoid repetition by refactoring to use our own pytest fixture: -=== "tests.py" - ```python - import pytest - - from src import index - - @pytest.fixture - def mock_parameter_response(monkeypatch): - def mockreturn(name): - return "mock_value" - - monkeypatch.setattr(index.parameters, "get_parameter", mockreturn) - - # Pass our fixture as an argument to all tests where we want to mock the get_parameter response - def test_handler(mock_parameter_response): - return_val = index.handler({}, {}) - assert return_val.get('message') == 'mock_value' - - ``` +=== "test_with_fixture.py" + ```python + --8<-- "examples/parameters/tests/test_with_fixture.py" + ``` Alternatively, if we need more fully featured mocking (for example checking the arguments passed to `get_parameter`), we can use [unittest.mock](https://docs.python.org/3/library/unittest.mock.html) from the python stdlib instead of pytest's `monkeypatch` fixture. In this example, we use the [patch](https://docs.python.org/3/library/unittest.mock.html#unittest.mock.patch) decorator to replace the `aws_lambda_powertools.utilities.parameters.get_parameter` function with a [MagicMock](https://docs.python.org/3/library/unittest.mock.html#unittest.mock.MagicMock) object named `get_parameter_mock`. -=== "tests.py" - ```python - from unittest.mock import patch - from src import index - - # Replaces "aws_lambda_powertools.utilities.parameters.get_parameter" with a Mock object - @patch("aws_lambda_powertools.utilities.parameters.get_parameter") - def test_handler(get_parameter_mock): - get_parameter_mock.return_value = 'mock_value' - - return_val = index.handler({}, {}) - get_parameter_mock.assert_called_with("my-parameter-name") - assert return_val.get('message') == 'mock_value' - - ``` +=== "test_with_monkeypatch.py" + ```python + --8<-- "examples/parameters/tests/test_with_monkeypatch.py" + ``` ### Clearing cache @@ -528,66 +502,17 @@ Parameters utility caches all parameter values for performance and cost reasons. Within your tests, you can use `clear_cache` method available in [every provider](#built-in-provider-class). When using multiple providers or higher level functions like `get_parameter`, use `clear_caches` standalone function to clear cache globally. -=== "clear_cache method" - ```python hl_lines="9" - import pytest - - from src import app - - - @pytest.fixture(scope="function", autouse=True) - def clear_parameters_cache(): - yield - app.ssm_provider.clear_cache() # This will clear SSMProvider cache - - @pytest.fixture - def mock_parameter_response(monkeypatch): - def mockreturn(name): - return "mock_value" - - monkeypatch.setattr(app.ssm_provider, "get", mockreturn) - - # Pass our fixture as an argument to all tests where we want to mock the get_parameter response - def test_handler(mock_parameter_response): - return_val = app.handler({}, {}) - assert return_val.get('message') == 'mock_value' - ``` - -=== "global clear_caches" - ```python hl_lines="10" - import pytest - - from aws_lambda_powertools.utilities import parameters - from src import app - - - @pytest.fixture(scope="function", autouse=True) - def clear_parameters_cache(): - yield - parameters.clear_caches() # This will clear all providers cache - - @pytest.fixture - def mock_parameter_response(monkeypatch): - def mockreturn(name): - return "mock_value" - - monkeypatch.setattr(app.ssm_provider, "get", mockreturn) +=== "test_clear_cache_method.py" + ```python + --8<-- "examples/parameters/tests/test_clear_cache_method.py" + ``` - # Pass our fixture as an argument to all tests where we want to mock the get_parameter response - def test_handler(mock_parameter_response): - return_val = app.handler({}, {}) - assert return_val.get('message') == 'mock_value' - ``` +=== "test_clear_cache_global.py" + ```python + --8<-- "examples/parameters/tests/test_clear_cache_global.py" + ``` === "app.py" - ```python - from aws_lambda_powertools.utilities import parameters - from botocore.config import Config - - ssm_provider = parameters.SSMProvider(config=Config(region_name="us-west-1")) - - - def handler(event, context): - value = ssm_provider.get("/my/parameter") - return {"message": value} - ``` + ```python + --8<-- "examples/parameters/tests/src/app.py" + ``` diff --git a/examples/parameters/tests/src/app.py b/examples/parameters/tests/src/app.py new file mode 100644 index 00000000000..b505749e31b --- /dev/null +++ b/examples/parameters/tests/src/app.py @@ -0,0 +1,10 @@ +from botocore.config import Config + +from aws_lambda_powertools.utilities import parameters + +ssm_provider = parameters.SSMProvider(config=Config(region_name="us-west-1")) + + +def handler(event, context): + value = ssm_provider.get("/my/parameter") + return {"message": value} diff --git a/examples/parameters/tests/test_clear_cache_global.py b/examples/parameters/tests/test_clear_cache_global.py new file mode 100644 index 00000000000..5b07f40caed --- /dev/null +++ b/examples/parameters/tests/test_clear_cache_global.py @@ -0,0 +1,24 @@ +import pytest +import src.app as app + +from aws_lambda_powertools.utilities import parameters + + +@pytest.fixture(scope="function", autouse=True) +def clear_parameters_cache(): + yield + parameters.clear_caches() # This will clear all providers cache + + +@pytest.fixture +def mock_parameter_response(monkeypatch): + def mockreturn(name): + return "mock_value" + + monkeypatch.setattr(app.ssm_provider, "get", mockreturn) + + +# Pass our fixture as an argument to all tests where we want to mock the get_parameter response +def test_handler(mock_parameter_response): + return_val = app.handler({}, {}) + assert return_val.get("message") == "mock_value" diff --git a/examples/parameters/tests/test_clear_cache_method.py b/examples/parameters/tests/test_clear_cache_method.py new file mode 100644 index 00000000000..da1eb181c68 --- /dev/null +++ b/examples/parameters/tests/test_clear_cache_method.py @@ -0,0 +1,22 @@ +import pytest +import src.app as app + + +@pytest.fixture(scope="function", autouse=True) +def clear_parameters_cache(): + yield + app.ssm_provider.clear_cache() # This will clear SSMProvider cache + + +@pytest.fixture +def mock_parameter_response(monkeypatch): + def mockreturn(name): + return "mock_value" + + monkeypatch.setattr(app.ssm_provider, "get", mockreturn) + + +# Pass our fixture as an argument to all tests where we want to mock the get_parameter response +def test_handler(mock_parameter_response): + return_val = app.handler({}, {}) + assert return_val.get("message") == "mock_value" diff --git a/examples/parameters/tests/test_with_fixture.py b/examples/parameters/tests/test_with_fixture.py new file mode 100644 index 00000000000..0d29ad28030 --- /dev/null +++ b/examples/parameters/tests/test_with_fixture.py @@ -0,0 +1,16 @@ +import pytest +import src.single_mock as single_mock + + +@pytest.fixture +def mock_parameter_response(monkeypatch): + def mockreturn(name): + return "mock_value" + + monkeypatch.setattr(single_mock.parameters, "get_parameter", mockreturn) + + +# Pass our fixture as an argument to all tests where we want to mock the get_parameter response +def test_handler(mock_parameter_response): + return_val = single_mock.handler({}, {}) + assert return_val.get("message") == "mock_value" diff --git a/examples/parameters/tests/test_with_monkeypatch.py b/examples/parameters/tests/test_with_monkeypatch.py new file mode 100644 index 00000000000..71ac4b406ed --- /dev/null +++ b/examples/parameters/tests/test_with_monkeypatch.py @@ -0,0 +1,13 @@ +from unittest.mock import patch + +import src.single_mock as single_mock + + +# Replaces "aws_lambda_powertools.utilities.parameters.get_parameter" with a Mock object +@patch("aws_lambda_powertools.utilities.parameters.get_parameter") +def test_handler(get_parameter_mock): + get_parameter_mock.return_value = "mock_value" + + return_val = single_mock.handler({}, {}) + get_parameter_mock.assert_called_with("my-parameter-name") + assert return_val.get("message") == "mock_value" From 2e7c5707b335c6764ed31388beb128ac0506026d Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sun, 2 Oct 2022 19:41:50 +0100 Subject: [PATCH 11/19] chore(parameters): adding appconfig builtin provider --- docs/utilities/parameters.md | 15 ++++-------- .../src/builtin_provider_appconfig.py | 23 +++++++++++++++++++ 2 files changed, 27 insertions(+), 11 deletions(-) create mode 100644 examples/parameters/src/builtin_provider_appconfig.py diff --git a/docs/utilities/parameters.md b/docs/utilities/parameters.md index 8fa9251bce2..7b8eb83e79c 100644 --- a/docs/utilities/parameters.md +++ b/docs/utilities/parameters.md @@ -256,17 +256,10 @@ DynamoDB provider can be customized at initialization to match your table struct #### AppConfigProvider -```python hl_lines="5 9" title="Using AppConfigProvider" -from aws_lambda_powertools.utilities import parameters -from botocore.config import Config - -config = Config(region_name="us-west-1") -appconf_provider = parameters.AppConfigProvider(environment="my_env", application="my_app", config=config) - -def handler(event, context): - # Retrieve a single secret - value: bytes = appconf_provider.get("my_conf") -``` +=== "builtin_provider_appconfig.py" + ```python hl_lines="3 5 9 10 15" + --8<-- "examples/parameters/src/builtin_provider_appconfig.py" + ``` ### Create your own provider diff --git a/examples/parameters/src/builtin_provider_appconfig.py b/examples/parameters/src/builtin_provider_appconfig.py new file mode 100644 index 00000000000..cf734e29af8 --- /dev/null +++ b/examples/parameters/src/builtin_provider_appconfig.py @@ -0,0 +1,23 @@ +from typing import Any + +import requests +from botocore.config import Config + +from aws_lambda_powertools.utilities import parameters +from aws_lambda_powertools.utilities.typing import LambdaContext + +config = Config(region_name="sa-east-1") +appconf_provider = parameters.AppConfigProvider(environment="dev", application="comments", config=config) + + +def lambda_handler(event: dict, context: LambdaContext): + try: + # Retrieve a single parameter + endpoint_comments: Any = appconf_provider.get("config") + + # the value of this parameter is https://jsonplaceholder.typicode.com/comments/ + comments: requests.Response = requests.get(endpoint_comments) + + return {"comments": comments.json()[:10], "statusCode": 200} + except parameters.exceptions.GetParameterError as error: + return {"comments": None, "message": str(error), "statusCode": 400} From ae995d693700d5a911bdd4c339dbd0a331228eb4 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sun, 2 Oct 2022 20:00:48 +0100 Subject: [PATCH 12/19] chore(parameters): adding final examples --- docs/utilities/parameters.md | 75 +++++-------------- .../src/handling_error_transform.py | 20 +++++ .../src/working_with_auto_transform.py | 10 +++ .../working_with_sdk_additional_arguments.py | 11 +++ .../src/working_with_transform_high_level.py | 19 +++++ .../src/working_with_transform_provider.py | 23 ++++++ 6 files changed, 101 insertions(+), 57 deletions(-) create mode 100644 examples/parameters/src/handling_error_transform.py create mode 100644 examples/parameters/src/working_with_auto_transform.py create mode 100644 examples/parameters/src/working_with_sdk_additional_arguments.py create mode 100644 examples/parameters/src/working_with_transform_high_level.py create mode 100644 examples/parameters/src/working_with_transform_provider.py diff --git a/docs/utilities/parameters.md b/docs/utilities/parameters.md index 7b8eb83e79c..9759474ee72 100644 --- a/docs/utilities/parameters.md +++ b/docs/utilities/parameters.md @@ -296,28 +296,14 @@ For parameters stored in JSON or Base64 format, you can use the `transform` argu ???+ info The `transform` argument is available across all providers, including the high level functions. -=== "High level functions" - - ```python hl_lines="4" - from aws_lambda_powertools.utilities import parameters - - def handler(event, context): - value_from_json = parameters.get_parameter("/my/json/parameter", transform="json") +=== "working_with_transform_high_level.py" + ```python hl_lines="3 5 9 10 15" + --8<-- "examples/parameters/src/working_with_transform_high_level.py" ``` -=== "Providers" - - ```python hl_lines="7 10" - from aws_lambda_powertools.utilities import parameters - - ssm_provider = parameters.SSMProvider() - - def handler(event, context): - # Transform a JSON string - value_from_json = ssm_provider.get("/my/json/parameter", transform="json") - - # Transform a Base64 encoded string - value_from_binary = ssm_provider.get("/my/binary/parameter", transform="binary") +=== "working_with_transform_provider.py" + ```python hl_lines="3 5 9 10 15" + --8<-- "examples/parameters/src/working_with_transform_provider.py" ``` #### Partial transform failures with `get_multiple()` @@ -328,26 +314,10 @@ You can override this by setting the `raise_on_transform_error` argument to `Tru For example, if you have three parameters, */param/a*, */param/b* and */param/c*, but */param/c* is malformed: -```python hl_lines="9 16" title="Raising TransformParameterError at first malformed parameter" -from aws_lambda_powertools.utilities import parameters - -ssm_provider = parameters.SSMProvider() - -def handler(event, context): - # This will display: - # /param/a: [some value] - # /param/b: [some value] - # /param/c: None - values = ssm_provider.get_multiple("/param", transform="json") - for k, v in values.items(): - print(f"{k}: {v}") - - try: - # This will raise a TransformParameterError exception - values = ssm_provider.get_multiple("/param", transform="json", raise_on_transform_error=True) - except parameters.exceptions.TransformParameterError: - ... -``` +=== "handling_error_transform.py" + ```python hl_lines="3 5 9 10 15" + --8<-- "examples/parameters/src/handling_error_transform.py" + ``` #### Auto-transform values on suffix @@ -358,14 +328,10 @@ You can do this with a single request by using `transform="auto"`. This will ins ???+ info `transform="auto"` feature is available across all providers, including the high level functions. -```python hl_lines="6" title="Deserializing parameter values based on their suffix" -from aws_lambda_powertools.utilities import parameters - -ssm_provider = parameters.SSMProvider() - -def handler(event, context): - values = ssm_provider.get_multiple("/param", transform="auto") -``` +=== "working_with_auto_transform.py" + ```python hl_lines="3 5 9 10 15" + --8<-- "examples/parameters/src/working_with_auto_transform.py" + ``` For example, if you have two parameters with the following suffixes `.json` and `.binary`: @@ -387,15 +353,10 @@ The return of `ssm_provider.get_multiple("/param", transform="auto")` call will You can use arbitrary keyword arguments to pass it directly to the underlying SDK method. -```python hl_lines="8" title="" -from aws_lambda_powertools.utilities import parameters - -secrets_provider = parameters.SecretsProvider() - -def handler(event, context): - # The 'VersionId' argument will be passed to the underlying get_secret_value() call. - value = secrets_provider.get("my-secret", VersionId="e62ec170-6b01-48c7-94f3-d7497851a8d2") -``` +=== "working_with_sdk_additional_arguments.py" + ```python hl_lines="3 5 9 10 15" + --8<-- "examples/parameters/src/working_with_sdk_additional_arguments.py" + ``` Here is the mapping between this utility's functions and methods and the underlying SDK: diff --git a/examples/parameters/src/handling_error_transform.py b/examples/parameters/src/handling_error_transform.py new file mode 100644 index 00000000000..8acda24b09f --- /dev/null +++ b/examples/parameters/src/handling_error_transform.py @@ -0,0 +1,20 @@ +from aws_lambda_powertools.utilities import parameters +from aws_lambda_powertools.utilities.typing import LambdaContext + +ssm_provider = parameters.SSMProvider() + + +def lambda_handler(event: dict, context: LambdaContext): + # This will display: + # /param/a: [some value] + # /param/b: [some value] + # /param/c: None + values = ssm_provider.get_multiple("/param", transform="json") + for key, value in values.items(): + print(f"{key}: {value}") + + try: + # This will raise a TransformParameterError exception + values = ssm_provider.get_multiple("/param", transform="json", raise_on_transform_error=True) + except parameters.exceptions.TransformParameterError: + ... diff --git a/examples/parameters/src/working_with_auto_transform.py b/examples/parameters/src/working_with_auto_transform.py new file mode 100644 index 00000000000..fc29d376606 --- /dev/null +++ b/examples/parameters/src/working_with_auto_transform.py @@ -0,0 +1,10 @@ +from aws_lambda_powertools.utilities import parameters +from aws_lambda_powertools.utilities.typing import LambdaContext + +ssm_provider = parameters.SSMProvider() + + +def lambda_handler(event: dict, context: LambdaContext): + values = ssm_provider.get_multiple("/param", transform="auto") + + return values diff --git a/examples/parameters/src/working_with_sdk_additional_arguments.py b/examples/parameters/src/working_with_sdk_additional_arguments.py new file mode 100644 index 00000000000..4f99714d817 --- /dev/null +++ b/examples/parameters/src/working_with_sdk_additional_arguments.py @@ -0,0 +1,11 @@ +from aws_lambda_powertools.utilities import parameters +from aws_lambda_powertools.utilities.typing import LambdaContext + +secrets_provider = parameters.SecretsProvider() + + +def lambda_handler(event: dict, context: LambdaContext): + # The 'VersionId' argument will be passed to the underlying get_secret_value() call. + value = secrets_provider.get("my-secret", VersionId="e62ec170-6b01-48c7-94f3-d7497851a8d2") + + return value diff --git a/examples/parameters/src/working_with_transform_high_level.py b/examples/parameters/src/working_with_transform_high_level.py new file mode 100644 index 00000000000..ee00862bf72 --- /dev/null +++ b/examples/parameters/src/working_with_transform_high_level.py @@ -0,0 +1,19 @@ +from typing import Any + +import requests + +from aws_lambda_powertools.utilities import parameters +from aws_lambda_powertools.utilities.typing import LambdaContext + + +def lambda_handler(event: dict, context: LambdaContext) -> dict: + try: + # Retrieve a single parameter + endpoint_comments: Any = parameters.get_parameter("/lambda-powertools/endpoint_comments", transform="json") + + # the value of this parameter is https://jsonplaceholder.typicode.com/comments/ + comments: requests.Response = requests.get(endpoint_comments) + + return {"comments": comments.json()[:10], "statusCode": 200} + except parameters.exceptions.GetParameterError as error: + return {"comments": None, "message": str(error), "statusCode": 400} diff --git a/examples/parameters/src/working_with_transform_provider.py b/examples/parameters/src/working_with_transform_provider.py new file mode 100644 index 00000000000..948ae1710b3 --- /dev/null +++ b/examples/parameters/src/working_with_transform_provider.py @@ -0,0 +1,23 @@ +from typing import Any + +import requests +from botocore.config import Config + +from aws_lambda_powertools.utilities import parameters +from aws_lambda_powertools.utilities.typing import LambdaContext + +config = Config(region_name="sa-east-1") +appconf_provider = parameters.AppConfigProvider(environment="dev", application="comments", config=config) + + +def lambda_handler(event: dict, context: LambdaContext): + try: + # Retrieve a single parameter + endpoint_comments: Any = appconf_provider.get("config", transform="json") + + # the value of this parameter is https://jsonplaceholder.typicode.com/comments/ + comments: requests.Response = requests.get(endpoint_comments) + + return {"comments": comments.json()[:10], "statusCode": 200} + except parameters.exceptions.GetParameterError as error: + return {"comments": None, "message": str(error), "statusCode": 400} From 57b788e754a6ac274e8b37af78a4e8777a0e9aa3 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sun, 2 Oct 2022 20:04:01 +0100 Subject: [PATCH 13/19] chore(parameters): adding final examples --- examples/parameters/src/handling_error_transform.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/parameters/src/handling_error_transform.py b/examples/parameters/src/handling_error_transform.py index 8acda24b09f..e81a9e81809 100644 --- a/examples/parameters/src/handling_error_transform.py +++ b/examples/parameters/src/handling_error_transform.py @@ -1,3 +1,5 @@ +from typing import Any + from aws_lambda_powertools.utilities import parameters from aws_lambda_powertools.utilities.typing import LambdaContext @@ -9,7 +11,7 @@ def lambda_handler(event: dict, context: LambdaContext): # /param/a: [some value] # /param/b: [some value] # /param/c: None - values = ssm_provider.get_multiple("/param", transform="json") + values: Any = ssm_provider.get_multiple("/param", transform="json") for key, value in values.items(): print(f"{key}: {value}") From 1da673db2b750267ea22d65e2fdbbb09d4d6aab6 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sun, 2 Oct 2022 21:06:46 +0100 Subject: [PATCH 14/19] chore(parameters): highlights - small changes --- docs/utilities/parameters.md | 84 ++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/docs/utilities/parameters.md b/docs/utilities/parameters.md index 9759474ee72..5d8881e9e9f 100644 --- a/docs/utilities/parameters.md +++ b/docs/utilities/parameters.md @@ -44,12 +44,12 @@ You can retrieve a single parameter using `get_parameter` high-level function. For multiple parameters, you can use `get_parameters` and pass a path to retrieve them recursively. === "getting_started_single_ssm_parameter.py" - ```python hl_lines="3 10 16" + ```python hl_lines="5 12" --8<-- "examples/parameters/src/getting_started_single_ssm_parameter.py" ``` === "getting_started_recursive_ssm_parameter.py" - ```python hl_lines="3 10 13 22" + ```python hl_lines="5 12 15" --8<-- "examples/parameters/src/getting_started_recursive_ssm_parameter.py" ``` @@ -58,7 +58,7 @@ For multiple parameters, you can use `get_parameters` and pass a path to retriev You can fetch secrets stored in Secrets Manager using `get_secrets`. === "getting_started_secret.py" - ```python hl_lines="3 13 20" + ```python hl_lines="5 15" --8<-- "examples/parameters/src/getting_started_secret.py" ``` @@ -69,7 +69,7 @@ You can fetch application configurations in AWS AppConfig using `get_app_config` The following will retrieve the latest version and store it in the cache. === "getting_started_appconfig.py" - ```python hl_lines="3 10 16" + ```python hl_lines="5 12" --8<-- "examples/parameters/src/getting_started_appconfig.py" ``` @@ -85,22 +85,22 @@ By default, we cache parameters retrieved in-memory for 5 seconds. You can adjust how long we should keep values in cache by using the param `max_age`, when using `get_parameter()`, `get_parameters()` and `get_secret()` methods across all providers. === "single_ssm_parameter_with_cache.py" - ```python hl_lines="3 10 16" + ```python hl_lines="5 12" --8<-- "examples/parameters/src/single_ssm_parameter_with_cache.py" ``` === "recursive_ssm_parameter_with_cache.py" - ```python hl_lines="3 10 13 22" + ```python hl_lines="5 12" --8<-- "examples/parameters/src/recursive_ssm_parameter_with_cache.py" ``` === "secret_with_cache.py" - ```python hl_lines="3 13 20" + ```python hl_lines="5 15" --8<-- "examples/parameters/src/secret_with_cache.py" ``` === "appconfig_with_cache.py" - ```python hl_lines="3 10 11 18" + ```python hl_lines="5 12-14" --8<-- "examples/parameters/src/appconfig_with_cache.py" ``` @@ -109,22 +109,22 @@ You can adjust how long we should keep values in cache by using the param `max_a If you'd like to always ensure you fetch the latest parameter from the store regardless if already available in cache, use `force_fetch` param. === "single_ssm_parameter_force_fetch.py" - ```python hl_lines="3 10 16" + ```python hl_lines="5 12" --8<-- "examples/parameters/src/single_ssm_parameter_force_fetch.py" ``` === "recursive_ssm_parameter_force_fetch.py" - ```python hl_lines="3 10 13 22" + ```python hl_lines="5 12" --8<-- "examples/parameters/src/recursive_ssm_parameter_force_fetch.py" ``` === "secret_force_fetch.py" - ```python hl_lines="3 13 20" + ```python hl_lines="5 15" --8<-- "examples/parameters/src/secret_force_fetch.py" ``` === "appconfig_force_fetch.py" - ```python hl_lines="3 10 11 18" + ```python hl_lines="5 12-14" --8<-- "examples/parameters/src/appconfig_force_fetch.py" ``` @@ -138,12 +138,12 @@ For greater flexibility such as configuring the underlying SDK client used by bu #### SSMProvider === "builtin_provider_ssm_single_parameter.py" - ```python hl_lines="3 5 9 10 15" + ```python hl_lines="6 11 12" --8<-- "examples/parameters/src/builtin_provider_ssm_single_parameter.py" ``` === "builtin_provider_ssm_recursive_parameter.py" - ```python hl_lines="3 5 9 10 15" + ```python hl_lines="6 19-25" --8<-- "examples/parameters/src/builtin_provider_ssm_recursive_parameter.py" ``` @@ -157,19 +157,19 @@ The AWS Systems Manager Parameter Store provider supports two additional argumen You can create `SecureString` parameters, which are parameters that have a plaintext parameter name and an encrypted parameter value. If you don't use the `decrypt` argument, you will get an encrypted value. Read [here](https://docs.aws.amazon.com/kms/latest/developerguide/services-parameter-store.html) about best practices using KMS to secure your parameters. === "builtin_provider_ssm_with_decrypt.py" - ```python hl_lines="3 5 9 10 15" + ```python hl_lines="6 10 16" --8<-- "examples/parameters/src/builtin_provider_ssm_with_decrypt.py" ``` === "builtin_provider_ssm_with_no_recursive.py" - ```python hl_lines="3 5 9 10 15" + ```python hl_lines="5 8 21" --8<-- "examples/parameters/src/builtin_provider_ssm_with_no_recursive.py" ``` #### SecretsProvider === "builtin_provider_secret.py" - ```python hl_lines="3 5 9 10 15" + ```python hl_lines="4 6 9" --8<-- "examples/parameters/src/builtin_provider_secret.py" ``` @@ -192,19 +192,19 @@ For single parameters, you must use `id` as the [partition key](https://docs.aws With this table, `dynamodb_provider.get("my-param")` will return `my-value`. === "builtin_provider_dynamodb_single_parameter.py" - ```python hl_lines="3 5 9 10 15" + ```python hl_lines="5 8 15" --8<-- "examples/parameters/src/builtin_provider_dynamodb_single_parameter.py" ``` === "sam_dynamodb_table_single.yaml" - ```yaml hl_lines="3 5 9 10 15" + ```yaml hl_lines="12-14" --8<-- "examples/parameters/sam/sam_dynamodb_table_single.yaml" ``` You can initialize the DynamoDB provider pointing to [DynamoDB Local](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.html) using `endpoint_url` parameter: === "builtin_provider_dynamodb_custom_endpoint.py" - ```python hl_lines="3 5 9 10 15" + ```python hl_lines="5 8 15" --8<-- "examples/parameters/src/builtin_provider_dynamodb_custom_endpoint.py" ``` @@ -224,12 +224,12 @@ You can retrieve multiple parameters sharing the same `id` by having a sort key With this table, `dynamodb_provider.get_multiple("config")` will return a dictionary response in the shape of `sk:value`. === "builtin_provider_dynamodb_recursive_parameter.py" - ```python hl_lines="3 5 9 10 15" + ```python hl_lines="5 8 15" --8<-- "examples/parameters/src/builtin_provider_dynamodb_recursive_parameter.py" ``` === "sam_dynamodb_table_recursive.yaml" - ```yaml hl_lines="3 5 9 10 15" + ```yaml hl_lines="15-18" --8<-- "examples/parameters/sam/sam_dynamodb_table_recursive.yaml" ``` @@ -250,14 +250,14 @@ DynamoDB provider can be customized at initialization to match your table struct ``` === "sam_dynamodb_custom_fields.yaml" - ```yaml hl_lines="3 5 9 10 15" + ```yaml hl_lines="5 8-10 17" --8<-- "examples/parameters/sam/sam_dynamodb_custom_fields.yaml" ``` #### AppConfigProvider === "builtin_provider_appconfig.py" - ```python hl_lines="3 5 9 10 15" + ```python hl_lines="6 9 10 16" --8<-- "examples/parameters/src/builtin_provider_appconfig.py" ``` @@ -270,22 +270,22 @@ All transformation and caching logic is handled by the `get()` and `get_multiple Here are two examples of implementing a custom parameter store. One using an external service like [Hashicorp Vault](https://www.vaultproject.io/), a widely popular key-value and secret storage and the other one using [Amazon S3](https://aws.amazon.com/s3/?nc1=h_ls), a popular object storage. === "working_with_own_provider_vault.py" - ```python hl_lines="3 5 9 10 15" + ```python hl_lines="5 13 20 24" --8<-- "examples/parameters/src/working_with_own_provider_vault.py" ``` === "custom_provider_vault.py" - ```python hl_lines="3 5 9 10 15" + ```python hl_lines="6 9 17 24" --8<-- "examples/parameters/src/custom_provider_vault.py" ``` === "working_with_own_provider_s3.py" - ```python hl_lines="3 5 9 10 15" + ```python hl_lines="4 11 18 21" --8<-- "examples/parameters/src/working_with_own_provider_s3.py" ``` === "custom_provider_s3.py" - ```python hl_lines="3 5 9 10 15" + ```python hl_lines="6 9 19 29" --8<-- "examples/parameters/src/custom_provider_s3.py" ``` @@ -297,12 +297,12 @@ For parameters stored in JSON or Base64 format, you can use the `transform` argu The `transform` argument is available across all providers, including the high level functions. === "working_with_transform_high_level.py" - ```python hl_lines="3 5 9 10 15" + ```python hl_lines="5 12" --8<-- "examples/parameters/src/working_with_transform_high_level.py" ``` === "working_with_transform_provider.py" - ```python hl_lines="3 5 9 10 15" + ```python hl_lines="6 9 16" --8<-- "examples/parameters/src/working_with_transform_provider.py" ``` @@ -315,7 +315,7 @@ You can override this by setting the `raise_on_transform_error` argument to `Tru For example, if you have three parameters, */param/a*, */param/b* and */param/c*, but */param/c* is malformed: === "handling_error_transform.py" - ```python hl_lines="3 5 9 10 15" + ```python hl_lines="3 14 20" --8<-- "examples/parameters/src/handling_error_transform.py" ``` @@ -329,7 +329,7 @@ You can do this with a single request by using `transform="auto"`. This will ins `transform="auto"` feature is available across all providers, including the high level functions. === "working_with_auto_transform.py" - ```python hl_lines="3 5 9 10 15" + ```python hl_lines="1 4 8" --8<-- "examples/parameters/src/working_with_auto_transform.py" ``` @@ -354,7 +354,7 @@ The return of `ssm_provider.get_multiple("/param", transform="auto")` call will You can use arbitrary keyword arguments to pass it directly to the underlying SDK method. === "working_with_sdk_additional_arguments.py" - ```python hl_lines="3 5 9 10 15" + ```python hl_lines="1 4 9" --8<-- "examples/parameters/src/working_with_sdk_additional_arguments.py" ``` @@ -386,7 +386,7 @@ You can use `boto3_client` parameter via any of the available [Provider Classes] Bringing them together in a single code snippet would look like this: === "custom_boto3_all_providers.py" - ```python + ```python hl_lines="4 6" --8<-- "examples/parameters/src/custom_boto3_all_providers.py" ``` @@ -403,17 +403,17 @@ The **`config`** , **`boto3_session`**, and **`boto3_client`** parameters enabl When using VPC private endpoints, you can pass a custom client altogether. It's also useful for testing when injecting fake instances. === "custom_boto_session.py" - ```python + ```python hl_lines="5 6" --8<-- "examples/parameters/src/custom_boto_session.py" ``` === "custom_boto_config.py" - ```python + ```python hl_lines="5 6" --8<-- "examples/parameters/src/custom_boto_config.py" ``` === "custom_boto_client.py" - ```python + ```python hl_lines="5 6" --8<-- "examples/parameters/src/custom_boto_client.py" ``` @@ -424,7 +424,7 @@ The **`config`** , **`boto3_session`**, and **`boto3_client`** parameters enabl For unit testing your applications, you can mock the calls to the parameters utility to avoid calling AWS APIs. This can be achieved in a number of ways - in this example, we use the [pytest monkeypatch fixture](https://docs.pytest.org/en/latest/how-to/monkeypatch.html) to patch the `parameters.get_parameter` method: === "test_single_mock.py" - ```python + ```python hl_lines="4 8" --8<-- "examples/parameters/tests/test_single_mock.py" ``` @@ -436,7 +436,7 @@ For unit testing your applications, you can mock the calls to the parameters uti If we need to use this pattern across multiple tests, we can avoid repetition by refactoring to use our own pytest fixture: === "test_with_fixture.py" - ```python + ```python hl_lines="5 10" --8<-- "examples/parameters/tests/test_with_fixture.py" ``` @@ -446,7 +446,7 @@ can use [unittest.mock](https://docs.python.org/3/library/unittest.mock.html) fr object named `get_parameter_mock`. === "test_with_monkeypatch.py" - ```python + ```python hl_lines="7 12" --8<-- "examples/parameters/tests/test_with_monkeypatch.py" ``` @@ -457,12 +457,12 @@ Parameters utility caches all parameter values for performance and cost reasons. Within your tests, you can use `clear_cache` method available in [every provider](#built-in-provider-class). When using multiple providers or higher level functions like `get_parameter`, use `clear_caches` standalone function to clear cache globally. === "test_clear_cache_method.py" - ```python + ```python hl_lines="8" --8<-- "examples/parameters/tests/test_clear_cache_method.py" ``` === "test_clear_cache_global.py" - ```python + ```python hl_lines="10" --8<-- "examples/parameters/tests/test_clear_cache_global.py" ``` From 68358cdd9665be9624c9b664673723cce1e2cb09 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 18 Oct 2022 15:27:42 +0200 Subject: [PATCH 15/19] fix(docs): broken link in boto3_session --- docs/utilities/parameters.md | 60 ++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/docs/utilities/parameters.md b/docs/utilities/parameters.md index 5d8881e9e9f..3ad8e75fde9 100644 --- a/docs/utilities/parameters.md +++ b/docs/utilities/parameters.md @@ -27,15 +27,15 @@ This utility requires additional permissions to work as expected. ???+ note Different parameter providers require different permissions. -| Provider | Function/Method | IAM Permission | -| ------------------- | -----------------------------------------------------------------| -----------------------------------------------------------------------------| -| SSM Parameter Store | `get_parameter`, `SSMProvider.get` | `ssm:GetParameter` | -| SSM Parameter Store | `get_parameters`, `SSMProvider.get_multiple` | `ssm:GetParametersByPath` | -| SSM Parameter Store | If using `decrypt=True` | You must add an additional permission `kms:Decrypt` | -| Secrets Manager | `get_secret`, `SecretsManager.get` | `secretsmanager:GetSecretValue` | -| DynamoDB | `DynamoDBProvider.get` | `dynamodb:GetItem` | -| DynamoDB | `DynamoDBProvider.get_multiple` | `dynamodb:Query` | -| App Config | `get_app_config`, `AppConfigProvider.get_app_config` | `appconfig:GetLatestConfiguration` and `appconfig:StartConfigurationSession` | +| Provider | Function/Method | IAM Permission | +| ------------------- | ---------------------------------------------------- | ---------------------------------------------------------------------------- | +| SSM Parameter Store | `get_parameter`, `SSMProvider.get` | `ssm:GetParameter` | +| SSM Parameter Store | `get_parameters`, `SSMProvider.get_multiple` | `ssm:GetParametersByPath` | +| SSM Parameter Store | If using `decrypt=True` | You must add an additional permission `kms:Decrypt` | +| Secrets Manager | `get_secret`, `SecretsManager.get` | `secretsmanager:GetSecretValue` | +| DynamoDB | `DynamoDBProvider.get` | `dynamodb:GetItem` | +| DynamoDB | `DynamoDBProvider.get_multiple` | `dynamodb:Query` | +| App Config | `get_app_config`, `AppConfigProvider.get_app_config` | `appconfig:GetLatestConfiguration` and `appconfig:StartConfigurationSession` | ### Fetching parameters @@ -133,7 +133,7 @@ If you'd like to always ensure you fetch the latest parameter from the store reg For greater flexibility such as configuring the underlying SDK client used by built-in providers, you can use their respective Provider Classes directly. ???+ tip - This is useful when you need to customize parameters for the SDK client, such as region, credentials, retries and others. For more information, read [botocore.config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html) and [boto3.session]( .com/v1/documentation/api/latest/reference/core/session.html#module-boto3.session). + This is useful when you need to customize parameters for the SDK client, such as region, credentials, retries and others. For more information, read [botocore.config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html) and [boto3.session](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html#module-boto3.session). #### SSMProvider @@ -216,10 +216,10 @@ You can retrieve multiple parameters sharing the same `id` by having a sort key DynamoDB table with `id` primary key, `sk` as sort key` and `value` as attribute - | id | sk | value | - | -------| ----------------- | ---------------------------------------------- | + | id | sk | value | + | ------ | ----------------- | ------------------------------------------------ | | config | endpoint_comments | | - | config | limit | 10 | + | config | limit | 10 | With this table, `dynamodb_provider.get_multiple("config")` will return a dictionary response in the shape of `sk:value`. @@ -360,28 +360,28 @@ You can use arbitrary keyword arguments to pass it directly to the underlying SD Here is the mapping between this utility's functions and methods and the underlying SDK: -| Provider | Function/Method | Client name | Function name | -| ------------------- | ------------------------------- | ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | -| SSM Parameter Store | `get_parameter` | `ssm` | [get_parameter](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ssm.html#SSM.Client.get_parameter) | -| SSM Parameter Store | `get_parameters` | `ssm` | [get_parameters_by_path](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ssm.html#SSM.Client.get_parameters_by_path) | -| SSM Parameter Store | `SSMProvider.get` | `ssm` | [get_parameter](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ssm.html#SSM.Client.get_parameter) | -| SSM Parameter Store | `SSMProvider.get_multiple` | `ssm` | [get_parameters_by_path](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ssm.html#SSM.Client.get_parameters_by_path) | -| Secrets Manager | `get_secret` | `secretsmanager` | [get_secret_value](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/secretsmanager.html#SecretsManager.Client.get_secret_value) | -| Secrets Manager | `SecretsManager.get` | `secretsmanager` | [get_secret_value](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/secretsmanager.html#SecretsManager.Client.get_secret_value) | -| DynamoDB | `DynamoDBProvider.get` | `dynamodb` | ([Table resource](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#table)) | [get_item](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#DynamoDB.Table.get_item) | -| DynamoDB | `DynamoDBProvider.get_multiple` | `dynamodb` | ([Table resource](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#table)) | [query](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#DynamoDB.Table.query) | -| App Config | `get_app_config` | `appconfigdata` | [start_configuration_session](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/appconfigdata.html#AppConfigData.Client.start_configuration_session) and [get_latest_configuration](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/appconfigdata.html#AppConfigData.Client.get_latest_configuration) | +| Provider | Function/Method | Client name | Function name | +| ------------------- | ------------------------------- | ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| SSM Parameter Store | `get_parameter` | `ssm` | [get_parameter](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ssm.html#SSM.Client.get_parameter) | +| SSM Parameter Store | `get_parameters` | `ssm` | [get_parameters_by_path](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ssm.html#SSM.Client.get_parameters_by_path) | +| SSM Parameter Store | `SSMProvider.get` | `ssm` | [get_parameter](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ssm.html#SSM.Client.get_parameter) | +| SSM Parameter Store | `SSMProvider.get_multiple` | `ssm` | [get_parameters_by_path](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ssm.html#SSM.Client.get_parameters_by_path) | +| Secrets Manager | `get_secret` | `secretsmanager` | [get_secret_value](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/secretsmanager.html#SecretsManager.Client.get_secret_value) | +| Secrets Manager | `SecretsManager.get` | `secretsmanager` | [get_secret_value](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/secretsmanager.html#SecretsManager.Client.get_secret_value) | +| DynamoDB | `DynamoDBProvider.get` | `dynamodb` | ([Table resource](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#table)) | [get_item](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#DynamoDB.Table.get_item) | +| DynamoDB | `DynamoDBProvider.get_multiple` | `dynamodb` | ([Table resource](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#table)) | [query](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#DynamoDB.Table.query) | +| App Config | `get_app_config` | `appconfigdata` | [start_configuration_session](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/appconfigdata.html#AppConfigData.Client.start_configuration_session) and [get_latest_configuration](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/appconfigdata.html#AppConfigData.Client.get_latest_configuration) | ### Bring your own boto client You can use `boto3_client` parameter via any of the available [Provider Classes](#built-in-provider-class). Some providers expect a low level boto3 client while others expect a high level boto3 client, here is the mapping for each of them: -| Provider | Type | Boto client construction | -| --------------------------------------- | ---------- | -------------------------------- | -| [SSMProvider](#ssmprovider) | low level | `boto3.client("ssm")` | -| [SecretsProvider](#secretsprovider) | low level | `boto3.client("secrets")` | -| [AppConfigProvider](#appconfigprovider) | low level | `boto3.client("appconfigdata")` | -| [DynamoDBProvider](#dynamodbprovider) | high level | `boto3.resource("dynamodb")` | +| Provider | Type | Boto client construction | +| --------------------------------------- | ---------- | ------------------------------- | +| [SSMProvider](#ssmprovider) | low level | `boto3.client("ssm")` | +| [SecretsProvider](#secretsprovider) | low level | `boto3.client("secrets")` | +| [AppConfigProvider](#appconfigprovider) | low level | `boto3.client("appconfigdata")` | +| [DynamoDBProvider](#dynamodbprovider) | high level | `boto3.resource("dynamodb")` | Bringing them together in a single code snippet would look like this: From ce369c6ce4764ba70754c525d7a7997860056958 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 18 Oct 2022 15:31:49 +0200 Subject: [PATCH 16/19] fix(docs): improve constrast ratio --- docs/utilities/parameters.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/utilities/parameters.md b/docs/utilities/parameters.md index 3ad8e75fde9..ed54f2c9dfb 100644 --- a/docs/utilities/parameters.md +++ b/docs/utilities/parameters.md @@ -27,15 +27,15 @@ This utility requires additional permissions to work as expected. ???+ note Different parameter providers require different permissions. -| Provider | Function/Method | IAM Permission | -| ------------------- | ---------------------------------------------------- | ---------------------------------------------------------------------------- | -| SSM Parameter Store | `get_parameter`, `SSMProvider.get` | `ssm:GetParameter` | -| SSM Parameter Store | `get_parameters`, `SSMProvider.get_multiple` | `ssm:GetParametersByPath` | -| SSM Parameter Store | If using `decrypt=True` | You must add an additional permission `kms:Decrypt` | -| Secrets Manager | `get_secret`, `SecretsManager.get` | `secretsmanager:GetSecretValue` | -| DynamoDB | `DynamoDBProvider.get` | `dynamodb:GetItem` | -| DynamoDB | `DynamoDBProvider.get_multiple` | `dynamodb:Query` | -| App Config | `get_app_config`, `AppConfigProvider.get_app_config` | `appconfig:GetLatestConfiguration` and `appconfig:StartConfigurationSession` | +| Provider | Function/Method | IAM Permission | +| ------------------- | ------------------------------------------------------------ | ------------------------------------------------------------------------------------ | +| SSM Parameter Store | **`get_parameter`**, **`SSMProvider.get`** | **`ssm:GetParameter`** | +| SSM Parameter Store | **`get_parameters`**, **`SSMProvider.get_multiple`** | **`ssm:GetParametersByPath`** | +| SSM Parameter Store | If using **`decrypt=True`** | You must add an additional permission **`kms:Decrypt`** | +| Secrets Manager | **`get_secret`**, **`SecretsManager.get`** | **`secretsmanager:GetSecretValue`** | +| DynamoDB | **`DynamoDBProvider.get`** | **`dynamodb:GetItem`** | +| DynamoDB | **`DynamoDBProvider.get_multiple`** | **`dynamodb:Query`** | +| App Config | **`get_app_config`**, **`AppConfigProvider.get_app_config`** | **`appconfig:GetLatestConfiguration`** and **`appconfig:StartConfigurationSession`** | ### Fetching parameters From 8ee7c2581681273ab1d6daa26bce60a9d7f6228a Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 18 Oct 2022 16:01:12 +0200 Subject: [PATCH 17/19] fix(docs): typing and correctness on null endpoint --- docs/utilities/parameters.md | 6 +++--- .../src/getting_started_recursive_ssm_parameter.py | 9 +++++---- .../src/getting_started_single_ssm_parameter.py | 4 +--- .../src/extending_built_in_models_with_json_mypy.py | 4 ++-- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/docs/utilities/parameters.md b/docs/utilities/parameters.md index ed54f2c9dfb..92b8ee08d85 100644 --- a/docs/utilities/parameters.md +++ b/docs/utilities/parameters.md @@ -39,17 +39,17 @@ This utility requires additional permissions to work as expected. ### Fetching parameters -You can retrieve a single parameter using `get_parameter` high-level function. +You can retrieve a single parameter using the `get_parameter` high-level function. For multiple parameters, you can use `get_parameters` and pass a path to retrieve them recursively. === "getting_started_single_ssm_parameter.py" - ```python hl_lines="5 12" + ```python hl_lines="3 10" --8<-- "examples/parameters/src/getting_started_single_ssm_parameter.py" ``` === "getting_started_recursive_ssm_parameter.py" - ```python hl_lines="5 12 15" + ```python hl_lines="3 10 13" --8<-- "examples/parameters/src/getting_started_recursive_ssm_parameter.py" ``` diff --git a/examples/parameters/src/getting_started_recursive_ssm_parameter.py b/examples/parameters/src/getting_started_recursive_ssm_parameter.py index 9cf48b39dde..5325a7fba96 100644 --- a/examples/parameters/src/getting_started_recursive_ssm_parameter.py +++ b/examples/parameters/src/getting_started_recursive_ssm_parameter.py @@ -1,5 +1,3 @@ -from typing import Any - import requests from aws_lambda_powertools.utilities import parameters @@ -9,14 +7,17 @@ def lambda_handler(event: dict, context: LambdaContext): try: # Retrieve multiple parameters from a path prefix - all_parameters: Any = parameters.get_parameters("/lambda-powertools/", max_age=20) - endpoint_comments = "https://jsonplaceholder.typicode.com/noexists/" + all_parameters: dict = parameters.get_parameters("/lambda-powertools/", max_age=20) + endpoint_comments = None for parameter, value in all_parameters.items(): if parameter == "endpoint_comments": endpoint_comments = value + if endpoint_comments is None: + return {"comments": None} + # the value of parameter is https://jsonplaceholder.typicode.com/comments/ comments: requests.Response = requests.get(endpoint_comments) diff --git a/examples/parameters/src/getting_started_single_ssm_parameter.py b/examples/parameters/src/getting_started_single_ssm_parameter.py index 199b402cc4a..d31c7a180f1 100644 --- a/examples/parameters/src/getting_started_single_ssm_parameter.py +++ b/examples/parameters/src/getting_started_single_ssm_parameter.py @@ -1,5 +1,3 @@ -from typing import Any - import requests from aws_lambda_powertools.utilities import parameters @@ -9,7 +7,7 @@ def lambda_handler(event: dict, context: LambdaContext) -> dict: try: # Retrieve a single parameter - endpoint_comments: Any = parameters.get_parameter("/lambda-powertools/endpoint_comments") + endpoint_comments: str = parameters.get_parameter("/lambda-powertools/endpoint_comments") # type: ignore[assignment] # noqa: E501 # the value of this parameter is https://jsonplaceholder.typicode.com/comments/ comments: requests.Response = requests.get(endpoint_comments) diff --git a/examples/parser/src/extending_built_in_models_with_json_mypy.py b/examples/parser/src/extending_built_in_models_with_json_mypy.py index 80314a814ce..813f757ad79 100644 --- a/examples/parser/src/extending_built_in_models_with_json_mypy.py +++ b/examples/parser/src/extending_built_in_models_with_json_mypy.py @@ -11,11 +11,11 @@ class CancelOrder(BaseModel): class CancelOrderModel(APIGatewayProxyEventV2Model): - body: Json[CancelOrder] # type: ignore[type-arg] + body: Json[CancelOrder] # type: ignore[assignment] @event_parser(model=CancelOrderModel) def handler(event: CancelOrderModel, context: LambdaContext): - cancel_order: CancelOrder = event.body # type: ignore[assignment] + cancel_order: CancelOrder = event.body assert cancel_order.order_id is not None From 31c4b314d8dd9f8ed9a90a7dbafd109cab671666 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 30 Jan 2023 21:24:20 +0000 Subject: [PATCH 18/19] (chore): addressing Ruben's feedbacks --- docs/utilities/parameters.md | 80 ++++--------------- .../get_parameter_by_name_error_handling.py | 21 +++++ .../src/getting_started_parameter_by_name.py | 17 ++++ 3 files changed, 55 insertions(+), 63 deletions(-) create mode 100644 examples/parameters/src/get_parameter_by_name_error_handling.py create mode 100644 examples/parameters/src/getting_started_parameter_by_name.py diff --git a/docs/utilities/parameters.md b/docs/utilities/parameters.md index 133d786adfa..d021ae44a61 100644 --- a/docs/utilities/parameters.md +++ b/docs/utilities/parameters.md @@ -40,54 +40,26 @@ This utility requires additional permissions to work as expected. ### Fetching parameters -You can retrieve a single parameter using the `get_parameter` high-level function. +You can retrieve a single parameter using the `get_parameter` high-level function. -```python hl_lines="5" title="Fetching a single parameter" -from aws_lambda_powertools.utilities import parameters - -def handler(event, context): - # Retrieve a single parameter - value = parameters.get_parameter("/my/parameter") - -``` +=== "getting_started_single_ssm_parameter.py" + ```python hl_lines="3 10" + --8<-- "examples/parameters/src/getting_started_single_ssm_parameter.py" + ``` For multiple parameters, you can use either: * `get_parameters` to recursively fetch all parameters by path. * `get_parameters_by_name` to fetch distinct parameters by their full name. It also accepts custom caching, transform, decrypt per parameter. -=== "get_parameters" - - ```python hl_lines="1 6" - from aws_lambda_powertools.utilities import parameters - - def handler(event, context): - # Retrieve multiple parameters from a path prefix recursively - # This returns a dict with the parameter name as key - values = parameters.get_parameters("/my/path/prefix") - for parameter, value in values.items(): - print(f"{parameter}: {value}") +=== "getting_started_recursive_ssm_parameter.py" + ```python hl_lines="3 10 13" + --8<-- "examples/parameters/src/getting_started_recursive_ssm_parameter.py" ``` -=== "get_parameters_by_name" - - ```python hl_lines="3 5 14" - from typing import Any - - from aws_lambda_powertools.utilities import get_parameters_by_name - - parameters = { - "/develop/service/commons/telemetry/config": {"max_age": 300, "transform": "json"}, - "/no_cache_param": {"max_age": 0}, - # inherit default values - "/develop/service/payment/api/capture/url": {}, - } - - def handler(event, context): - # This returns a dict with the parameter name as key - response: dict[str, Any] = parameters.get_parameters_by_name(parameters=parameters, max_age=60) - for parameter, value in response.items(): - print(f"{parameter}: {value}") +=== "getting_started_parameter_by_name.py" + ```python hl_lines="3 14" + --8<-- "examples/parameters/src/getting_started_parameter_by_name.py" ``` ???+ tip "`get_parameters_by_name` supports graceful error handling" @@ -99,32 +71,14 @@ For multiple parameters, you can use either: * Keep only successful parameter names and their values in the response * Raise `GetParameterError` if any of your parameters is named `_errors` -```python hl_lines="3 5 12-13 15" title="Graceful error handling" -from typing import Any - -from aws_lambda_powertools.utilities import get_parameters_by_name - -parameters = { - "/develop/service/commons/telemetry/config": {"max_age": 300, "transform": "json"}, - # it would fail by default - "/this/param/does/not/exist" -} - -def handler(event, context): - values: dict[str, Any] = parameters.get_parameters_by_name(parameters=parameters, raise_on_error=False) - errors: list[str] = values.get("_errors", []) - - # Handle gracefully, since '/this/param/does/not/exist' will only be available in `_errors` - if errors: - ... - - for parameter, value in values.items(): - print(f"{parameter}: {value}") -``` +=== "get_parameter_by_name_error_handling.py" + ```python hl_lines="3 5 12-13 15" + --8<-- "examples/parameters/src/get_parameter_by_name_error_handling.py" + ``` ### Fetching secrets -You can fetch secrets stored in Secrets Manager using `get_secrets`. +You can fetch secrets stored in Secrets Manager using `get_secret`. === "getting_started_secret.py" ```python hl_lines="5 15" @@ -314,7 +268,7 @@ DynamoDB provider can be customized at initialization to match your table struct | **value_attr** | No | `value` | Name of the attribute containing the parameter value. | === "builtin_provider_dynamodb_custom_fields.py" - ```python hl_lines="3 5 9 10 15" + ```python hl_lines="3 8-10 17" --8<-- "examples/parameters/src/builtin_provider_dynamodb_custom_fields.py" ``` diff --git a/examples/parameters/src/get_parameter_by_name_error_handling.py b/examples/parameters/src/get_parameter_by_name_error_handling.py new file mode 100644 index 00000000000..60ad5c9d8cc --- /dev/null +++ b/examples/parameters/src/get_parameter_by_name_error_handling.py @@ -0,0 +1,21 @@ +from typing import Any + +from aws_lambda_powertools.utilities import get_parameters_by_name + +parameters = { + "/develop/service/commons/telemetry/config": {"max_age": 300, "transform": "json"}, + # it would fail by default + "/this/param/does/not/exist": {}, +} + + +def handler(event, context): + values: dict[str, Any] = get_parameters_by_name(parameters=parameters, raise_on_error=False) + errors: list[str] = values.get("_errors", []) + + # Handle gracefully, since '/this/param/does/not/exist' will only be available in `_errors` + if errors: + ... + + for parameter, value in values.items(): + print(f"{parameter}: {value}") diff --git a/examples/parameters/src/getting_started_parameter_by_name.py b/examples/parameters/src/getting_started_parameter_by_name.py new file mode 100644 index 00000000000..cbfbe0f5d48 --- /dev/null +++ b/examples/parameters/src/getting_started_parameter_by_name.py @@ -0,0 +1,17 @@ +from typing import Any + +from aws_lambda_powertools.utilities import get_parameters_by_name + +parameters = { + "/develop/service/commons/telemetry/config": {"max_age": 300, "transform": "json"}, + "/no_cache_param": {"max_age": 0}, + # inherit default values + "/develop/service/payment/api/capture/url": {}, +} + + +def handler(event, context): + # This returns a dict with the parameter name as key + response: dict[str, Any] = get_parameters_by_name(parameters=parameters, max_age=60) + for parameter, value in response.items(): + print(f"{parameter}: {value}") From b423a3e35d3acb66b9eb26a797601a9c708d75c7 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 30 Jan 2023 21:26:05 +0000 Subject: [PATCH 19/19] (chore): import errors --- examples/parameters/src/get_parameter_by_name_error_handling.py | 2 +- examples/parameters/src/getting_started_parameter_by_name.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/parameters/src/get_parameter_by_name_error_handling.py b/examples/parameters/src/get_parameter_by_name_error_handling.py index 60ad5c9d8cc..7cae4525e83 100644 --- a/examples/parameters/src/get_parameter_by_name_error_handling.py +++ b/examples/parameters/src/get_parameter_by_name_error_handling.py @@ -1,6 +1,6 @@ from typing import Any -from aws_lambda_powertools.utilities import get_parameters_by_name +from aws_lambda_powertools.utilities.parameters.ssm import get_parameters_by_name parameters = { "/develop/service/commons/telemetry/config": {"max_age": 300, "transform": "json"}, diff --git a/examples/parameters/src/getting_started_parameter_by_name.py b/examples/parameters/src/getting_started_parameter_by_name.py index cbfbe0f5d48..95d63937ab7 100644 --- a/examples/parameters/src/getting_started_parameter_by_name.py +++ b/examples/parameters/src/getting_started_parameter_by_name.py @@ -1,6 +1,6 @@ from typing import Any -from aws_lambda_powertools.utilities import get_parameters_by_name +from aws_lambda_powertools.utilities.parameters.ssm import get_parameters_by_name parameters = { "/develop/service/commons/telemetry/config": {"max_age": 300, "transform": "json"},