feature: add environment variables (#19)

Co-authored-by: Ran Isenberg <ran.isenberg@cyberark.com>

Author: ran-isenberg

README.md

@@ -48,7 +48,7 @@ The utilities cover multiple aspect of a production-ready service, including:
 1.  [Logging](https://www.ranthebuilder.cloud/post/aws-lambda-cookbook-elevate-your-handler-s-code-part-1-logging)
 2.  [Observability: Monitoring and Tracing](https://www.ranthebuilder.cloud/post/aws-lambda-cookbook-elevate-your-handler-s-code-part-2-observability)
 3.  [Observability: Business Domain Metrics](https://www.ranthebuilder.cloud/post/aws-lambda-cookbook-elevate-your-handler-s-code-part-3-business-domain-observability)
-4.  Environment variables.
+4.  [Environment variables](https://www.ranthebuilder.cloud/post/aws-lambda-cookbook-elevate-your-handler-s-code-part-3-business-domain-observability)
 5.  Input validation
 6.  Features flags & dynamic configuration
 

cdk/aws_lambda_handler_cookbook/service_stack/cookbook_construct.py

@@ -77,8 +77,10 @@ def __add_get_lambda_integration(self, api_name: aws_apigateway.Resource):
             code=_lambda.Code.from_asset(BUILD_FOLDER),
             handler='service.handlers.my_handler.my_handler',
             environment={
-                POWERTOOLS_SERVICE_NAME: SERVICE_NAME,
-                POWER_TOOLS_LOG_LEVEL: 'DEBUG',
+                POWERTOOLS_SERVICE_NAME: SERVICE_NAME,  # for logger, tracer and metrics
+                POWER_TOOLS_LOG_LEVEL: 'DEBUG',  # for logger
+                'REST_API': 'https://www.ranthebuilder.cloud/api',  # for env vars example
+                'ROLE_ARN': 'arn:partition:service:region:account-id:resource-type:resource-id',  # for env vars example
             },
             tracing=_lambda.Tracing.ACTIVE,
             retry_attempts=0,

docs/best_practices/environment_variables.md

@@ -0,0 +1,113 @@
+---
+title: Environment Variables
+description: Environment Variables
+---
+Environment Variables decorator is a simple parser for environment variables that run at the start of the handler invocation.
+
+![Environment Variables](../media/pydantic.png){: style="height:50%;width:20%"}
+
+## Key features
+* A defined [Pydantic](https://pydantic-docs.helpmanual.io/){:target="_blank" rel="noopener"} schema for all required environment variables
+* A decorator that parses and validates environment variables, value constraints included
+* Global getter for parsed & valid schema dataclass with all environment variables
+
+
+
+The best practice for handling environment variables is to validate & parse them according to a predefined schema as soon as the AWS Lambda function is triggered.
+
+In case of misconfiguration, a validation exception is raised with all the relevant exception details.
+
+
+## Blog Reference
+Read more about the importance of validating environment variables and how this utility works. Click [**HERE**](https://www.ranthebuilder.cloud/post/aws-lambda-cookbook-elevate-your-handler-s-code-part-3-business-domain-observability){:target="_blank" rel="noopener"}
+
+
+## Schema Definition
+
+You need to define all your environment variables in a Pydantic schema class that extend Pydantic's BaseModel class.
+
+For example:
+=== "schemas/env_vars.py"
+
+    ```python hl_lines="5"
+    from typing import Literal
+
+    from pydantic import BaseModel, HttpUrl, constr
+
+    class MyHandlerEnvVars(BaseModel):
+        REST_API: HttpUrl
+        ROLE_ARN: constr(min_length=20, max_length=2048)
+        POWERTOOLS_SERVICE_NAME: constr(min_length=1)
+        LOG_LEVEL: Literal['DEBUG', 'INFO', 'ERROR', 'CRITICAL', 'WARNING', 'EXCEPTION']
+
+    ```
+
+All Pydantic schemas extend Pydantic’s ‘BaseModel’ class, turning them into a dataclass.
+
+The schema defines four environment variables: ‘LOG_LEVEL,’ ‘POWERTOOLS_SERVICE_NAME,’ ‘ROLE_ARN,’ and ‘REST_API.’
+
+This schema makes sure that:
+
+- ‘LOG_LEVEL’ is one of the strings in the Literal list.
+- ‘ROLE_ARN’ exists and is between 20 and 2048 characters long, as defined here.
+- ‘REST_API’ is a valid HTTP URL.
+- ‘POWERTOOLS_SERVICE_NAME’ is a non-empty string.
+
+Read [here](https://pydantic-docs.helpmanual.io/usage/models/){:target="_blank" rel="noopener"} about Pydantic Model capabilities.
+
+## Decorator Usage
+The decorator 'init_environment_variables' is defined under the utility folder **service.utils.env_vars_parser.py** and imported in the handler.
+
+The decorator requires a **model** parameter, which in this example is the name of the schema class we defined above.
+
+=== "handlers/my_handler.py"
+
+    ```python hl_lines="11"
+    import json
+    from http import HTTPStatus
+    from typing import Any, Dict
+
+    from aws_lambda_powertools.utilities.typing import LambdaContext
+
+    from service.handlers.schemas.env_vars import MyHandlerEnvVars
+    from service.handlers.utils.env_vars_parser import init_environment_variables
+
+
+    @init_environment_variables(model=MyHandlerEnvVars)
+    def my_handler(event: Dict[str, Any], context: LambdaContext) -> Dict[str, Any]:
+        return {'statusCode': HTTPStatus.OK, 'headers': {'Content-Type': 'application/json'}, 'body': json.dumps({'message': 'success'})}
+    ```
+
+## Global Getter Usage
+The getter function 'get_environment_variables' is defined under the utility folder **service.utils.env_vars_parser.py** and imported in the handler.
+
+The getter function returns a parsed and validated global instance of the environment variables Pydantic schema class.
+
+It can be used *anywhere* in the function code, not just the handler.
+
+=== "handlers/my_handler.py"
+
+    ```python hl_lines="13"
+    import json
+    from http import HTTPStatus
+    from typing import Any, Dict
+
+    from aws_lambda_powertools.utilities.typing import LambdaContext
+
+    from service.handlers.schemas.env_vars import MyHandlerEnvVars
+    from service.handlers.utils.env_vars_parser import get_environment_variables, init_environment_variables
+
+
+    @init_environment_variables(model=MyHandlerEnvVars)
+    def my_handler(event: Dict[str, Any], context: LambdaContext) -> Dict[str, Any]:
+        env_vars: MyHandlerEnvVars = get_environment_variables()
+        return {'statusCode': HTTPStatus.OK, 'headers': {'Content-Type': 'application/json'}, 'body': json.dumps({'message': 'success'})}
+    ```
+
+
+
+## More Details
+
+Read [here](https://pydantic-docs.helpmanual.io/usage/types/){:target="_blank" rel="noopener"} about Pydantic field types.
+
+Read [here](https://pydantic-docs.helpmanual.io/usage/validators/){:target="_blank" rel="noopener"} about custom validators and advanced value constraints.

docs/best_practices/logger.md

@@ -14,11 +14,11 @@ It’s a wrapper of Python’s logging library that provides extra capabilities
 
 
 ## Usage in Handler
-The logger is a singleton which is defined under the utility folder **service.utils.infra.py** and imported in the handler.
+The logger is a singleton which is defined under the utility folder **service.utils.observability.py** and imported in the handler.
 
 ## Blog Reference
 Read more about the importance of the logger and how to use AWS CloudWatch logs in my blog. Click [**HERE**](https://www.ranthebuilder.cloud/post/aws-lambda-cookbook-elevate-your-handler-s-code-part-1-logging){:target="_blank" rel="noopener"}
 
 
-## More details
+## More Details
 You can find more information at the official documentation. Go to [https://awslabs.github.io/aws-lambda-powertools-python/latest/core/logger/](https://awslabs.github.io/aws-lambda-powertools-python/latest/core/logger/){:target="_blank" rel="noopener"}

docs/best_practices/metrics.md

@@ -20,11 +20,11 @@ These metrics can be visualized through [Amazon CloudWatch Console](https://cons
 
 
 ## Usage in Handler
-The metrics is a singleton which is defined under the utility folder **service.utils.infra.py** and imported in the handler.
+The metrics is a singleton which is defined under the utility folder **service.utils.observability.py** and imported in the handler.
 
 ## Blog Reference
 Read more about the importance of the business KPis and metrics in my blog. Click [**HERE**](https://www.ranthebuilder.cloud/post/aws-lambda-cookbook-elevate-your-handler-s-code-part-3-business-domain-observability){:target="_blank" rel="noopener"}
 
 
-## More details
+## More Details
 You can find more information at the official documentation. Go to [https://awslabs.github.io/aws-lambda-powertools-python/latest/core/metrics/](https://awslabs.github.io/aws-lambda-powertools-python/latest/core/metrics/){:target="_blank" rel="noopener"}

docs/best_practices/tracer.md

@@ -14,11 +14,11 @@ Tracer is a thin wrapper for [AWS X-Ray Python SDK](https://github.com/aws/aws-x
 
 
 ## Usage in Handler
-The tracer is a singleton which is defined under the utility folder **service.utils.infra.py** and imported in the handler.
+The tracer is a singleton which is defined under the utility folder **service.utils.observability.py** and imported in the handler.
 
 ## Blog Reference
 Read more about the importance of observability and traces in my blog. Click [**HERE**](https://www.ranthebuilder.cloud/post/aws-lambda-cookbook-elevate-your-handler-s-code-part-2-observability){:target="_blank" rel="noopener"}
 
 
-## More details
+## More Details
 You can find more information at the official documentation. Go to [https://awslabs.github.io/aws-lambda-powertools-python/latest/core/tracer/](https://awslabs.github.io/aws-lambda-powertools-python/latest/core/tracer/){:target="_blank" rel="noopener"}

docs/index.md

@@ -36,7 +36,7 @@ The utilities cover multiple aspects of a production-ready service, including:
 * [**Logging**](best_practices/logger.md)
 * [**Observability: Monitoring and Tracing**](best_practices/tracer.md)
 * [**Observability: Business KPI Metrics**](best_practices/metrics.md)
-* **Environment variables** - Not published yet
+* [**Environment variables**](best_practices/environment_variables.md)
 * **Input validation** - Not published yet
 * **Features flags & dynamic configuration** - Not published yet
 

docs/media/pydantic.png

@@ -13,6 +13,7 @@ nav:
       - best_practices/logger.md
       - best_practices/tracer.md
       - best_practices/metrics.md
+      - best_practices/environment_variables.md
 
 theme:
   name: material
@@ -60,7 +61,11 @@ markdown_extensions:
   - attr_list
   - pymdownx.emoji
   - pymdownx.inlinehilite
-  - pymdownx.superfences
+  - pymdownx.superfences:
+      custom_fences:
+        - name: mermaid
+          class: mermaid
+          format: !!python/name:pymdownx.superfences.fence_code_format
 
 copyright: Copyright © 2022 Ran Isenberg
 

mkdocs.yml

@@ -5,20 +5,27 @@
 from aws_lambda_powertools.metrics.metrics import MetricUnit
 from aws_lambda_powertools.utilities.typing import LambdaContext
 
-from service.utils.infra import logger, metrics, tracer
+from service.handlers.schemas.env_vars import MyHandlerEnvVars
+from service.handlers.utils.env_vars_parser import get_environment_variables, init_environment_variables
+from service.handlers.utils.observability import logger, metrics, tracer
 
 
 @tracer.capture_method(capture_response=False)
 def inner_function_example(event: Dict[str, Any]) -> Dict[str, Any]:
     return {}
 
 
+@init_environment_variables(model=MyHandlerEnvVars)
 @metrics.log_metrics
 @tracer.capture_lambda_handler(capture_response=False)
 def my_handler(event: Dict[str, Any], context: LambdaContext) -> Dict[str, Any]:
     logger.set_correlation_id(context.aws_request_id)
-    logger.debug('my_handler is called, calling inner_function_example')
+    logger.info('my_handler is called, calling inner_function_example')
+
+    env_vars: MyHandlerEnvVars = get_environment_variables()
+    logger.debug('environment variables', extra=env_vars.dict())
+
     inner_function_example(event)
-    logger.debug('inner_function_example finished successfully')
+    logger.info('inner_function_example finished successfully')
     metrics.add_metric(name='ValidEvents', unit=MetricUnit.Count, value=1)
     return {'statusCode': HTTPStatus.OK, 'headers': {'Content-Type': 'application/json'}, 'body': json.dumps({'message': 'success'})}

service/handlers/my_handler.py

@@ -0,0 +1,13 @@
+from typing import Literal
+
+from pydantic import BaseModel, HttpUrl, constr
+
+
+class Observability(BaseModel):
+    POWERTOOLS_SERVICE_NAME: constr(min_length=1)
+    LOG_LEVEL: Literal['DEBUG', 'INFO', 'ERROR', 'CRITICAL', 'WARNING', 'EXCEPTION']
+
+
+class MyHandlerEnvVars(Observability):
+    REST_API: HttpUrl
+    ROLE_ARN: constr(min_length=20, max_length=2048)

service/utils/__init__.py renamed to service/handlers/schemas/__init__.py

@@ -0,0 +1,35 @@
+import os
+from typing import Any, TypeVar
+
+from aws_lambda_powertools.middleware_factory import lambda_handler_decorator
+from pydantic import BaseModel, ValidationError
+
+Model = TypeVar('Model', bound=BaseModel)
+
+# global instance of the parsed Pydantic data class
+ENV_CONF: BaseModel = None
+
+
+@lambda_handler_decorator
+def init_environment_variables(handler, event, context, model: Model) -> Any:
+    global ENV_CONF
+    try:
+        # parse the os environment variables dict
+        ENV_CONF = model(**os.environ)
+    except (ValidationError, TypeError) as exc:
+        raise ValueError(f'failed to load environment variables, exception={str(exc)}') from exc
+
+    return handler(event, context)
+
+
+def get_environment_variables() -> BaseModel:
+    global ENV_CONF
+    if ENV_CONF is None:
+        raise ValueError('get_environment_variables was called before init_environment_variables, environment variables were not loaded')
+    return ENV_CONF
+
+
+# used for testing purposes
+def _clear_env_conf() -> None:
+    global ENV_CONF
+    ENV_CONF = None

service/handlers/schemas/env_vars.py

@@ -0,0 +1,67 @@
+import os
+from typing import Any, Dict, Literal
+from unittest import mock
+
+import pytest
+from pydantic import BaseModel, HttpUrl
+
+from cdk.aws_lambda_handler_cookbook.service_stack.constants import POWER_TOOLS_LOG_LEVEL, POWERTOOLS_SERVICE_NAME, SERVICE_NAME
+from service.handlers.utils.env_vars_parser import _clear_env_conf, get_environment_variables, init_environment_variables
+from tests.utils import generate_context
+
+
+class MySchema(BaseModel):
+    POWERTOOLS_SERVICE_NAME: str
+    LOG_LEVEL: Literal['DEBUG', 'INFO', 'ERROR', 'CRITICAL', 'WARNING', 'EXCEPTION']
+    REST_API: HttpUrl
+
+
+@mock.patch.dict(os.environ, {
+    POWERTOOLS_SERVICE_NAME: SERVICE_NAME,
+    POWER_TOOLS_LOG_LEVEL: 'DEBUG',
+    'REST_API': 'https://www.ranthebuilder.cloud/api'
+})
+def test_handler_schema_ok():
+
+    @init_environment_variables(model=MySchema)
+    def my_handler(event, context) -> Dict[str, Any]:
+        env_vars: MySchema = get_environment_variables()
+        assert env_vars.POWERTOOLS_SERVICE_NAME == SERVICE_NAME
+        assert env_vars.LOG_LEVEL == 'DEBUG'
+        assert str(env_vars.REST_API) == 'https://www.ranthebuilder.cloud/api'
+        return {}
+
+    my_handler({}, generate_context())
+    _clear_env_conf()
+
+
+def test_handler_missing_env_var():
+
+    @init_environment_variables(model=MySchema)
+    def my_handler1(event, context) -> Dict[str, Any]:
+        return {}
+
+    with pytest.raises(ValueError):
+        my_handler1({}, generate_context())
+
+
+@mock.patch.dict(os.environ, {POWERTOOLS_SERVICE_NAME: SERVICE_NAME, POWER_TOOLS_LOG_LEVEL: 'DEBUG', 'REST_API': 'fakeapi'})
+def test_handler_invalid_env_var_value():
+
+    @init_environment_variables(model=MySchema)
+    def my_handler2(event, context) -> Dict[str, Any]:
+        return {}
+
+    with pytest.raises(ValueError):
+        my_handler2({}, generate_context())
+
+
+@mock.patch.dict(os.environ, {POWERTOOLS_SERVICE_NAME: SERVICE_NAME, POWER_TOOLS_LOG_LEVEL: 'DEBUG', 'REST_API': 'fakeapi'})
+def test_handler_get_env_var_without_init():
+
+    def my_handler3(event, context) -> Dict[str, Any]:
+        get_environment_variables()
+        return {}
+
+    with pytest.raises(ValueError):
+        my_handler3({}, generate_context())

service/handlers/utils/__init__.py

@@ -13,6 +13,8 @@
 def init():
     os.environ[POWERTOOLS_SERVICE_NAME] = SERVICE_NAME
     os.environ[POWER_TOOLS_LOG_LEVEL] = 'DEBUG'
+    os.environ['REST_API'] = 'https://www.ranthebuilder.cloud/api'
+    os.environ['ROLE_ARN'] = 'arn:partition:service:region:account-id:resource-type:resource-id'
 
 
 def test_handler_200_ok():