Merge branch 'develop' of github.com:risenberg-cyberark/aws-lambda-powertools-python into pydantic

# Conflicts:
#	pyproject.toml

Author: Ran Isenberg

.pre-commit-config.yaml

@@ -1,5 +1,5 @@
 # We use poetry to run formatting and linting before commit/push
-# Longers checks such as tests, security and complexity baseline
+# Longer checks such as tests, security and complexity baseline
 # are run as part of CI to prevent slower feedback loop
 # All checks can be run locally via `make pr`
 

CHANGELOG.md

@@ -6,6 +6,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+- **Metrics**: Support adding multiple metric values to a metric name
+- **Utilities**: Add new `Validator` utility to validate inbound events and responses using JSON Schema
+
+## [1.5.0] - 2020-09-04
+
+### Added
+- **Logger**: Add `xray_trace_id` to log output to improve integration with CloudWatch Service Lens
+- **Logger**: Allow reordering of logged output
+- **Utilities**: Add new `SQS batch processing` utility to handle partial failures in processing message batches
+- **Utilities**: Add typing utility providing static type for lambda context object
+- **Utilities**: Add `transform=auto` in parameters utility to deserialize parameter values based on the key name
+
+### Fixed
+- **Logger**: The value of `json_default` formatter is no longer written to logs
+
 ## [1.4.0] - 2020-08-25
 
 ### Added

CONTRIBUTING.md

@@ -69,6 +69,19 @@ opensource-codeofconduct@amazon.com with any additional questions or comments.
 ## Security issue notifications
 If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue.
 
+## Troubleshooting
+
+### API reference documentation
+
+When you are working on the codebase and you use the local API reference documentation to preview your changes, you might see the following message: `Module aws_lambda_powertools not found`.
+
+This happens when:
+
+* You did not install the local dev environment yet
+    - You can install dev deps with `make dev` command
+* The code in the repository is raising an exception while the `pdoc` is scanning the codebase
+    - Unfortunately, this exception is not shown to you, but if you run, `poetry run pdoc --pdf aws_lambda_powertools`, the exception is shown and you can prevent the exception from being raised
+    - Once resolved the documentation should load correctly again
 
 ## Licensing
 

Makefile

@@ -4,7 +4,7 @@ target:
 
 dev:
 	pip install --upgrade pip poetry pre-commit
-	poetry install
+	poetry install --extras "jmespath"
 	pre-commit install
 
 dev-docs:

README.md

@@ -3,10 +3,12 @@
 ![Build](https://github.com/awslabs/aws-lambda-powertools/workflows/Powertools%20Python/badge.svg?branch=master)
 ![PythonSupport](https://img.shields.io/static/v1?label=python&message=3.6%20|%203.7|%203.8&color=blue?style=flat-square&logo=python) ![PyPI version](https://badge.fury.io/py/aws-lambda-powertools.svg) ![PyPi monthly downloads](https://img.shields.io/pypi/dm/aws-lambda-powertools)
 
-A suite of utilities for AWS Lambda Functions that makes tracing with AWS X-Ray, structured logging and creating custom metrics asynchronously easier.
+A suite of utilities for AWS Lambda functions that makes tracing with AWS X-Ray, structured logging and creating custom metrics asynchronously easier.
 
 **[📜Documentation](https://awslabs.github.io/aws-lambda-powertools-python/)** | **[API Docs](https://awslabs.github.io/aws-lambda-powertools-python/api/)** | **[🐍PyPi](https://pypi.org/project/aws-lambda-powertools/)** | **[Feature request](https://github.com/awslabs/aws-lambda-powertools-python/issues/new?assignees=&labels=feature-request%2C+triage&template=feature_request.md&title=)** | **[🐛Bug Report](https://github.com/awslabs/aws-lambda-powertools-python/issues/new?assignees=&labels=bug%2C+triage&template=bug_report.md&title=)** | **[Kitchen sink example](https://github.com/awslabs/aws-lambda-powertools-python/tree/develop/example)** | **[Detailed blog post](https://aws.amazon.com/blogs/opensource/simplifying-serverless-best-practices-with-lambda-powertools/)**
 
+> **Join us on the AWS Developers Slack at `#lambda-powertools`** - **[Invite, if you don't have an account](https://join.slack.com/t/awsdevelopers/shared_invite/zt-gu30gquv-EhwIYq3kHhhysaZ2aIX7ew)**
+
 ## Features
 
 * **[Tracing](https://awslabs.github.io/aws-lambda-powertools-python/core/tracer/)** - Decorators and utilities to trace Lambda function handlers, and both synchronous and asynchronous functions

aws_lambda_powertools/logging/formatter.py

@@ -1,27 +1,6 @@
 import json
 import logging
-from typing import Any
-
-
-def json_formatter(unserializable_value: Any):
-    """JSON custom serializer to cast unserializable values to strings.
-
-    Example
-    -------
-
-    **Serialize unserializable value to string**
-
-        class X: pass
-        value = {"x": X()}
-
-        json.dumps(value, default=json_formatter)
-
-    Parameters
-    ----------
-    unserializable_value: Any
-        Python object unserializable by JSON
-    """
-    return str(unserializable_value)
+import os
 
 
 class JsonFormatter(logging.Formatter):
@@ -42,19 +21,37 @@ def __init__(self, **kwargs):
         unserializable values.  It must not throw.  Defaults to a function that
         coerces the value to a string.
 
+        The `log_record_order` kwarg is used to specify the order of the keys used in
+        the structured json logs. By default the order is: "level", "location", "message", "timestamp",
+        "service" and "sampling_rate".
+
         Other kwargs are used to specify log field format strings.
         """
-        self.default_json_formatter = kwargs.pop("json_default", json_formatter)
-        datefmt = kwargs.pop("datefmt", None)
-
-        super(JsonFormatter, self).__init__(datefmt=datefmt)
+        # Set the default unserializable function, by default values will be cast as str.
+        self.default_json_formatter = kwargs.pop("json_default", str)
+        # Set the insertion order for the log messages
+        self.format_dict = dict.fromkeys(kwargs.pop("log_record_order", ["level", "location", "message", "timestamp"]))
         self.reserved_keys = ["timestamp", "level", "location"]
-        self.format_dict = {
-            "timestamp": "%(asctime)s",
+        # Set the date format used by `asctime`
+        super(JsonFormatter, self).__init__(datefmt=kwargs.pop("datefmt", None))
+
+        self.format_dict.update(self._build_root_keys(**kwargs))
+
+    @staticmethod
+    def _build_root_keys(**kwargs):
+        return {
             "level": "%(levelname)s",
             "location": "%(funcName)s:%(lineno)d",
+            "timestamp": "%(asctime)s",
+            **kwargs,
         }
-        self.format_dict.update(kwargs)
+
+    @staticmethod
+    def _get_latest_trace_id():
+        xray_trace_id = os.getenv("_X_AMZN_TRACE_ID")
+        trace_id = xray_trace_id.split(";")[0].replace("Root=", "") if xray_trace_id else None
+
+        return trace_id
 
     def update_formatter(self, **kwargs):
         self.format_dict.update(kwargs)
@@ -94,4 +91,10 @@ def format(self, record):  # noqa: A003
         if record.exc_text:
             log_dict["exception"] = record.exc_text
 
+        # fetch latest X-Ray Trace ID, if any
+        log_dict.update({"xray_trace_id": self._get_latest_trace_id()})
+
+        # Filter out top level key with values that are None
+        log_dict = {k: v for k, v in log_dict.items() if v is not None}
+
         return json.dumps(log_dict, default=self.default_json_formatter)

aws_lambda_powertools/logging/logger.py

@@ -136,16 +136,6 @@ def __getattr__(self, name):
         # https://github.com/awslabs/aws-lambda-powertools-python/issues/97
         return getattr(self._logger, name)
 
-    def _get_log_level(self, level: Union[str, int]) -> Union[str, int]:
-        """ Returns preferred log level set by the customer in upper case """
-        if isinstance(level, int):
-            return level
-
-        log_level: str = level or os.getenv("LOG_LEVEL")
-        log_level = log_level.upper() if log_level is not None else logging.INFO
-
-        return log_level
-
     def _get_logger(self):
         """ Returns a Logger named {self.service}, or {self.service.filename} for child loggers"""
         logger_name = self.service
@@ -154,17 +144,6 @@ def _get_logger(self):
 
         return logging.getLogger(logger_name)
 
-    def _get_caller_filename(self):
-        """ Return caller filename by finding the caller frame """
-        # Current frame         => _get_logger()
-        # Previous frame        => logger.py
-        # Before previous frame => Caller
-        frame = inspect.currentframe()
-        caller_frame = frame.f_back.f_back.f_back
-        filename = caller_frame.f_globals["__name__"]
-
-        return filename
-
     def _init_logger(self, **kwargs):
         """Configures new logger"""
 
@@ -207,6 +186,8 @@ def inject_lambda_context(self, lambda_handler: Callable[[Dict, Any], Any] = Non
 
         Parameters
         ----------
+        lambda_handler : Callable
+            Method to inject the lambda context
         log_event : bool, optional
             Instructs logger to log Lambda Event, by default False
 
@@ -254,14 +235,14 @@ def handler(event, context):
 
         @functools.wraps(lambda_handler)
         def decorate(event, context):
+            lambda_context = build_lambda_context_model(context)
+            cold_start = _is_cold_start()
+            self.structure_logs(append=True, cold_start=cold_start, **lambda_context.__dict__)
+
             if log_event:
                 logger.debug("Event received")
                 self.info(event)
 
-            lambda_context = build_lambda_context_model(context)
-            cold_start = _is_cold_start()
-
-            self.structure_logs(append=True, cold_start=cold_start, **lambda_context.__dict__)
             return lambda_handler(event, context)
 
         return decorate
@@ -291,6 +272,29 @@ def structure_logs(self, append: bool = False, **kwargs):
                 # Set a new formatter for a logger handler
                 handler.setFormatter(JsonFormatter(**self._default_log_keys, **kwargs))
 
+    @staticmethod
+    def _get_log_level(level: Union[str, int]) -> Union[str, int]:
+        """ Returns preferred log level set by the customer in upper case """
+        if isinstance(level, int):
+            return level
+
+        log_level: str = level or os.getenv("LOG_LEVEL")
+        log_level = log_level.upper() if log_level is not None else logging.INFO
+
+        return log_level
+
+    @staticmethod
+    def _get_caller_filename():
+        """ Return caller filename by finding the caller frame """
+        # Current frame         => _get_logger()
+        # Previous frame        => logger.py
+        # Before previous frame => Caller
+        frame = inspect.currentframe()
+        caller_frame = frame.f_back.f_back.f_back
+        filename = caller_frame.f_globals["__name__"]
+
+        return filename
+
 
 def set_package_logger(
     level: Union[str, int] = logging.DEBUG, stream: sys.stdout = None, formatter: logging.Formatter = None

aws_lambda_powertools/metrics/base.py

@@ -4,6 +4,7 @@
 import numbers
 import os
 import pathlib
+from collections import defaultdict
 from enum import Enum
 from typing import Any, Dict, List, Union
 
@@ -93,7 +94,7 @@ def __init__(
         self._metric_unit_options = list(MetricUnit.__members__)
         self.metadata_set = self.metadata_set if metadata_set is not None else {}
 
-    def add_metric(self, name: str, unit: MetricUnit, value: Union[float, int]):
+    def add_metric(self, name: str, unit: Union[MetricUnit, str], value: float):
         """Adds given metric
 
         Example
@@ -110,7 +111,7 @@ def add_metric(self, name: str, unit: MetricUnit, value: Union[float, int]):
         ----------
         name : str
             Metric name
-        unit : MetricUnit
+        unit : Union[MetricUnit, str]
             `aws_lambda_powertools.helper.models.MetricUnit`
         value : float
             Metric value
@@ -124,7 +125,9 @@ def add_metric(self, name: str, unit: MetricUnit, value: Union[float, int]):
             raise MetricValueError(f"{value} is not a valid number")
 
         unit = self.__extract_metric_unit_value(unit=unit)
-        metric = {"Unit": unit, "Value": float(value)}
+        metric = self.metric_set.get(name, defaultdict(list))
+        metric["Unit"] = unit
+        metric["Value"].append(float(value))
         logger.debug(f"Adding metric: {name} with {metric}")
         self.metric_set[name] = metric
 
@@ -146,6 +149,8 @@ def serialize_metric_set(self, metrics: Dict = None, dimensions: Dict = None, me
             Dictionary of metrics to serialize, by default None
         dimensions : Dict, optional
             Dictionary of dimensions to serialize, by default None
+        metadata: Dict, optional
+            Dictionary of metadata to serialize, by default None
 
         Example
         -------
@@ -183,7 +188,7 @@ def serialize_metric_set(self, metrics: Dict = None, dimensions: Dict = None, me
         metric_names_and_values: Dict[str, str] = {}  # { "metric_name": 1.0 }
 
         for metric_name in metrics:
-            metric: str = metrics[metric_name]
+            metric: dict = metrics[metric_name]
             metric_value: int = metric.get("Value", 0)
             metric_unit: str = metric.get("Unit", "")
 
@@ -257,7 +262,7 @@ def add_metadata(self, key: str, value: Any):
 
         Parameters
         ----------
-        name : str
+        key : str
             Metadata key
         value : any
             Metadata value

aws_lambda_powertools/utilities/batch/__init__.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+
+"""
+Batch processing utility
+"""
+
+from .base import BasePartialProcessor, batch_processor
+from .sqs import PartialSQSProcessor, sqs_batch_processor
+
+__all__ = ("BasePartialProcessor", "PartialSQSProcessor", "batch_processor", "sqs_batch_processor")