From 35a319517b4a226a714b110493d150b383286886 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 1 May 2023 14:21:52 +0200 Subject: [PATCH 1/6] feat(logger): add DatadogLogFormatter --- aws_lambda_powertools/logging/formatter.py | 4 +-- .../logging/formatters/__init__.py | 3 ++ .../logging/formatters/datadog.py | 31 +++++++++++++++++++ .../test_logger_powertools_formatter.py | 20 ++++++++++++ 4 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 aws_lambda_powertools/logging/formatters/__init__.py create mode 100644 aws_lambda_powertools/logging/formatters/datadog.py diff --git a/aws_lambda_powertools/logging/formatter.py b/aws_lambda_powertools/logging/formatter.py index 93e91e0dc8d..db80876c798 100644 --- a/aws_lambda_powertools/logging/formatter.py +++ b/aws_lambda_powertools/logging/formatter.py @@ -139,11 +139,11 @@ def __init__( if self.utc: self.converter = time.gmtime - super(LambdaPowertoolsFormatter, self).__init__(datefmt=self.datefmt) - self.keys_combined = {**self._build_default_keys(), **kwargs} self.log_format.update(**self.keys_combined) + super().__init__(datefmt=self.datefmt) + def serialize(self, log: Dict) -> str: """Serialize structured log dict to JSON str""" return self.json_serializer(log) diff --git a/aws_lambda_powertools/logging/formatters/__init__.py b/aws_lambda_powertools/logging/formatters/__init__.py new file mode 100644 index 00000000000..51b7b9e4d24 --- /dev/null +++ b/aws_lambda_powertools/logging/formatters/__init__.py @@ -0,0 +1,3 @@ +from aws_lambda_powertools.logging.formatters.datadog import DatadogLogFormatter + +__all__ = ["DatadogLogFormatter"] diff --git a/aws_lambda_powertools/logging/formatters/datadog.py b/aws_lambda_powertools/logging/formatters/datadog.py new file mode 100644 index 00000000000..fcfd06e0510 --- /dev/null +++ b/aws_lambda_powertools/logging/formatters/datadog.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from typing import Any, Callable + +from aws_lambda_powertools.logging.formatter import LambdaPowertoolsFormatter + + +class DatadogLogFormatter(LambdaPowertoolsFormatter): + def __init__( + self, + json_serializer: Callable[[dict], str] | None = None, + json_deserializer: Callable[[dict | str | bool | int | float], str] | None = None, + json_default: Callable[[Any], Any] | None = None, + datefmt: str | None = None, + use_datetime_directive: bool = False, + log_record_order: list[str] | None = None, + utc: bool = False, + use_rfc3339: bool = True, + **kwargs, + ): + super().__init__( + json_serializer, + json_deserializer, + json_default, + datefmt, + use_datetime_directive, + log_record_order, + utc, + use_rfc3339, + **kwargs, + ) diff --git a/tests/functional/test_logger_powertools_formatter.py b/tests/functional/test_logger_powertools_formatter.py index 7276f49d487..2ad4748fd0b 100644 --- a/tests/functional/test_logger_powertools_formatter.py +++ b/tests/functional/test_logger_powertools_formatter.py @@ -3,12 +3,14 @@ import json import os import random +import re import string import time import pytest from aws_lambda_powertools import Logger +from aws_lambda_powertools.logging.formatters import DatadogLogFormatter @pytest.fixture @@ -22,6 +24,10 @@ def service_name(): return "".join(random.SystemRandom().choice(chars) for _ in range(15)) +def capture_logging_output(stdout): + return json.loads(stdout.getvalue().strip()) + + @pytest.mark.parametrize("level", ["DEBUG", "WARNING", "ERROR", "INFO", "CRITICAL"]) def test_setup_with_valid_log_levels(stdout, level, service_name): logger = Logger(service=service_name, level=level, stream=stdout, request_id="request id!", another="value") @@ -309,3 +315,17 @@ def test_log_json_pretty_indent(stdout, service_name, monkeypatch): # THEN the json should contain more than line new_lines = stdout.getvalue().count(os.linesep) assert new_lines > 1 + + +def test_datadog_formatter_use_rfc3339_date(stdout, service_name): + # GIVEN Datadog Log Formatter is used + logger = Logger(service=service_name, stream=stdout, logger_formatter=DatadogLogFormatter()) + RFC3339_REGEX = r"^((?:(\d{4}-\d{2}-\d{2})T(\d{2}:\d{2}:\d{2}(?:\.\d+)?))(Z|[\+-]\d{2}:\d{2})?)$" + + # WHEN a log statement happens + logger.info({}) + + # THEN the timestamp uses RFC3339 by default + log = capture_logging_output(stdout) + + assert re.fullmatch(RFC3339_REGEX, log["timestamp"]) # "2022-10-27T17:42:26.841+0200" From 97965da2e11b06ff09ae550e019b39f1324a7c63 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 1 May 2023 14:53:51 +0200 Subject: [PATCH 2/6] docs(logger): new observability provider section --- docs/core/logger.md | 20 +++++++++++++++++++ ...servability_provider_builtin_formatters.py | 5 +++++ 2 files changed, 25 insertions(+) create mode 100644 examples/logger/src/observability_provider_builtin_formatters.py diff --git a/docs/core/logger.md b/docs/core/logger.md index 8bccdafeec3..2f0472368c3 100644 --- a/docs/core/logger.md +++ b/docs/core/logger.md @@ -445,6 +445,26 @@ If you prefer configuring it separately, or you'd want to bring this JSON Format --8<-- "examples/logger/src/powertools_formatter_setup.py" ``` +### Observability providers + +!!! note "In this context, an observability provider is an [AWS Lambda Partner](https://go.aws/3HtU6CZ){target="_blank"} offering a platform for logging, metrics, traces, etc." + +You can send logs to the observability provider of your choice via [Lambda Extensions](https://aws.amazon.com/blogs/compute/using-aws-lambda-extensions-to-send-logs-to-custom-destinations/){target="_blank"}. In most cases, you shouldn't need any custom Logger configuration, and logs will be shipped async without any performance impact. + +#### Built-in formatters + +In rare circumstances where JSON logs are not parsed correctly by your provider, we offer built-in formatters to make this transition easier. + +| Provider | Formatter | Notes | +| -------- | --------------------- | ---------------------------------------------------- | +| Datadog | `DatadogLogFormatter` | Modifies default timestamp to use RFC3339 by default | + +You can use import and use them as any other Logger formatter via `logger_formatter` parameter: + +```python hl_lines="2 4" title="Using built-in Logger Formatters" +--8<-- "examples/logger/src/observability_provider_builtin_formatters.py" +``` + ### Migrating from other Loggers If you're migrating from other Loggers, there are few key points to be aware of: [Service parameter](#the-service-parameter), [Inheriting Loggers](#inheriting-loggers), [Overriding Log records](#overriding-log-records), and [Logging exceptions](#logging-exceptions). diff --git a/examples/logger/src/observability_provider_builtin_formatters.py b/examples/logger/src/observability_provider_builtin_formatters.py new file mode 100644 index 00000000000..aa8c178a083 --- /dev/null +++ b/examples/logger/src/observability_provider_builtin_formatters.py @@ -0,0 +1,5 @@ +from aws_lambda_powertools import Logger +from aws_lambda_powertools.logging.formatters import DatadogLogFormatter + +logger = Logger(service="payment", logger_formatter=DatadogLogFormatter()) +logger.info("hello") From e3b4e5a7d12a9dc821f8244afa6c5b23264c3a03 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 1 May 2023 15:01:32 +0200 Subject: [PATCH 3/6] chore: remove barrel import to avoid Tracer-like situations --- aws_lambda_powertools/logging/formatters/__init__.py | 6 ++++-- .../logger/src/observability_provider_builtin_formatters.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/aws_lambda_powertools/logging/formatters/__init__.py b/aws_lambda_powertools/logging/formatters/__init__.py index 51b7b9e4d24..b6974414f4c 100644 --- a/aws_lambda_powertools/logging/formatters/__init__.py +++ b/aws_lambda_powertools/logging/formatters/__init__.py @@ -1,3 +1,5 @@ -from aws_lambda_powertools.logging.formatters.datadog import DatadogLogFormatter +"""Built-in Logger formatters for Observability Providers that require custom config.""" -__all__ = ["DatadogLogFormatter"] +# NOTE: we don't expose formatters directly (barrel import) +# as we cannot know if they'll need additional dependencies in the future +# so we isolate to avoid a performance hit and workarounds like lazy imports diff --git a/examples/logger/src/observability_provider_builtin_formatters.py b/examples/logger/src/observability_provider_builtin_formatters.py index aa8c178a083..3817f1f1b55 100644 --- a/examples/logger/src/observability_provider_builtin_formatters.py +++ b/examples/logger/src/observability_provider_builtin_formatters.py @@ -1,5 +1,5 @@ from aws_lambda_powertools import Logger -from aws_lambda_powertools.logging.formatters import DatadogLogFormatter +from aws_lambda_powertools.logging.formatters.datadog import DatadogLogFormatter logger = Logger(service="payment", logger_formatter=DatadogLogFormatter()) logger.info("hello") From ad667d5964e685a49ddd20d5b7a2c99b6a501da2 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 1 May 2023 15:10:44 +0200 Subject: [PATCH 4/6] fix: correct test import after refactor --- tests/functional/test_logger_powertools_formatter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/test_logger_powertools_formatter.py b/tests/functional/test_logger_powertools_formatter.py index 2ad4748fd0b..8b874894e27 100644 --- a/tests/functional/test_logger_powertools_formatter.py +++ b/tests/functional/test_logger_powertools_formatter.py @@ -10,7 +10,7 @@ import pytest from aws_lambda_powertools import Logger -from aws_lambda_powertools.logging.formatters import DatadogLogFormatter +from aws_lambda_powertools.logging.formatters.datadog import DatadogLogFormatter @pytest.fixture From dd58bb5182c72328fa1a7c67ae43ed35f244d518 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 1 May 2023 15:38:00 +0200 Subject: [PATCH 5/6] chore: add docstring and note on field change --- .../logging/formatters/datadog.py | 48 ++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/aws_lambda_powertools/logging/formatters/datadog.py b/aws_lambda_powertools/logging/formatters/datadog.py index fcfd06e0510..db09d04b83d 100644 --- a/aws_lambda_powertools/logging/formatters/datadog.py +++ b/aws_lambda_powertools/logging/formatters/datadog.py @@ -15,9 +15,55 @@ def __init__( use_datetime_directive: bool = False, log_record_order: list[str] | None = None, utc: bool = False, - use_rfc3339: bool = True, + use_rfc3339: bool = True, # NOTE: The only change from our base formatter **kwargs, ): + """Datadog formatter to comply with Datadog log parsing + + Changes compared to the default Logger Formatter: + + - timestamp format to use RFC3339 e.g., "2023-05-01T15:34:26.841+0200" + + + Parameters + ---------- + log_record_order : list[str] | None, optional + _description_, by default None + + Parameters + ---------- + json_serializer : Callable, optional + function to serialize `obj` to a JSON formatted `str`, by default json.dumps + json_deserializer : Callable, optional + function to deserialize `str`, `bytes`, bytearray` containing a JSON document to a Python `obj`, + by default json.loads + json_default : Callable, optional + function to coerce unserializable values, by default str + + Only used when no custom JSON encoder is set + + datefmt : str, optional + String directives (strftime) to format log timestamp. + + See https://docs.python.org/3/library/time.html#time.strftime or + use_datetime_directive: str, optional + Interpret `datefmt` as a format string for `datetime.datetime.strftime`, rather than + `time.strftime` - Only useful when used alongside `datefmt`. + + See https://docs.python.org/3/library/datetime.html#strftime-strptime-behavior . This + also supports a custom %F directive for milliseconds. + + log_record_order : list, optional + set order of log keys when logging, by default ["level", "location", "message", "timestamp"] + + utc : bool, optional + set logging timestamp to UTC, by default False to continue to use local time as per stdlib + use_rfc3339: bool, optional + Whether to use a popular dateformat that complies with both RFC3339 and ISO8601. + e.g., 2022-10-27T16:27:43.738+02:00. + kwargs + Key-value to persist in all log messages + """ super().__init__( json_serializer, json_deserializer, From 3b2fc9a2fdda103f997e35f8aa5971a6de430267 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 1 May 2023 15:39:20 +0200 Subject: [PATCH 6/6] chore: use keyword-arg in super to ease future refactoring --- .../logging/formatters/datadog.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/aws_lambda_powertools/logging/formatters/datadog.py b/aws_lambda_powertools/logging/formatters/datadog.py index db09d04b83d..fa92bf74598 100644 --- a/aws_lambda_powertools/logging/formatters/datadog.py +++ b/aws_lambda_powertools/logging/formatters/datadog.py @@ -65,13 +65,13 @@ def __init__( Key-value to persist in all log messages """ super().__init__( - json_serializer, - json_deserializer, - json_default, - datefmt, - use_datetime_directive, - log_record_order, - utc, - use_rfc3339, + json_serializer=json_serializer, + json_deserializer=json_deserializer, + json_default=json_default, + datefmt=datefmt, + use_datetime_directive=use_datetime_directive, + log_record_order=log_record_order, + utc=utc, + use_rfc3339=use_rfc3339, **kwargs, )