From 868de96286cca0f004e97e2dedf47cef9154902b Mon Sep 17 00:00:00 2001 From: Ruben Suarez Alvarez Date: Thu, 20 Mar 2025 14:26:24 +0100 Subject: [PATCH 1/7] feat: :sparkles: support DictConfigurator prefixes for rename_fields and static_fields --- src/pythonjsonlogger/core.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/pythonjsonlogger/core.py b/src/pythonjsonlogger/core.py index 1a4dee3..375384b 100644 --- a/src/pythonjsonlogger/core.py +++ b/src/pythonjsonlogger/core.py @@ -215,9 +215,17 @@ def __init__( ## JSON Logging specific ## --------------------------------------------------------------------- self.prefix = prefix - self.rename_fields = rename_fields if rename_fields is not None else {} + self.rename_fields = ( + {k: rename_fields[k] for k, v in rename_fields.items()} + if rename_fields is not None + else {} + ) self.rename_fields_keep_missing = rename_fields_keep_missing - self.static_fields = static_fields if static_fields is not None else {} + self.static_fields = ( + {k: static_fields[k] for k, v in static_fields.items()} + if static_fields is not None + else {} + ) self.reserved_attrs = set(reserved_attrs if reserved_attrs is not None else RESERVED_ATTRS) self.timestamp = timestamp From 3d968b27ae9434ffedc64682b11954d4784eb630 Mon Sep 17 00:00:00 2001 From: Ruben Suarez Alvarez Date: Fri, 21 Mar 2025 07:26:13 +0100 Subject: [PATCH 2/7] refactor: :recycle: simplify dict recreation (since the value is not used) --- src/pythonjsonlogger/core.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/pythonjsonlogger/core.py b/src/pythonjsonlogger/core.py index 375384b..f591222 100644 --- a/src/pythonjsonlogger/core.py +++ b/src/pythonjsonlogger/core.py @@ -215,16 +215,16 @@ def __init__( ## JSON Logging specific ## --------------------------------------------------------------------- self.prefix = prefix + # We recreate the dict to support internal/external reference which require getting the item to do the conversion. + # More details: https://github.com/nhairs/python-json-logger/pull/45 self.rename_fields = ( - {k: rename_fields[k] for k, v in rename_fields.items()} - if rename_fields is not None - else {} + {key: rename_fields[key] for key in rename_fields} if rename_fields is not None else {} ) self.rename_fields_keep_missing = rename_fields_keep_missing + # We recreate the dict to support internal/external reference which require getting the item to do the conversion. + # More details: https://github.com/nhairs/python-json-logger/pull/45 self.static_fields = ( - {k: static_fields[k] for k, v in static_fields.items()} - if static_fields is not None - else {} + {key: static_fields[key] for key in static_fields} if static_fields is not None else {} ) self.reserved_attrs = set(reserved_attrs if reserved_attrs is not None else RESERVED_ATTRS) self.timestamp = timestamp From 65f31c5cc942d86151613a23eba4ed87d921bc17 Mon Sep 17 00:00:00 2001 From: Ruben Suarez Alvarez Date: Fri, 21 Mar 2025 07:28:33 +0100 Subject: [PATCH 3/7] docs: :memo: replace the ini example with yaml + dictConfig example --- docs/cookbook.md | 72 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 49 insertions(+), 23 deletions(-) diff --git a/docs/cookbook.md b/docs/cookbook.md index 11edc2a..fdbb880 100644 --- a/docs/cookbook.md +++ b/docs/cookbook.md @@ -139,35 +139,61 @@ main_3() ## Using `fileConfig` -To use the module with a config file using the [`fileConfig` function](https://docs.python.org/3/library/logging.config.html#logging.config.fileConfig), use the class `pythonjsonlogger.json.JsonFormatter`. Here is a sample config file. - -```ini -[loggers] -keys = root,custom +To use the module with a yaml config file using the [`fileConfig` function](https://docs.python.org/3/library/logging.config.html#logging.config.fileConfig), use the class `pythonjsonlogger.json.JsonFormatter`. Here is a sample config file. + +```yaml +version: 1 +disable_existing_loggers: False +formatters: + default: + "()": pythonjsonlogger.json.JsonFormatter + format: "%(asctime)s %(levelname)s %(name)s %(module)s %(funcName)s %(lineno)s %(message)s" + rename_fields: + "asctime": "timestamp" + "levelname": "status" + static_fields: + "service": ext://logging_config.PROJECT_NAME + "env": ext://logging_config.ENVIRONMENT + "version": ext://logging_config.PROJECT_VERSION + "app_log": "true" +handlers: + default: + formatter: default + class: logging.StreamHandler + stream: ext://sys.stderr + access: + formatter: default + class: logging.StreamHandler + stream: ext://sys.stdout +loggers: + uvicorn.error: + level: INFO + handlers: + - default + propagate: no + uvicorn.access: + level: INFO + handlers: + - access + propagate: no +``` -[logger_root] -handlers = +then, you can have following logging_config.py file for resolving external references (*service*, *env* and *version*) from project metadata or environment variables. -[logger_custom] -level = INFO -handlers = custom -qualname = custom +```python +import importlib.metadata +import os -[handlers] -keys = custom -[handler_custom] -class = StreamHandler -level = INFO -formatter = json -args = (sys.stdout,) +def get_version_metadata(): + # https://stackoverflow.com/a/78082532 + version = importlib.metadata.version(PROJECT_NAME) + return version -[formatters] -keys = json -[formatter_json] -format = %(message)s -class = pythonjsonlogger.jsonlogger.JsonFormatter +PROJECT_NAME = 'test-api' +PROJECT_VERSION = get_version_metadata() +ENVIRONMENT = os.environ.get('ENVIRONMENT', 'dev') ``` ## Logging Expensive to Compute Data From ec424aee2d02dbb07706333a42653a6318c33fe4 Mon Sep 17 00:00:00 2001 From: Ruben Suarez Alvarez Date: Fri, 21 Mar 2025 07:35:35 +0100 Subject: [PATCH 4/7] docs: :memo: update changelog --- docs/changelog.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/changelog.md b/docs/changelog.md index 8d01b63..9424e8a 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [UNRELEASED] + +### Added +- Support `DictConfigurator` prefixes for `rename_fields` and `static_fields`. [#45](https://github.com/nhairs/python-json-logger/pull/45) + + ## [3.3.0](https://github.com/nhairs/python-json-logger/compare/v3.2.1...v3.3.0) - 2025-03-06 ### Added From 22d0bb6f400a5c7eb2dba45a85c7b7accbe055e4 Mon Sep 17 00:00:00 2001 From: Ruben Suarez Alvarez Date: Fri, 21 Mar 2025 10:00:36 +0100 Subject: [PATCH 5/7] test: :white_check_mark: test dictConfig external reference support --- tests/test_dictconfig.py | 80 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 tests/test_dictconfig.py diff --git a/tests/test_dictconfig.py b/tests/test_dictconfig.py new file mode 100644 index 0000000..e956c03 --- /dev/null +++ b/tests/test_dictconfig.py @@ -0,0 +1,80 @@ +### IMPORTS +### ============================================================================ +## Future +from __future__ import annotations + +## Standard Library +from dataclasses import dataclass +import io +import json +import logging +import logging.config +from typing import Any, Generator + +## Installed +import pytest + +### SETUP +### ============================================================================ +_LOGGER_COUNT = 0 +EXT_VAL = 999 + +LOGGING_CONFIG = { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "default": { + "()": "pythonjsonlogger.json.JsonFormatter", + "static_fields": {"ext-val": "ext://tests.test_dictconfig.EXT_VAL"}, + } + }, + "handlers": { + "default": { + "level": "DEBUG", + "formatter": "default", + "class": "logging.StreamHandler", + "stream": "ext://sys.stdout", # Default is stderr + }, + }, + "loggers": { + "": {"handlers": ["default"], "level": "WARNING", "propagate": False}, # root logger + }, +} + + +@dataclass +class LoggingEnvironment: + logger: logging.Logger + buffer: io.StringIO + + def load_json(self) -> Any: + return json.loads(self.buffer.getvalue()) + + +@pytest.fixture +def env() -> Generator[LoggingEnvironment, None, None]: + global _LOGGER_COUNT # pylint: disable=global-statement + _LOGGER_COUNT += 1 + logging.config.dictConfig(LOGGING_CONFIG) + default_formatter = logging.root.handlers[0].formatter + logger = logging.getLogger(f"pythonjsonlogger.tests.{_LOGGER_COUNT}") + logger.setLevel(logging.DEBUG) + buffer = io.StringIO() + handler = logging.StreamHandler(buffer) + handler.setFormatter(default_formatter) + logger.addHandler(handler) + yield LoggingEnvironment(logger=logger, buffer=buffer) + logger.removeHandler(handler) + logger.setLevel(logging.NOTSET) + buffer.close() + return + + +### TESTS +### ============================================================================ +def test_external_reference_support(env: LoggingEnvironment): + env.logger.info("hello") + log_json = env.load_json() + + assert log_json["ext-val"] == EXT_VAL + return From 871f837264010ab592842cddbb9f6d686b3ae7c9 Mon Sep 17 00:00:00 2001 From: Ruben Suarez Alvarez Date: Fri, 21 Mar 2025 10:43:20 +0100 Subject: [PATCH 6/7] chore: :bookmark: set version to 3.3.1.dev0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2381b3d..c91ca72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "python-json-logger" -version = "3.3.0" +version = "3.3.1.dev0" description = "JSON Log Formatter for the Python Logging Package" authors = [ {name = "Zakaria Zajac", email = "zak@madzak.com"}, From 05fb70a1bc7df29c385fd38ba9b317c5609814a2 Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Sun, 30 Mar 2025 16:35:20 +1100 Subject: [PATCH 7/7] Wording changes etc --- docs/changelog.md | 2 ++ docs/cookbook.md | 8 ++++---- src/pythonjsonlogger/core.py | 11 ++++++----- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 9424e8a..03a9b71 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -8,7 +8,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Support `DictConfigurator` prefixes for `rename_fields` and `static_fields`. [#45](https://github.com/nhairs/python-json-logger/pull/45) + - Allows using values like `ext://sys.stderr` in `fileConfig`/`dictConfig` value fields. +Thanks @rubensa ## [3.3.0](https://github.com/nhairs/python-json-logger/compare/v3.2.1...v3.3.0) - 2025-03-06 diff --git a/docs/cookbook.md b/docs/cookbook.md index fdbb880..c0755d5 100644 --- a/docs/cookbook.md +++ b/docs/cookbook.md @@ -139,9 +139,9 @@ main_3() ## Using `fileConfig` -To use the module with a yaml config file using the [`fileConfig` function](https://docs.python.org/3/library/logging.config.html#logging.config.fileConfig), use the class `pythonjsonlogger.json.JsonFormatter`. Here is a sample config file. +To use the module with a yaml config file using the [`fileConfig` function](https://docs.python.org/3/library/logging.config.html#logging.config.fileConfig), use the class `pythonjsonlogger.json.JsonFormatter`. Here is a sample config file: -```yaml +```yaml title="example_config.yaml" version: 1 disable_existing_loggers: False formatters: @@ -178,9 +178,9 @@ loggers: propagate: no ``` -then, you can have following logging_config.py file for resolving external references (*service*, *env* and *version*) from project metadata or environment variables. +You'll notice that we are using `ext://...` for the `static_fields`. This will load data from other modules such as the one below. -```python +```python title="logging_config.py" import importlib.metadata import os diff --git a/src/pythonjsonlogger/core.py b/src/pythonjsonlogger/core.py index f591222..a00510b 100644 --- a/src/pythonjsonlogger/core.py +++ b/src/pythonjsonlogger/core.py @@ -215,17 +215,18 @@ def __init__( ## JSON Logging specific ## --------------------------------------------------------------------- self.prefix = prefix - # We recreate the dict to support internal/external reference which require getting the item to do the conversion. - # More details: https://github.com/nhairs/python-json-logger/pull/45 + + # We recreate the dict in rename_fields and static_fields to support internal/external + # references which require getting the item to do the conversion. + # For more details see: https://github.com/nhairs/python-json-logger/pull/45 self.rename_fields = ( {key: rename_fields[key] for key in rename_fields} if rename_fields is not None else {} ) - self.rename_fields_keep_missing = rename_fields_keep_missing - # We recreate the dict to support internal/external reference which require getting the item to do the conversion. - # More details: https://github.com/nhairs/python-json-logger/pull/45 self.static_fields = ( {key: static_fields[key] for key in static_fields} if static_fields is not None else {} ) + + self.rename_fields_keep_missing = rename_fields_keep_missing self.reserved_attrs = set(reserved_attrs if reserved_attrs is not None else RESERVED_ATTRS) self.timestamp = timestamp