diff --git a/docs/index.md b/docs/index.md index 65bfb58c..63459ea5 100644 --- a/docs/index.md +++ b/docs/index.md @@ -371,6 +371,39 @@ print(Settings().model_dump()) #> {'numbers': [1, 2, 3]} ``` +## Nested model default partial updates + +By default, Pydantic settings does not allow partial updates to nested model default objects. This behavior can be +overriden by setting the `nested_model_default_partial_update` flag to `True`, which will allow partial updates on +nested model default object fields. + +```py +import os + +from pydantic import BaseModel + +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class SubModel(BaseModel): + val: int = 0 + flag: bool = False + + +class Settings(BaseSettings): + model_config = SettingsConfigDict( + env_nested_delimiter='__', nested_model_default_partial_update=True + ) + + nested_model: SubModel = SubModel() + + +# Apply a partial update to the default object using environment variables +os.environ['NESTED_MODEL__FLAG'] = 'True' + +assert Settings().model_dump() == {'nested_model': {'val': 0, 'flag': True}} +``` + ## Dotenv (.env) support Dotenv files (generally named `.env`) are a common pattern that make it easy to use environment variables in a @@ -474,7 +507,8 @@ models. There are two primary use cases for Pydantic settings CLI: By default, the experience is tailored towards use case #1 and builds on the foundations established in [parsing environment variables](#parsing-environment-variable-values). If your use case primarily falls into #2, you will likely -want to enable [enforcing required arguments at the CLI](#enforce-required-arguments-at-cli). +want to enable [enforcing required arguments at the CLI](#enforce-required-arguments-at-cli) and [nested model default +partial updates](#nested-model-default-partial-updates). ### The Basics diff --git a/pydantic_settings/main.py b/pydantic_settings/main.py index 5433442c..33ea245c 100644 --- a/pydantic_settings/main.py +++ b/pydantic_settings/main.py @@ -11,6 +11,7 @@ from .sources import ( ENV_FILE_SENTINEL, CliSettingsSource, + DefaultSettingsSource, DotEnvSettingsSource, DotenvType, EnvSettingsSource, @@ -23,6 +24,7 @@ class SettingsConfigDict(ConfigDict, total=False): case_sensitive: bool + nested_model_default_partial_update: bool | None env_prefix: str env_file: DotenvType | None env_file_encoding: str | None @@ -89,6 +91,8 @@ class BaseSettings(BaseModel): Args: _case_sensitive: Whether environment variables names should be read with case-sensitivity. Defaults to `None`. + _nested_model_default_partial_update: Whether to allow partial updates on nested model default object fields. + Defaults to `False`. _env_prefix: Prefix for all environment variables. Defaults to `None`. _env_file: The env file(s) to load settings values from. Defaults to `Path('')`, which means that the value from `model_config['env_file']` should be used. You can also pass @@ -123,6 +127,7 @@ class BaseSettings(BaseModel): def __init__( __pydantic_self__, _case_sensitive: bool | None = None, + _nested_model_default_partial_update: bool | None = None, _env_prefix: str | None = None, _env_file: DotenvType | None = ENV_FILE_SENTINEL, _env_file_encoding: str | None = None, @@ -149,6 +154,7 @@ def __init__( **__pydantic_self__._settings_build_values( values, _case_sensitive=_case_sensitive, + _nested_model_default_partial_update=_nested_model_default_partial_update, _env_prefix=_env_prefix, _env_file=_env_file, _env_file_encoding=_env_file_encoding, @@ -199,6 +205,7 @@ def _settings_build_values( self, init_kwargs: dict[str, Any], _case_sensitive: bool | None = None, + _nested_model_default_partial_update: bool | None = None, _env_prefix: str | None = None, _env_file: DotenvType | None = None, _env_file_encoding: str | None = None, @@ -222,6 +229,11 @@ def _settings_build_values( # Determine settings config values case_sensitive = _case_sensitive if _case_sensitive is not None else self.model_config.get('case_sensitive') env_prefix = _env_prefix if _env_prefix is not None else self.model_config.get('env_prefix') + nested_model_default_partial_update = ( + _nested_model_default_partial_update + if _nested_model_default_partial_update is not None + else self.model_config.get('nested_model_default_partial_update') + ) env_file = _env_file if _env_file != ENV_FILE_SENTINEL else self.model_config.get('env_file') env_file_encoding = ( _env_file_encoding if _env_file_encoding is not None else self.model_config.get('env_file_encoding') @@ -273,7 +285,14 @@ def _settings_build_values( secrets_dir = _secrets_dir if _secrets_dir is not None else self.model_config.get('secrets_dir') # Configure built-in sources - init_settings = InitSettingsSource(self.__class__, init_kwargs=init_kwargs) + default_settings = DefaultSettingsSource( + self.__class__, nested_model_default_partial_update=nested_model_default_partial_update + ) + init_settings = InitSettingsSource( + self.__class__, + init_kwargs=init_kwargs, + nested_model_default_partial_update=nested_model_default_partial_update, + ) env_settings = EnvSettingsSource( self.__class__, case_sensitive=case_sensitive, @@ -305,7 +324,7 @@ def _settings_build_values( env_settings=env_settings, dotenv_settings=dotenv_settings, file_secret_settings=file_secret_settings, - ) + ) + (default_settings,) if not any([source for source in sources if isinstance(source, CliSettingsSource)]): if cli_parse_args is not None or cli_settings_source is not None: cli_settings = ( @@ -352,6 +371,7 @@ def _settings_build_values( validate_default=True, case_sensitive=False, env_prefix='', + nested_model_default_partial_update=False, env_file=None, env_file_encoding=None, env_ignore_empty=False, diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index bbdeda07..29ef583a 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -13,7 +13,7 @@ from argparse import BooleanOptionalAction from argparse import SUPPRESS, ArgumentParser, Namespace, RawDescriptionHelpFormatter, _SubParsersAction from collections import deque -from dataclasses import is_dataclass +from dataclasses import asdict, is_dataclass from enum import Enum from pathlib import Path from textwrap import dedent @@ -22,6 +22,7 @@ TYPE_CHECKING, Any, Callable, + Dict, Generic, Iterator, List, @@ -38,8 +39,9 @@ import typing_extensions from dotenv import dotenv_values -from pydantic import AliasChoices, AliasPath, BaseModel, Json, RootModel +from pydantic import AliasChoices, AliasPath, BaseModel, Json, RootModel, TypeAdapter from pydantic._internal._repr import Representation +from pydantic._internal._signature import _field_name_for_signature from pydantic._internal._typing_extra import WithArgsTypes, origin_is_union, typing_base from pydantic._internal._utils import deep_update, is_model_class, lenient_issubclass from pydantic.dataclasses import is_pydantic_dataclass @@ -261,21 +263,71 @@ def __call__(self) -> dict[str, Any]: pass +class DefaultSettingsSource(PydanticBaseSettingsSource): + """ + Source class for loading default object values. + + Args: + settings_cls: The Settings class. + nested_model_default_partial_update: Whether to allow partial updates on nested model default object fields. + Defaults to `False`. + """ + + def __init__(self, settings_cls: type[BaseSettings], nested_model_default_partial_update: bool | None = None): + super().__init__(settings_cls) + self.defaults: dict[str, Any] = {} + self.nested_model_default_partial_update = ( + nested_model_default_partial_update + if nested_model_default_partial_update is not None + else self.config.get('nested_model_default_partial_update', False) + ) + if self.nested_model_default_partial_update: + for field_name, field_info in settings_cls.model_fields.items(): + if is_dataclass(type(field_info.default)): + self.defaults[_field_name_for_signature(field_name, field_info)] = asdict(field_info.default) + elif is_model_class(type(field_info.default)): + self.defaults[_field_name_for_signature(field_name, field_info)] = field_info.default.model_dump() + + def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]: + # Nothing to do here. Only implement the return statement to make mypy happy + return None, '', False + + def __call__(self) -> dict[str, Any]: + return self.defaults + + def __repr__(self) -> str: + return f'DefaultSettingsSource(nested_model_default_partial_update={self.nested_model_default_partial_update})' + + class InitSettingsSource(PydanticBaseSettingsSource): """ Source class for loading values provided during settings class initialization. """ - def __init__(self, settings_cls: type[BaseSettings], init_kwargs: dict[str, Any]): + def __init__( + self, + settings_cls: type[BaseSettings], + init_kwargs: dict[str, Any], + nested_model_default_partial_update: bool | None = None, + ): self.init_kwargs = init_kwargs super().__init__(settings_cls) + self.nested_model_default_partial_update = ( + nested_model_default_partial_update + if nested_model_default_partial_update is not None + else self.config.get('nested_model_default_partial_update', False) + ) def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]: # Nothing to do here. Only implement the return statement to make mypy happy return None, '', False def __call__(self) -> dict[str, Any]: - return self.init_kwargs + return ( + TypeAdapter(Dict[str, Any]).dump_python(self.init_kwargs) + if self.nested_model_default_partial_update + else self.init_kwargs + ) def __repr__(self) -> str: return f'InitSettingsSource(init_kwargs={self.init_kwargs!r})' @@ -1581,7 +1633,7 @@ def _add_parser_submodels( if self.cli_use_class_docs_for_groups and len(sub_models) == 1: model_group_kwargs['description'] = None if sub_models[0].__doc__ is None else dedent(sub_models[0].__doc__) - if model_default not in (PydanticUndefined, None): + if model_default is not PydanticUndefined: if is_model_class(type(model_default)) or is_pydantic_dataclass(type(model_default)): model_default = getattr(model_default, field_name) else: diff --git a/tests/test_settings.py b/tests/test_settings.py index 1e6d8767..b653e947 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -56,6 +56,7 @@ CliPositionalArg, CliSettingsSource, CliSubCommand, + DefaultSettingsSource, SettingsError, ) @@ -499,6 +500,80 @@ class ComplexSettings(BaseSettings): ] +def test_class_nested_model_default_partial_update(env): + class NestedA(BaseModel): + v0: bool + v1: bool + + @pydantic_dataclasses.dataclass + class NestedB: + v0: bool + v1: bool + + @dataclasses.dataclass + class NestedC: + v0: bool + v1: bool + + class NestedD(BaseModel): + v0: bool = False + v1: bool = True + + class SettingsDefaultsA(BaseSettings, env_nested_delimiter='__', nested_model_default_partial_update=True): + nested_a: NestedA = NestedA(v0=False, v1=True) + nested_b: NestedB = NestedB(v0=False, v1=True) + nested_d: NestedC = NestedC(v0=False, v1=True) + nested_c: NestedD = NestedD() + + env.set('NESTED_A__V0', 'True') + env.set('NESTED_B__V0', 'True') + env.set('NESTED_C__V0', 'True') + env.set('NESTED_D__V0', 'True') + assert SettingsDefaultsA().model_dump() == { + 'nested_a': {'v0': True, 'v1': True}, + 'nested_b': {'v0': True, 'v1': True}, + 'nested_c': {'v0': True, 'v1': True}, + 'nested_d': {'v0': True, 'v1': True}, + } + + +def test_init_kwargs_nested_model_default_partial_update(env): + class DeepSubModel(BaseModel): + v4: str + + class SubModel(BaseModel): + v1: str + v2: bytes + v3: int + deep: DeepSubModel + + class Settings(BaseSettings, env_nested_delimiter='__', nested_model_default_partial_update=True): + v0: str + sub_model: SubModel + + @classmethod + def settings_customise_sources( + cls, settings_cls, init_settings, env_settings, dotenv_settings, file_secret_settings + ): + return env_settings, dotenv_settings, init_settings, file_secret_settings + + env.set('SUB_MODEL__DEEP__V4', 'override-v4') + + s_final = {'v0': '0', 'sub_model': {'v1': 'init-v1', 'v2': b'init-v2', 'v3': 3, 'deep': {'v4': 'override-v4'}}} + + s = Settings(v0='0', sub_model={'v1': 'init-v1', 'v2': b'init-v2', 'v3': 3, 'deep': {'v4': 'init-v4'}}) + assert s.model_dump() == s_final + + s = Settings(v0='0', sub_model=SubModel(v1='init-v1', v2=b'init-v2', v3=3, deep=DeepSubModel(v4='init-v4'))) + assert s.model_dump() == s_final + + s = Settings(v0='0', sub_model=SubModel(v1='init-v1', v2=b'init-v2', v3=3, deep={'v4': 'init-v4'})) + assert s.model_dump() == s_final + + s = Settings(v0='0', sub_model={'v1': 'init-v1', 'v2': b'init-v2', 'v3': 3, 'deep': DeepSubModel(v4='init-v4')}) + assert s.model_dump() == s_final + + def test_env_str(env): class Settings(BaseSettings): apple: str = Field(None, validation_alias='BOOM') @@ -1575,6 +1650,10 @@ def settings_customise_sources(cls, *args, **kwargs): def test_builtins_settings_source_repr(): + assert ( + repr(DefaultSettingsSource(BaseSettings, nested_model_default_partial_update=True)) + == 'DefaultSettingsSource(nested_model_default_partial_update=True)' + ) assert ( repr(InitSettingsSource(BaseSettings, init_kwargs={'apple': 'value 0', 'banana': 'value 1'})) == "InitSettingsSource(init_kwargs={'apple': 'value 0', 'banana': 'value 1'})"