From 9e0a9440ca7ad0cc315709b0d28588e8f9a7ae4a Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Mon, 22 Jul 2024 09:10:57 -0600 Subject: [PATCH 01/12] Add default settings source. --- pydantic_settings/main.py | 4 ++- pydantic_settings/sources.py | 70 +++++++++++++++++++++++++++++++++++- tests/test_settings.py | 20 +++++++++++ 3 files changed, 92 insertions(+), 2 deletions(-) diff --git a/pydantic_settings/main.py b/pydantic_settings/main.py index 1dd4ac76..8524b20e 100644 --- a/pydantic_settings/main.py +++ b/pydantic_settings/main.py @@ -11,6 +11,7 @@ from .sources import ( ENV_FILE_SENTINEL, CliSettingsSource, + DefaultSettingsSource, DotEnvSettingsSource, DotenvType, EnvSettingsSource, @@ -264,6 +265,7 @@ def _settings_build_values( secrets_dir = _secrets_dir if _secrets_dir is not None else self.model_config.get('secrets_dir') # Configure built-in sources + default_settings = DefaultSettingsSource(self.__class__) init_settings = InitSettingsSource(self.__class__, init_kwargs=init_kwargs) env_settings = EnvSettingsSource( self.__class__, @@ -296,7 +298,7 @@ def _settings_build_values( env_settings=env_settings, dotenv_settings=dotenv_settings, file_secret_settings=file_secret_settings, - ) + ) + (default_settings,) if not any([source for source in sources if isinstance(source, CliSettingsSource)]): if cli_parse_args is not None or cli_settings_source is not None: cli_settings = ( diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index e62cfaa6..095d3a54 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -10,7 +10,7 @@ from abc import ABC, abstractmethod from argparse import SUPPRESS, ArgumentParser, HelpFormatter, Namespace, _SubParsersAction from collections import deque -from dataclasses import is_dataclass +from dataclasses import asdict, is_dataclass from enum import Enum from pathlib import Path from types import FunctionType @@ -246,6 +246,74 @@ def __call__(self) -> dict[str, Any]: pass +class DefaultSettingsSource(PydanticBaseSettingsSource): + """ + Source class for loading default values. + """ + + def __init__(self, settings_cls: type[BaseSettings]): + super().__init__(settings_cls) + self.defaults = self._get_defaults(settings_cls) + + def _get_defaults(self, settings_cls: type[BaseSettings]) -> dict[str, Any]: + defaults: dict[str, Any] = {} + if self.config.get('validate_default'): + fields = ( + settings_cls.__pydantic_fields__ if is_pydantic_dataclass(settings_cls) else settings_cls.model_fields + ) + for field_name, field_info in fields.items(): + if field_info.validate_default is not False: + resolved_name = self._get_resolved_name(field_name, field_info) + if field_info.default not in (PydanticUndefined, None): + if is_model_class(field_info.annotation): + defaults[resolved_name] = field_info.default.model_dump() + elif is_dataclass(field_info.annotation): + defaults[resolved_name] = asdict(field_info.default) + else: + defaults[resolved_name] = field_info.default + elif field_info.default_factory is not None: + defaults[resolved_name] = field_info.default_factory + return defaults + + def _get_resolved_name(self, field_name: str, field_info: FieldInfo) -> str: + if not any((field_info.alias, field_info.validation_alias)): + return field_name + + resolved_names: list[str] = [] + is_alias_path_only: bool = True + new_alias_paths: list[AliasPath] = [] + for alias in (field_info.alias, field_info.validation_alias): + if alias is None: + continue + elif isinstance(alias, str): + resolved_names.append(alias) + is_alias_path_only = False + elif isinstance(alias, AliasChoices): + for name in alias.choices: + if isinstance(name, str): + resolved_names.append(name) + is_alias_path_only = False + else: + new_alias_paths.append(name) + else: + new_alias_paths.append(alias) + for alias_path in new_alias_paths: + name = cast(str, alias_path.path[0]) + if not resolved_names and is_alias_path_only: + resolved_names.append(name) + return tuple(dict.fromkeys(resolved_names))[0] + + def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]: + # Nothing to do here. Only implement the return statement to make mypy happy + return None, '', False + + def __call__(self) -> dict[str, Any]: + return self.defaults + + def __repr__(self) -> str: + return f'DefaultSettingsSource(init_kwargs={self.defaults!r})' + + class InitSettingsSource(PydanticBaseSettingsSource): """ Source class for loading values provided during settings class initialization. diff --git a/tests/test_settings.py b/tests/test_settings.py index 7ead164d..61714956 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -491,6 +491,26 @@ class ComplexSettings(BaseSettings): ] +def test_env_default_settings(env): + class NestedA(BaseModel): + v0: bool + v1: bool + + class NestedB(BaseModel): + v0: bool = False + v1: bool = True + + class SettingsDefaultsA(BaseSettings, env_nested_delimiter='__'): + nested: NestedB = NestedB() + + class SettingsDefaultsB(BaseSettings, env_nested_delimiter='__'): + nested: NestedA = NestedA(v0=False, v1=True) + + env.set('NESTED__V0', 'True') + assert SettingsDefaultsA().model_dump() == {'nested': {'v0': True, 'v1': True}} + assert SettingsDefaultsB().model_dump() == {'nested': {'v0': True, 'v1': True}} + + def test_env_str(env): class Settings(BaseSettings): apple: str = Field(None, validation_alias='BOOM') From 585817610758616f1d3a8f8e4baf9c94ba94ec3b Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Mon, 22 Jul 2024 09:25:00 -0600 Subject: [PATCH 02/12] repr. --- pydantic_settings/sources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 095d3a54..f7a4b3b0 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -311,7 +311,7 @@ def __call__(self) -> dict[str, Any]: return self.defaults def __repr__(self) -> str: - return f'DefaultSettingsSource(init_kwargs={self.defaults!r})' + return f'DefaultSettingsSource()' class InitSettingsSource(PydanticBaseSettingsSource): From 0dd8d74230beaaf0e3e04de1746ef7d6efaa8edd Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Mon, 22 Jul 2024 09:35:18 -0600 Subject: [PATCH 03/12] Retain default typing. --- pydantic_settings/sources.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index f7a4b3b0..551949fe 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -10,7 +10,7 @@ from abc import ABC, abstractmethod from argparse import SUPPRESS, ArgumentParser, HelpFormatter, Namespace, _SubParsersAction from collections import deque -from dataclasses import asdict, is_dataclass +from dataclasses import is_dataclass from enum import Enum from pathlib import Path from types import FunctionType @@ -265,12 +265,7 @@ def _get_defaults(self, settings_cls: type[BaseSettings]) -> dict[str, Any]: if field_info.validate_default is not False: resolved_name = self._get_resolved_name(field_name, field_info) if field_info.default not in (PydanticUndefined, None): - if is_model_class(field_info.annotation): - defaults[resolved_name] = field_info.default.model_dump() - elif is_dataclass(field_info.annotation): - defaults[resolved_name] = asdict(field_info.default) - else: - defaults[resolved_name] = field_info.default + defaults[resolved_name] = field_info.default elif field_info.default_factory is not None: defaults[resolved_name] = field_info.default_factory return defaults @@ -311,7 +306,7 @@ def __call__(self) -> dict[str, Any]: return self.defaults def __repr__(self) -> str: - return f'DefaultSettingsSource()' + return 'DefaultSettingsSource()' class InitSettingsSource(PydanticBaseSettingsSource): From bcc628e32fed83b375977503d3081fd6666ff8d4 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Mon, 12 Aug 2024 08:48:11 -0600 Subject: [PATCH 04/12] Add default_objects_copy_by_value flag. --- pydantic_settings/main.py | 20 +++++++- pydantic_settings/sources.py | 91 ++++++++++++++++-------------------- tests/test_settings.py | 86 +++++++++++++++++++++++++++++----- 3 files changed, 132 insertions(+), 65 deletions(-) diff --git a/pydantic_settings/main.py b/pydantic_settings/main.py index 8524b20e..b3c789b7 100644 --- a/pydantic_settings/main.py +++ b/pydantic_settings/main.py @@ -24,6 +24,7 @@ class SettingsConfigDict(ConfigDict, total=False): case_sensitive: bool + default_objects_copy_by_value: bool | None env_prefix: str env_file: DotenvType | None env_file_encoding: str | None @@ -89,6 +90,8 @@ class BaseSettings(BaseModel): Args: _case_sensitive: Whether environment variables names should be read with case-sensitivity. Defaults to `None`. + _default_objects_copy_by_value: Whether default `BaseModel` objects should use copy-by-value instead of + copy-by-reference when compiling sources. Defaults to `False`. _env_prefix: Prefix for all environment variables. Defaults to `None`. _env_file: The env file(s) to load settings values from. Defaults to `Path('')`, which means that the value from `model_config['env_file']` should be used. You can also pass @@ -121,6 +124,7 @@ class BaseSettings(BaseModel): def __init__( __pydantic_self__, _case_sensitive: bool | None = None, + _default_objects_copy_by_value: bool | None = None, _env_prefix: str | None = None, _env_file: DotenvType | None = ENV_FILE_SENTINEL, _env_file_encoding: str | None = None, @@ -146,6 +150,7 @@ def __init__( **__pydantic_self__._settings_build_values( values, _case_sensitive=_case_sensitive, + _default_objects_copy_by_value=_default_objects_copy_by_value, _env_prefix=_env_prefix, _env_file=_env_file, _env_file_encoding=_env_file_encoding, @@ -195,6 +200,7 @@ def _settings_build_values( self, init_kwargs: dict[str, Any], _case_sensitive: bool | None = None, + _default_objects_copy_by_value: bool | None = None, _env_prefix: str | None = None, _env_file: DotenvType | None = None, _env_file_encoding: str | None = None, @@ -217,6 +223,11 @@ def _settings_build_values( # Determine settings config values case_sensitive = _case_sensitive if _case_sensitive is not None else self.model_config.get('case_sensitive') env_prefix = _env_prefix if _env_prefix is not None else self.model_config.get('env_prefix') + default_objects_copy_by_value = ( + _default_objects_copy_by_value + if _default_objects_copy_by_value is not None + else self.model_config.get('default_objects_copy_by_value') + ) env_file = _env_file if _env_file != ENV_FILE_SENTINEL else self.model_config.get('env_file') env_file_encoding = ( _env_file_encoding if _env_file_encoding is not None else self.model_config.get('env_file_encoding') @@ -265,8 +276,12 @@ def _settings_build_values( secrets_dir = _secrets_dir if _secrets_dir is not None else self.model_config.get('secrets_dir') # Configure built-in sources - default_settings = DefaultSettingsSource(self.__class__) - init_settings = InitSettingsSource(self.__class__, init_kwargs=init_kwargs) + default_settings = DefaultSettingsSource( + self.__class__, default_objects_copy_by_value=default_objects_copy_by_value + ) + init_settings = InitSettingsSource( + self.__class__, init_kwargs=init_kwargs, default_objects_copy_by_value=default_objects_copy_by_value + ) env_settings = EnvSettingsSource( self.__class__, case_sensitive=case_sensitive, @@ -344,6 +359,7 @@ def _settings_build_values( validate_default=True, case_sensitive=False, env_prefix='', + default_objects_copy_by_value=False, env_file=None, env_file_encoding=None, env_ignore_empty=False, diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 551949fe..a0351cd6 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -10,7 +10,7 @@ from abc import ABC, abstractmethod from argparse import SUPPRESS, ArgumentParser, HelpFormatter, Namespace, _SubParsersAction from collections import deque -from dataclasses import is_dataclass +from dataclasses import asdict, is_dataclass from enum import Enum from pathlib import Path from types import FunctionType @@ -18,6 +18,7 @@ TYPE_CHECKING, Any, Callable, + Dict, Generic, Iterator, List, @@ -34,8 +35,9 @@ import typing_extensions from dotenv import dotenv_values -from pydantic import AliasChoices, AliasPath, BaseModel, Json +from pydantic import AliasChoices, AliasPath, BaseModel, Json, TypeAdapter from pydantic._internal._repr import Representation +from pydantic._internal._signature import _field_name_for_signature from pydantic._internal._typing_extra import WithArgsTypes, origin_is_union, typing_base from pydantic._internal._utils import deep_update, is_model_class, lenient_issubclass from pydantic.dataclasses import is_pydantic_dataclass @@ -248,55 +250,23 @@ def __call__(self) -> dict[str, Any]: class DefaultSettingsSource(PydanticBaseSettingsSource): """ - Source class for loading default values. + Source class for loading default object values. """ - def __init__(self, settings_cls: type[BaseSettings]): + def __init__(self, settings_cls: type[BaseSettings], default_objects_copy_by_value: bool | None = None): super().__init__(settings_cls) - self.defaults = self._get_defaults(settings_cls) - - def _get_defaults(self, settings_cls: type[BaseSettings]) -> dict[str, Any]: - defaults: dict[str, Any] = {} - if self.config.get('validate_default'): - fields = ( - settings_cls.__pydantic_fields__ if is_pydantic_dataclass(settings_cls) else settings_cls.model_fields - ) - for field_name, field_info in fields.items(): - if field_info.validate_default is not False: - resolved_name = self._get_resolved_name(field_name, field_info) - if field_info.default not in (PydanticUndefined, None): - defaults[resolved_name] = field_info.default - elif field_info.default_factory is not None: - defaults[resolved_name] = field_info.default_factory - return defaults - - def _get_resolved_name(self, field_name: str, field_info: FieldInfo) -> str: - if not any((field_info.alias, field_info.validation_alias)): - return field_name - - resolved_names: list[str] = [] - is_alias_path_only: bool = True - new_alias_paths: list[AliasPath] = [] - for alias in (field_info.alias, field_info.validation_alias): - if alias is None: - continue - elif isinstance(alias, str): - resolved_names.append(alias) - is_alias_path_only = False - elif isinstance(alias, AliasChoices): - for name in alias.choices: - if isinstance(name, str): - resolved_names.append(name) - is_alias_path_only = False - else: - new_alias_paths.append(name) - else: - new_alias_paths.append(alias) - for alias_path in new_alias_paths: - name = cast(str, alias_path.path[0]) - if not resolved_names and is_alias_path_only: - resolved_names.append(name) - return tuple(dict.fromkeys(resolved_names))[0] + self.defaults: dict[str, Any] = {} + self.default_objects_copy_by_value = ( + default_objects_copy_by_value + if default_objects_copy_by_value is not None + else self.config.get('default_objects_copy_by_value', False) + ) + if self.default_objects_copy_by_value: + for field_name, field_info in settings_cls.model_fields.items(): + if is_dataclass(type(field_info.default)): + self.defaults[_field_name_for_signature(field_name, field_info)] = asdict(field_info.default) + elif is_model_class(type(field_info.default)): + self.defaults[_field_name_for_signature(field_name, field_info)] = field_info.default.model_dump() def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]: # Nothing to do here. Only implement the return statement to make mypy happy @@ -306,7 +276,7 @@ def __call__(self) -> dict[str, Any]: return self.defaults def __repr__(self) -> str: - return 'DefaultSettingsSource()' + return f'DefaultSettingsSource(default_objects_copy_by_value={self.default_objects_copy_by_value})' class InitSettingsSource(PydanticBaseSettingsSource): @@ -314,19 +284,36 @@ class InitSettingsSource(PydanticBaseSettingsSource): Source class for loading values provided during settings class initialization. """ - def __init__(self, settings_cls: type[BaseSettings], init_kwargs: dict[str, Any]): + def __init__( + self, + settings_cls: type[BaseSettings], + init_kwargs: dict[str, Any], + default_objects_copy_by_value: bool | None = None, + ): self.init_kwargs = init_kwargs super().__init__(settings_cls) + self.default_objects_copy_by_value = ( + default_objects_copy_by_value + if default_objects_copy_by_value is not None + else self.config.get('default_objects_copy_by_value', False) + ) def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]: # Nothing to do here. Only implement the return statement to make mypy happy return None, '', False def __call__(self) -> dict[str, Any]: - return self.init_kwargs + return ( + TypeAdapter(Dict[str, Any]).dump_python(self.init_kwargs) + if self.default_objects_copy_by_value + else self.init_kwargs + ) def __repr__(self) -> str: - return f'InitSettingsSource(init_kwargs={self.init_kwargs!r})' + return ( + f'InitSettingsSource(init_kwargs={self.init_kwargs!r}, ' + f'default_objects_copy_by_value={self.default_objects_copy_by_value})' + ) class PydanticBaseEnvSettingsSource(PydanticBaseSettingsSource): diff --git a/tests/test_settings.py b/tests/test_settings.py index 61714956..8f7dddef 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -49,7 +49,13 @@ TomlConfigSettingsSource, YamlConfigSettingsSource, ) -from pydantic_settings.sources import CliPositionalArg, CliSettingsSource, CliSubCommand, SettingsError +from pydantic_settings.sources import ( + CliPositionalArg, + CliSettingsSource, + CliSubCommand, + DefaultSettingsSource, + SettingsError, +) try: import dotenv @@ -491,24 +497,78 @@ class ComplexSettings(BaseSettings): ] -def test_env_default_settings(env): +def test_class_default_objects_copy_by_value(env): class NestedA(BaseModel): v0: bool v1: bool - class NestedB(BaseModel): + @pydantic_dataclasses.dataclass + class NestedB: + v0: bool + v1: bool + + @dataclasses.dataclass + class NestedC: + v0: bool + v1: bool + + class NestedD(BaseModel): v0: bool = False v1: bool = True - class SettingsDefaultsA(BaseSettings, env_nested_delimiter='__'): - nested: NestedB = NestedB() + class SettingsDefaultsA(BaseSettings, env_nested_delimiter='__', default_objects_copy_by_value=True): + nested_a: NestedA = NestedA(v0=False, v1=True) + nested_b: NestedB = NestedB(v0=False, v1=True) + nested_d: NestedC = NestedC(v0=False, v1=True) + nested_c: NestedD = NestedD() + + env.set('NESTED_A__V0', 'True') + env.set('NESTED_B__V0', 'True') + env.set('NESTED_C__V0', 'True') + env.set('NESTED_D__V0', 'True') + assert SettingsDefaultsA().model_dump() == { + 'nested_a': {'v0': True, 'v1': True}, + 'nested_b': {'v0': True, 'v1': True}, + 'nested_c': {'v0': True, 'v1': True}, + 'nested_d': {'v0': True, 'v1': True}, + } - class SettingsDefaultsB(BaseSettings, env_nested_delimiter='__'): - nested: NestedA = NestedA(v0=False, v1=True) - env.set('NESTED__V0', 'True') - assert SettingsDefaultsA().model_dump() == {'nested': {'v0': True, 'v1': True}} - assert SettingsDefaultsB().model_dump() == {'nested': {'v0': True, 'v1': True}} +def test_init_kwargs_default_objects_copy_by_value(env): + class DeepSubModel(BaseModel): + v4: str + + class SubModel(BaseModel): + v1: str + v2: bytes + v3: int + deep: DeepSubModel + + class Settings(BaseSettings, env_nested_delimiter='__', default_objects_copy_by_value=True): + v0: str + sub_model: SubModel + + @classmethod + def settings_customise_sources( + cls, settings_cls, init_settings, env_settings, dotenv_settings, file_secret_settings + ): + return env_settings, dotenv_settings, init_settings, file_secret_settings + + env.set('SUB_MODEL__DEEP__V4', 'override-v4') + + s_final = {'v0': '0', 'sub_model': {'v1': 'init-v1', 'v2': b'init-v2', 'v3': 3, 'deep': {'v4': 'override-v4'}}} + + s = Settings(v0='0', sub_model={'v1': 'init-v1', 'v2': b'init-v2', 'v3': 3, 'deep': {'v4': 'init-v4'}}) + assert s.model_dump() == s_final + + s = Settings(v0='0', sub_model=SubModel(v1='init-v1', v2=b'init-v2', v3=3, deep=DeepSubModel(v4='init-v4'))) + assert s.model_dump() == s_final + + s = Settings(v0='0', sub_model=SubModel(v1='init-v1', v2=b'init-v2', v3=3, deep={'v4': 'init-v4'})) + assert s.model_dump() == s_final + + s = Settings(v0='0', sub_model={'v1': 'init-v1', 'v2': b'init-v2', 'v3': 3, 'deep': DeepSubModel(v4='init-v4')}) + assert s.model_dump() == s_final def test_env_str(env): @@ -1569,9 +1629,13 @@ def settings_customise_sources(cls, *args, **kwargs): def test_builtins_settings_source_repr(): + assert ( + repr(DefaultSettingsSource(BaseSettings, default_objects_copy_by_value=True)) + == 'DefaultSettingsSource(default_objects_copy_by_value=True)' + ) assert ( repr(InitSettingsSource(BaseSettings, init_kwargs={'apple': 'value 0', 'banana': 'value 1'})) - == "InitSettingsSource(init_kwargs={'apple': 'value 0', 'banana': 'value 1'})" + == "InitSettingsSource(init_kwargs={'apple': 'value 0', 'banana': 'value 1'}, default_objects_copy_by_value=False)" ) assert ( repr(EnvSettingsSource(BaseSettings, env_nested_delimiter='__')) From 01c1057fe7a9c93f4d8d5ad9d84bc9a37e03f592 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Mon, 12 Aug 2024 08:56:14 -0600 Subject: [PATCH 05/12] Propegate None defaults. --- pydantic_settings/sources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index ae8b9bad..7b028722 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -1571,7 +1571,7 @@ def _add_parser_submodels( if self.cli_use_class_docs_for_groups and len(sub_models) == 1: model_group_kwargs['description'] = None if sub_models[0].__doc__ is None else dedent(sub_models[0].__doc__) - if model_default not in (PydanticUndefined, None): + if model_default is not PydanticUndefined: if is_model_class(type(model_default)) or is_pydantic_dataclass(type(model_default)): model_default = getattr(model_default, field_name) else: From e0f3feb9c6b3fb7259b2cdd5f447878edf4317f8 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Mon, 12 Aug 2024 21:06:15 -0600 Subject: [PATCH 06/12] Doc updates. --- docs/index.md | 66 ++++++++++++++++++++++++++++++++++++ pydantic_settings/sources.py | 14 ++++---- 2 files changed, 74 insertions(+), 6 deletions(-) diff --git a/docs/index.md b/docs/index.md index 85d059d5..56ed9609 100644 --- a/docs/index.md +++ b/docs/index.md @@ -464,6 +464,72 @@ class Settings(BaseSettings): So if you provide extra values in a dotenv file, whether they start with `env_prefix` or not, a `ValidationError` will be raised. +## Parsing default objects + +Pydantic settings uses copy-by-reference when merging default (`BaseModel`) objects from different sources. This ensures +the original object reference is maintained in the final object instantiation. However, due to an internal limitation, +it is not possible to partially update a nested sub model field in a default object when using copy-by-reference; the +entirety of the sub model must be provided. + +This behavior can be overriden by setting the `default_objects_copy_by_value` flag to `True`, which will allow partial +updates to sub model fields. Note of course the original default object reference will not be retained. + +```py +import os + +from pydantic import BaseModel, ValidationError + +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class SubModel(BaseModel): + val: int = 0 + flag: bool = False + + +ORIGINAL_OBJECT = SubModel() + + +class SettingsCopyByReference(BaseSettings): + model_config = SettingsConfigDict(env_nested_delimiter='__') + + default_object: SubModel = ORIGINAL_OBJECT + + +class SettingsCopyByValue(BaseSettings): + model_config = SettingsConfigDict( + env_nested_delimiter='__', default_objects_copy_by_value=True + ) + + default_object: SubModel = ORIGINAL_OBJECT + + +by_ref = SettingsCopyByReference() +assert by_ref.default_object is ORIGINAL_OBJECT + +# Apply a partial update to the default object using environment variables +os.environ['DEFAULT_OBJECT__FLAG'] = 'True' + +try: + # Copy by reference will fail + SettingsCopyByReference() +except ValidationError as err: + print(err) + """ + 1 validation error for SettingsCopyByReference + nested.val + Field required [type=missing, input_value={'flag': 'TRUE'}, input_type=dict] + For further information visit https://errors.pydantic.dev/2/v/missing + """ + +# Copy by value will pass +by_val = SettingsCopyByValue() +assert by_val.default_object is not ORIGINAL_OBJECT + +print(by_val.model_dump()) +#> {'default_object': {'val': 0, 'flag': True}} +``` + ## Command Line Support Pydantic settings provides integrated CLI support, making it easy to quickly define CLI applications using Pydantic diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 7b028722..3a5ff449 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -262,12 +262,14 @@ def __init__(self, settings_cls: type[BaseSettings], default_objects_copy_by_val if default_objects_copy_by_value is not None else self.config.get('default_objects_copy_by_value', False) ) - if self.default_objects_copy_by_value: - for field_name, field_info in settings_cls.model_fields.items(): - if is_dataclass(type(field_info.default)): - self.defaults[_field_name_for_signature(field_name, field_info)] = asdict(field_info.default) - elif is_model_class(type(field_info.default)): - self.defaults[_field_name_for_signature(field_name, field_info)] = field_info.default.model_dump() + for field_name, field_info in settings_cls.model_fields.items(): + if not self.default_objects_copy_by_value: + if is_dataclass(type(field_info.default)) or is_model_class(type(field_info.default)): + self.defaults[_field_name_for_signature(field_name, field_info)] = field_info.default + elif is_dataclass(type(field_info.default)): + self.defaults[_field_name_for_signature(field_name, field_info)] = asdict(field_info.default) + elif is_model_class(type(field_info.default)): + self.defaults[_field_name_for_signature(field_name, field_info)] = field_info.default.model_dump() def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]: # Nothing to do here. Only implement the return statement to make mypy happy From f812cb6e058b31a8702cd0fd512d07bc4873fc79 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Mon, 12 Aug 2024 21:14:09 -0600 Subject: [PATCH 07/12] Move placement, add note in CLI doc. --- docs/index.md | 135 +++++++++++++++++++++++++------------------------- 1 file changed, 68 insertions(+), 67 deletions(-) diff --git a/docs/index.md b/docs/index.md index 56ed9609..1606c5fc 100644 --- a/docs/index.md +++ b/docs/index.md @@ -371,6 +371,72 @@ print(Settings().model_dump()) #> {'numbers': [1, 2, 3]} ``` +## Parsing default objects + +Pydantic settings uses copy-by-reference when merging default (`BaseModel`) objects from different sources. This ensures +the original object reference is maintained in the final object instantiation. However, due to an internal limitation, +it is not possible to partially update a nested sub model field in a default object when using copy-by-reference; the +entirety of the sub model must be provided. + +This behavior can be overriden by setting the `default_objects_copy_by_value` flag to `True`, which will allow partial +updates to sub model fields. Note of course the original default object reference will not be retained. + +```py +import os + +from pydantic import BaseModel, ValidationError + +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class SubModel(BaseModel): + val: int = 0 + flag: bool = False + + +ORIGINAL_OBJECT = SubModel() + + +class SettingsCopyByReference(BaseSettings): + model_config = SettingsConfigDict(env_nested_delimiter='__') + + default_object: SubModel = ORIGINAL_OBJECT + + +class SettingsCopyByValue(BaseSettings): + model_config = SettingsConfigDict( + env_nested_delimiter='__', default_objects_copy_by_value=True + ) + + default_object: SubModel = ORIGINAL_OBJECT + + +by_ref = SettingsCopyByReference() +assert by_ref.default_object is ORIGINAL_OBJECT + +# Apply a partial update to the default object using environment variables +os.environ['DEFAULT_OBJECT__FLAG'] = 'True' + +try: + # Copy by reference will fail + SettingsCopyByReference() +except ValidationError as err: + print(err) + """ + 1 validation error for SettingsCopyByReference + nested.val + Field required [type=missing, input_value={'flag': 'TRUE'}, input_type=dict] + For further information visit https://errors.pydantic.dev/2/v/missing + """ + +# Copy by value will pass +by_val = SettingsCopyByValue() +assert by_val.default_object is not ORIGINAL_OBJECT + +print(by_val.model_dump()) +#> {'default_object': {'val': 0, 'flag': True}} +``` + ## Dotenv (.env) support Dotenv files (generally named `.env`) are a common pattern that make it easy to use environment variables in a @@ -464,72 +530,6 @@ class Settings(BaseSettings): So if you provide extra values in a dotenv file, whether they start with `env_prefix` or not, a `ValidationError` will be raised. -## Parsing default objects - -Pydantic settings uses copy-by-reference when merging default (`BaseModel`) objects from different sources. This ensures -the original object reference is maintained in the final object instantiation. However, due to an internal limitation, -it is not possible to partially update a nested sub model field in a default object when using copy-by-reference; the -entirety of the sub model must be provided. - -This behavior can be overriden by setting the `default_objects_copy_by_value` flag to `True`, which will allow partial -updates to sub model fields. Note of course the original default object reference will not be retained. - -```py -import os - -from pydantic import BaseModel, ValidationError - -from pydantic_settings import BaseSettings, SettingsConfigDict - - -class SubModel(BaseModel): - val: int = 0 - flag: bool = False - - -ORIGINAL_OBJECT = SubModel() - - -class SettingsCopyByReference(BaseSettings): - model_config = SettingsConfigDict(env_nested_delimiter='__') - - default_object: SubModel = ORIGINAL_OBJECT - - -class SettingsCopyByValue(BaseSettings): - model_config = SettingsConfigDict( - env_nested_delimiter='__', default_objects_copy_by_value=True - ) - - default_object: SubModel = ORIGINAL_OBJECT - - -by_ref = SettingsCopyByReference() -assert by_ref.default_object is ORIGINAL_OBJECT - -# Apply a partial update to the default object using environment variables -os.environ['DEFAULT_OBJECT__FLAG'] = 'True' - -try: - # Copy by reference will fail - SettingsCopyByReference() -except ValidationError as err: - print(err) - """ - 1 validation error for SettingsCopyByReference - nested.val - Field required [type=missing, input_value={'flag': 'TRUE'}, input_type=dict] - For further information visit https://errors.pydantic.dev/2/v/missing - """ - -# Copy by value will pass -by_val = SettingsCopyByValue() -assert by_val.default_object is not ORIGINAL_OBJECT - -print(by_val.model_dump()) -#> {'default_object': {'val': 0, 'flag': True}} -``` - ## Command Line Support Pydantic settings provides integrated CLI support, making it easy to quickly define CLI applications using Pydantic @@ -540,7 +540,8 @@ models. There are two primary use cases for Pydantic settings CLI: By default, the experience is tailored towards use case #1 and builds on the foundations established in [parsing environment variables](#parsing-environment-variable-values). If your use case primarily falls into #2, you will likely -want to enable [enforcing required arguments at the CLI](#enforce-required-arguments-at-cli). +want to enable [enforcing required arguments at the CLI](#enforce-required-arguments-at-cli) and copy-by-value when +[parsing default objects](#parsing-default-objects). ### The Basics From 31ea380c784e40447accefc2ca6898647cba8383 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Mon, 12 Aug 2024 21:19:45 -0600 Subject: [PATCH 08/12] Remove parens. --- docs/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index 1606c5fc..9d85b863 100644 --- a/docs/index.md +++ b/docs/index.md @@ -373,7 +373,7 @@ print(Settings().model_dump()) ## Parsing default objects -Pydantic settings uses copy-by-reference when merging default (`BaseModel`) objects from different sources. This ensures +Pydantic settings uses copy-by-reference when merging default `BaseModel` objects from different sources. This ensures the original object reference is maintained in the final object instantiation. However, due to an internal limitation, it is not possible to partially update a nested sub model field in a default object when using copy-by-reference; the entirety of the sub model must be provided. From 3fd2cc2d60aeb718009d26356e52a57fc2be2305 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Tue, 13 Aug 2024 07:22:08 -0600 Subject: [PATCH 09/12] Review updates. --- docs/index.md | 20 +++++++------------- pydantic_settings/sources.py | 19 +++++++------------ tests/test_settings.py | 2 +- 3 files changed, 15 insertions(+), 26 deletions(-) diff --git a/docs/index.md b/docs/index.md index 9d85b863..c63ace1e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -394,13 +394,10 @@ class SubModel(BaseModel): flag: bool = False -ORIGINAL_OBJECT = SubModel() - - class SettingsCopyByReference(BaseSettings): model_config = SettingsConfigDict(env_nested_delimiter='__') - default_object: SubModel = ORIGINAL_OBJECT + default_object: SubModel = SubModel() class SettingsCopyByValue(BaseSettings): @@ -408,18 +405,18 @@ class SettingsCopyByValue(BaseSettings): env_nested_delimiter='__', default_objects_copy_by_value=True ) - default_object: SubModel = ORIGINAL_OBJECT + default_object: SubModel = SubModel() -by_ref = SettingsCopyByReference() -assert by_ref.default_object is ORIGINAL_OBJECT +s = SettingsCopyByReference() +assert s.model_dump() == {'default_object': {'val': 0, 'flag': False}} # Apply a partial update to the default object using environment variables os.environ['DEFAULT_OBJECT__FLAG'] = 'True' try: # Copy by reference will fail - SettingsCopyByReference() + s = SettingsCopyByReference() except ValidationError as err: print(err) """ @@ -430,11 +427,8 @@ except ValidationError as err: """ # Copy by value will pass -by_val = SettingsCopyByValue() -assert by_val.default_object is not ORIGINAL_OBJECT - -print(by_val.model_dump()) -#> {'default_object': {'val': 0, 'flag': True}} +s = SettingsCopyByValue() +assert s.model_dump() == {'default_object': {'val': 0, 'flag': True}} ``` ## Dotenv (.env) support diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 8aa40294..de81fa80 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -262,14 +262,12 @@ def __init__(self, settings_cls: type[BaseSettings], default_objects_copy_by_val if default_objects_copy_by_value is not None else self.config.get('default_objects_copy_by_value', False) ) - for field_name, field_info in settings_cls.model_fields.items(): - if not self.default_objects_copy_by_value: - if is_dataclass(type(field_info.default)) or is_model_class(type(field_info.default)): - self.defaults[_field_name_for_signature(field_name, field_info)] = field_info.default - elif is_dataclass(type(field_info.default)): - self.defaults[_field_name_for_signature(field_name, field_info)] = asdict(field_info.default) - elif is_model_class(type(field_info.default)): - self.defaults[_field_name_for_signature(field_name, field_info)] = field_info.default.model_dump() + if self.default_objects_copy_by_value: + for field_name, field_info in settings_cls.model_fields.items(): + if is_dataclass(type(field_info.default)): + self.defaults[_field_name_for_signature(field_name, field_info)] = asdict(field_info.default) + elif is_model_class(type(field_info.default)): + self.defaults[_field_name_for_signature(field_name, field_info)] = field_info.default.model_dump() def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]: # Nothing to do here. Only implement the return statement to make mypy happy @@ -313,10 +311,7 @@ def __call__(self) -> dict[str, Any]: ) def __repr__(self) -> str: - return ( - f'InitSettingsSource(init_kwargs={self.init_kwargs!r}, ' - f'default_objects_copy_by_value={self.default_objects_copy_by_value})' - ) + return f'InitSettingsSource(init_kwargs={self.init_kwargs!r})' class PydanticBaseEnvSettingsSource(PydanticBaseSettingsSource): diff --git a/tests/test_settings.py b/tests/test_settings.py index df435df4..97cd900e 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -1654,7 +1654,7 @@ def test_builtins_settings_source_repr(): ) assert ( repr(InitSettingsSource(BaseSettings, init_kwargs={'apple': 'value 0', 'banana': 'value 1'})) - == "InitSettingsSource(init_kwargs={'apple': 'value 0', 'banana': 'value 1'}, default_objects_copy_by_value=False)" + == "InitSettingsSource(init_kwargs={'apple': 'value 0', 'banana': 'value 1'})" ) assert ( repr(EnvSettingsSource(BaseSettings, env_nested_delimiter='__')) From 2b269277067c7c37cb3b95045ffde99e11d04bdf Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Tue, 13 Aug 2024 09:28:20 -0600 Subject: [PATCH 10/12] Review updates. --- docs/index.md | 46 +++++++----------------------------- pydantic_settings/main.py | 25 ++++++++++---------- pydantic_settings/sources.py | 26 ++++++++++---------- tests/test_settings.py | 12 +++++----- 4 files changed, 40 insertions(+), 69 deletions(-) diff --git a/docs/index.md b/docs/index.md index c63ace1e..7bd5e833 100644 --- a/docs/index.md +++ b/docs/index.md @@ -371,20 +371,15 @@ print(Settings().model_dump()) #> {'numbers': [1, 2, 3]} ``` -## Parsing default objects +## Nested model partial updates -Pydantic settings uses copy-by-reference when merging default `BaseModel` objects from different sources. This ensures -the original object reference is maintained in the final object instantiation. However, due to an internal limitation, -it is not possible to partially update a nested sub model field in a default object when using copy-by-reference; the -entirety of the sub model must be provided. - -This behavior can be overriden by setting the `default_objects_copy_by_value` flag to `True`, which will allow partial -updates to sub model fields. Note of course the original default object reference will not be retained. +By default, Pydantic settings does not allow partial updates to nested models. This behavior can be overriden by setting +the `nested_model_partial_update` flag to `True`, which will allow partial updates on nested model fields. ```py import os -from pydantic import BaseModel, ValidationError +from pydantic import BaseModel from pydantic_settings import BaseSettings, SettingsConfigDict @@ -394,41 +389,18 @@ class SubModel(BaseModel): flag: bool = False -class SettingsCopyByReference(BaseSettings): - model_config = SettingsConfigDict(env_nested_delimiter='__') - - default_object: SubModel = SubModel() - - -class SettingsCopyByValue(BaseSettings): +class Settings(BaseSettings): model_config = SettingsConfigDict( - env_nested_delimiter='__', default_objects_copy_by_value=True + env_nested_delimiter='__', nested_model_partial_update=True ) - default_object: SubModel = SubModel() - + nested_model: SubModel = SubModel() -s = SettingsCopyByReference() -assert s.model_dump() == {'default_object': {'val': 0, 'flag': False}} # Apply a partial update to the default object using environment variables -os.environ['DEFAULT_OBJECT__FLAG'] = 'True' - -try: - # Copy by reference will fail - s = SettingsCopyByReference() -except ValidationError as err: - print(err) - """ - 1 validation error for SettingsCopyByReference - nested.val - Field required [type=missing, input_value={'flag': 'TRUE'}, input_type=dict] - For further information visit https://errors.pydantic.dev/2/v/missing - """ +os.environ['NESTED_MODEL__FLAG'] = 'True' -# Copy by value will pass -s = SettingsCopyByValue() -assert s.model_dump() == {'default_object': {'val': 0, 'flag': True}} +assert Settings().model_dump() == {'nested_model': {'val': 0, 'flag': True}} ``` ## Dotenv (.env) support diff --git a/pydantic_settings/main.py b/pydantic_settings/main.py index b3c789b7..79abe2dc 100644 --- a/pydantic_settings/main.py +++ b/pydantic_settings/main.py @@ -24,7 +24,7 @@ class SettingsConfigDict(ConfigDict, total=False): case_sensitive: bool - default_objects_copy_by_value: bool | None + nested_model_partial_update: bool | None env_prefix: str env_file: DotenvType | None env_file_encoding: str | None @@ -90,8 +90,7 @@ class BaseSettings(BaseModel): Args: _case_sensitive: Whether environment variables names should be read with case-sensitivity. Defaults to `None`. - _default_objects_copy_by_value: Whether default `BaseModel` objects should use copy-by-value instead of - copy-by-reference when compiling sources. Defaults to `False`. + _nested_model_partial_update: Whether to allow partial updates on nested model fields. Defaults to `False`. _env_prefix: Prefix for all environment variables. Defaults to `None`. _env_file: The env file(s) to load settings values from. Defaults to `Path('')`, which means that the value from `model_config['env_file']` should be used. You can also pass @@ -124,7 +123,7 @@ class BaseSettings(BaseModel): def __init__( __pydantic_self__, _case_sensitive: bool | None = None, - _default_objects_copy_by_value: bool | None = None, + _nested_model_partial_update: bool | None = None, _env_prefix: str | None = None, _env_file: DotenvType | None = ENV_FILE_SENTINEL, _env_file_encoding: str | None = None, @@ -150,7 +149,7 @@ def __init__( **__pydantic_self__._settings_build_values( values, _case_sensitive=_case_sensitive, - _default_objects_copy_by_value=_default_objects_copy_by_value, + _nested_model_partial_update=_nested_model_partial_update, _env_prefix=_env_prefix, _env_file=_env_file, _env_file_encoding=_env_file_encoding, @@ -200,7 +199,7 @@ def _settings_build_values( self, init_kwargs: dict[str, Any], _case_sensitive: bool | None = None, - _default_objects_copy_by_value: bool | None = None, + _nested_model_partial_update: bool | None = None, _env_prefix: str | None = None, _env_file: DotenvType | None = None, _env_file_encoding: str | None = None, @@ -223,10 +222,10 @@ def _settings_build_values( # Determine settings config values case_sensitive = _case_sensitive if _case_sensitive is not None else self.model_config.get('case_sensitive') env_prefix = _env_prefix if _env_prefix is not None else self.model_config.get('env_prefix') - default_objects_copy_by_value = ( - _default_objects_copy_by_value - if _default_objects_copy_by_value is not None - else self.model_config.get('default_objects_copy_by_value') + nested_model_partial_update = ( + _nested_model_partial_update + if _nested_model_partial_update is not None + else self.model_config.get('nested_model_partial_update') ) env_file = _env_file if _env_file != ENV_FILE_SENTINEL else self.model_config.get('env_file') env_file_encoding = ( @@ -277,10 +276,10 @@ def _settings_build_values( # Configure built-in sources default_settings = DefaultSettingsSource( - self.__class__, default_objects_copy_by_value=default_objects_copy_by_value + self.__class__, nested_model_partial_update=nested_model_partial_update ) init_settings = InitSettingsSource( - self.__class__, init_kwargs=init_kwargs, default_objects_copy_by_value=default_objects_copy_by_value + self.__class__, init_kwargs=init_kwargs, nested_model_partial_update=nested_model_partial_update ) env_settings = EnvSettingsSource( self.__class__, @@ -359,7 +358,7 @@ def _settings_build_values( validate_default=True, case_sensitive=False, env_prefix='', - default_objects_copy_by_value=False, + nested_model_partial_update=False, env_file=None, env_file_encoding=None, env_ignore_empty=False, diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index de81fa80..92eaf5e7 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -254,15 +254,15 @@ class DefaultSettingsSource(PydanticBaseSettingsSource): Source class for loading default object values. """ - def __init__(self, settings_cls: type[BaseSettings], default_objects_copy_by_value: bool | None = None): + def __init__(self, settings_cls: type[BaseSettings], nested_model_partial_update: bool | None = None): super().__init__(settings_cls) self.defaults: dict[str, Any] = {} - self.default_objects_copy_by_value = ( - default_objects_copy_by_value - if default_objects_copy_by_value is not None - else self.config.get('default_objects_copy_by_value', False) + self.nested_model_partial_update = ( + nested_model_partial_update + if nested_model_partial_update is not None + else self.config.get('nested_model_partial_update', False) ) - if self.default_objects_copy_by_value: + if self.nested_model_partial_update: for field_name, field_info in settings_cls.model_fields.items(): if is_dataclass(type(field_info.default)): self.defaults[_field_name_for_signature(field_name, field_info)] = asdict(field_info.default) @@ -277,7 +277,7 @@ def __call__(self) -> dict[str, Any]: return self.defaults def __repr__(self) -> str: - return f'DefaultSettingsSource(default_objects_copy_by_value={self.default_objects_copy_by_value})' + return f'DefaultSettingsSource(nested_model_partial_update={self.nested_model_partial_update})' class InitSettingsSource(PydanticBaseSettingsSource): @@ -289,14 +289,14 @@ def __init__( self, settings_cls: type[BaseSettings], init_kwargs: dict[str, Any], - default_objects_copy_by_value: bool | None = None, + nested_model_partial_update: bool | None = None, ): self.init_kwargs = init_kwargs super().__init__(settings_cls) - self.default_objects_copy_by_value = ( - default_objects_copy_by_value - if default_objects_copy_by_value is not None - else self.config.get('default_objects_copy_by_value', False) + self.nested_model_partial_update = ( + nested_model_partial_update + if nested_model_partial_update is not None + else self.config.get('nested_model_partial_update', False) ) def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]: @@ -306,7 +306,7 @@ def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, def __call__(self) -> dict[str, Any]: return ( TypeAdapter(Dict[str, Any]).dump_python(self.init_kwargs) - if self.default_objects_copy_by_value + if self.nested_model_partial_update else self.init_kwargs ) diff --git a/tests/test_settings.py b/tests/test_settings.py index 97cd900e..ba91f278 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -498,7 +498,7 @@ class ComplexSettings(BaseSettings): ] -def test_class_default_objects_copy_by_value(env): +def test_class_nested_model_partial_update(env): class NestedA(BaseModel): v0: bool v1: bool @@ -517,7 +517,7 @@ class NestedD(BaseModel): v0: bool = False v1: bool = True - class SettingsDefaultsA(BaseSettings, env_nested_delimiter='__', default_objects_copy_by_value=True): + class SettingsDefaultsA(BaseSettings, env_nested_delimiter='__', nested_model_partial_update=True): nested_a: NestedA = NestedA(v0=False, v1=True) nested_b: NestedB = NestedB(v0=False, v1=True) nested_d: NestedC = NestedC(v0=False, v1=True) @@ -535,7 +535,7 @@ class SettingsDefaultsA(BaseSettings, env_nested_delimiter='__', default_objects } -def test_init_kwargs_default_objects_copy_by_value(env): +def test_init_kwargs_nested_model_partial_update(env): class DeepSubModel(BaseModel): v4: str @@ -545,7 +545,7 @@ class SubModel(BaseModel): v3: int deep: DeepSubModel - class Settings(BaseSettings, env_nested_delimiter='__', default_objects_copy_by_value=True): + class Settings(BaseSettings, env_nested_delimiter='__', nested_model_partial_update=True): v0: str sub_model: SubModel @@ -1649,8 +1649,8 @@ def settings_customise_sources(cls, *args, **kwargs): def test_builtins_settings_source_repr(): assert ( - repr(DefaultSettingsSource(BaseSettings, default_objects_copy_by_value=True)) - == 'DefaultSettingsSource(default_objects_copy_by_value=True)' + repr(DefaultSettingsSource(BaseSettings, nested_model_partial_update=True)) + == 'DefaultSettingsSource(nested_model_partial_update=True)' ) assert ( repr(InitSettingsSource(BaseSettings, init_kwargs={'apple': 'value 0', 'banana': 'value 1'})) From 2513ac2b9de87e9d4632c96814723b4238e30724 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Tue, 13 Aug 2024 09:30:56 -0600 Subject: [PATCH 11/12] CLI doc update. --- docs/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/index.md b/docs/index.md index 7bd5e833..377df7e0 100644 --- a/docs/index.md +++ b/docs/index.md @@ -506,8 +506,8 @@ models. There are two primary use cases for Pydantic settings CLI: By default, the experience is tailored towards use case #1 and builds on the foundations established in [parsing environment variables](#parsing-environment-variable-values). If your use case primarily falls into #2, you will likely -want to enable [enforcing required arguments at the CLI](#enforce-required-arguments-at-cli) and copy-by-value when -[parsing default objects](#parsing-default-objects). +want to enable [enforcing required arguments at the CLI](#enforce-required-arguments-at-cli) and [nested model partial +updates](#nested-model-partial-updates). ### The Basics From 93e8558c12e1e891e144879a4d54615270b74b1e Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Tue, 20 Aug 2024 06:42:26 -0600 Subject: [PATCH 12/12] Review updates. --- docs/index.md | 13 +++++++------ pydantic_settings/main.py | 27 +++++++++++++++------------ pydantic_settings/sources.py | 31 ++++++++++++++++++------------- tests/test_settings.py | 12 ++++++------ 4 files changed, 46 insertions(+), 37 deletions(-) diff --git a/docs/index.md b/docs/index.md index 377df7e0..00937258 100644 --- a/docs/index.md +++ b/docs/index.md @@ -371,10 +371,11 @@ print(Settings().model_dump()) #> {'numbers': [1, 2, 3]} ``` -## Nested model partial updates +## Nested model default partial updates -By default, Pydantic settings does not allow partial updates to nested models. This behavior can be overriden by setting -the `nested_model_partial_update` flag to `True`, which will allow partial updates on nested model fields. +By default, Pydantic settings does not allow partial updates to nested model default objects. This behavior can be +overriden by setting the `nested_model_default_partial_update` flag to `True`, which will allow partial updates on +nested model default object fields. ```py import os @@ -391,7 +392,7 @@ class SubModel(BaseModel): class Settings(BaseSettings): model_config = SettingsConfigDict( - env_nested_delimiter='__', nested_model_partial_update=True + env_nested_delimiter='__', nested_model_default_partial_update=True ) nested_model: SubModel = SubModel() @@ -506,8 +507,8 @@ models. There are two primary use cases for Pydantic settings CLI: By default, the experience is tailored towards use case #1 and builds on the foundations established in [parsing environment variables](#parsing-environment-variable-values). If your use case primarily falls into #2, you will likely -want to enable [enforcing required arguments at the CLI](#enforce-required-arguments-at-cli) and [nested model partial -updates](#nested-model-partial-updates). +want to enable [enforcing required arguments at the CLI](#enforce-required-arguments-at-cli) and [nested model default +partial updates](#nested-model-default-partial-updates). ### The Basics diff --git a/pydantic_settings/main.py b/pydantic_settings/main.py index 79abe2dc..66150336 100644 --- a/pydantic_settings/main.py +++ b/pydantic_settings/main.py @@ -24,7 +24,7 @@ class SettingsConfigDict(ConfigDict, total=False): case_sensitive: bool - nested_model_partial_update: bool | None + nested_model_default_partial_update: bool | None env_prefix: str env_file: DotenvType | None env_file_encoding: str | None @@ -90,7 +90,8 @@ class BaseSettings(BaseModel): Args: _case_sensitive: Whether environment variables names should be read with case-sensitivity. Defaults to `None`. - _nested_model_partial_update: Whether to allow partial updates on nested model fields. Defaults to `False`. + _nested_model_default_partial_update: Whether to allow partial updates on nested model default object fields. + Defaults to `False`. _env_prefix: Prefix for all environment variables. Defaults to `None`. _env_file: The env file(s) to load settings values from. Defaults to `Path('')`, which means that the value from `model_config['env_file']` should be used. You can also pass @@ -123,7 +124,7 @@ class BaseSettings(BaseModel): def __init__( __pydantic_self__, _case_sensitive: bool | None = None, - _nested_model_partial_update: bool | None = None, + _nested_model_default_partial_update: bool | None = None, _env_prefix: str | None = None, _env_file: DotenvType | None = ENV_FILE_SENTINEL, _env_file_encoding: str | None = None, @@ -149,7 +150,7 @@ def __init__( **__pydantic_self__._settings_build_values( values, _case_sensitive=_case_sensitive, - _nested_model_partial_update=_nested_model_partial_update, + _nested_model_default_partial_update=_nested_model_default_partial_update, _env_prefix=_env_prefix, _env_file=_env_file, _env_file_encoding=_env_file_encoding, @@ -199,7 +200,7 @@ def _settings_build_values( self, init_kwargs: dict[str, Any], _case_sensitive: bool | None = None, - _nested_model_partial_update: bool | None = None, + _nested_model_default_partial_update: bool | None = None, _env_prefix: str | None = None, _env_file: DotenvType | None = None, _env_file_encoding: str | None = None, @@ -222,10 +223,10 @@ def _settings_build_values( # Determine settings config values case_sensitive = _case_sensitive if _case_sensitive is not None else self.model_config.get('case_sensitive') env_prefix = _env_prefix if _env_prefix is not None else self.model_config.get('env_prefix') - nested_model_partial_update = ( - _nested_model_partial_update - if _nested_model_partial_update is not None - else self.model_config.get('nested_model_partial_update') + nested_model_default_partial_update = ( + _nested_model_default_partial_update + if _nested_model_default_partial_update is not None + else self.model_config.get('nested_model_default_partial_update') ) env_file = _env_file if _env_file != ENV_FILE_SENTINEL else self.model_config.get('env_file') env_file_encoding = ( @@ -276,10 +277,12 @@ def _settings_build_values( # Configure built-in sources default_settings = DefaultSettingsSource( - self.__class__, nested_model_partial_update=nested_model_partial_update + self.__class__, nested_model_default_partial_update=nested_model_default_partial_update ) init_settings = InitSettingsSource( - self.__class__, init_kwargs=init_kwargs, nested_model_partial_update=nested_model_partial_update + self.__class__, + init_kwargs=init_kwargs, + nested_model_default_partial_update=nested_model_default_partial_update, ) env_settings = EnvSettingsSource( self.__class__, @@ -358,7 +361,7 @@ def _settings_build_values( validate_default=True, case_sensitive=False, env_prefix='', - nested_model_partial_update=False, + nested_model_default_partial_update=False, env_file=None, env_file_encoding=None, env_ignore_empty=False, diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 92eaf5e7..12ecffbf 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -252,17 +252,22 @@ def __call__(self) -> dict[str, Any]: class DefaultSettingsSource(PydanticBaseSettingsSource): """ Source class for loading default object values. + + Args: + settings_cls: The Settings class. + nested_model_default_partial_update: Whether to allow partial updates on nested model default object fields. + Defaults to `False`. """ - def __init__(self, settings_cls: type[BaseSettings], nested_model_partial_update: bool | None = None): + def __init__(self, settings_cls: type[BaseSettings], nested_model_default_partial_update: bool | None = None): super().__init__(settings_cls) self.defaults: dict[str, Any] = {} - self.nested_model_partial_update = ( - nested_model_partial_update - if nested_model_partial_update is not None - else self.config.get('nested_model_partial_update', False) + self.nested_model_default_partial_update = ( + nested_model_default_partial_update + if nested_model_default_partial_update is not None + else self.config.get('nested_model_default_partial_update', False) ) - if self.nested_model_partial_update: + if self.nested_model_default_partial_update: for field_name, field_info in settings_cls.model_fields.items(): if is_dataclass(type(field_info.default)): self.defaults[_field_name_for_signature(field_name, field_info)] = asdict(field_info.default) @@ -277,7 +282,7 @@ def __call__(self) -> dict[str, Any]: return self.defaults def __repr__(self) -> str: - return f'DefaultSettingsSource(nested_model_partial_update={self.nested_model_partial_update})' + return f'DefaultSettingsSource(nested_model_default_partial_update={self.nested_model_default_partial_update})' class InitSettingsSource(PydanticBaseSettingsSource): @@ -289,14 +294,14 @@ def __init__( self, settings_cls: type[BaseSettings], init_kwargs: dict[str, Any], - nested_model_partial_update: bool | None = None, + nested_model_default_partial_update: bool | None = None, ): self.init_kwargs = init_kwargs super().__init__(settings_cls) - self.nested_model_partial_update = ( - nested_model_partial_update - if nested_model_partial_update is not None - else self.config.get('nested_model_partial_update', False) + self.nested_model_default_partial_update = ( + nested_model_default_partial_update + if nested_model_default_partial_update is not None + else self.config.get('nested_model_default_partial_update', False) ) def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]: @@ -306,7 +311,7 @@ def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, def __call__(self) -> dict[str, Any]: return ( TypeAdapter(Dict[str, Any]).dump_python(self.init_kwargs) - if self.nested_model_partial_update + if self.nested_model_default_partial_update else self.init_kwargs ) diff --git a/tests/test_settings.py b/tests/test_settings.py index ba91f278..c76deecf 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -498,7 +498,7 @@ class ComplexSettings(BaseSettings): ] -def test_class_nested_model_partial_update(env): +def test_class_nested_model_default_partial_update(env): class NestedA(BaseModel): v0: bool v1: bool @@ -517,7 +517,7 @@ class NestedD(BaseModel): v0: bool = False v1: bool = True - class SettingsDefaultsA(BaseSettings, env_nested_delimiter='__', nested_model_partial_update=True): + class SettingsDefaultsA(BaseSettings, env_nested_delimiter='__', nested_model_default_partial_update=True): nested_a: NestedA = NestedA(v0=False, v1=True) nested_b: NestedB = NestedB(v0=False, v1=True) nested_d: NestedC = NestedC(v0=False, v1=True) @@ -535,7 +535,7 @@ class SettingsDefaultsA(BaseSettings, env_nested_delimiter='__', nested_model_pa } -def test_init_kwargs_nested_model_partial_update(env): +def test_init_kwargs_nested_model_default_partial_update(env): class DeepSubModel(BaseModel): v4: str @@ -545,7 +545,7 @@ class SubModel(BaseModel): v3: int deep: DeepSubModel - class Settings(BaseSettings, env_nested_delimiter='__', nested_model_partial_update=True): + class Settings(BaseSettings, env_nested_delimiter='__', nested_model_default_partial_update=True): v0: str sub_model: SubModel @@ -1649,8 +1649,8 @@ def settings_customise_sources(cls, *args, **kwargs): def test_builtins_settings_source_repr(): assert ( - repr(DefaultSettingsSource(BaseSettings, nested_model_partial_update=True)) - == 'DefaultSettingsSource(nested_model_partial_update=True)' + repr(DefaultSettingsSource(BaseSettings, nested_model_default_partial_update=True)) + == 'DefaultSettingsSource(nested_model_default_partial_update=True)' ) assert ( repr(InitSettingsSource(BaseSettings, init_kwargs={'apple': 'value 0', 'banana': 'value 1'}))