From 5870d28bbe640f6a9b05316118775ff0b5428880 Mon Sep 17 00:00:00 2001 From: Shane Spencer <305301+whardier@users.noreply.github.com> Date: Mon, 13 Dec 2021 19:43:01 +0000 Subject: [PATCH 1/9] initial whack at adding in callable transforms --- .../utilities/parameters/base.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/aws_lambda_powertools/utilities/parameters/base.py b/aws_lambda_powertools/utilities/parameters/base.py index b059a3b2483..6d5876ea94d 100644 --- a/aws_lambda_powertools/utilities/parameters/base.py +++ b/aws_lambda_powertools/utilities/parameters/base.py @@ -9,6 +9,7 @@ from datetime import datetime, timedelta from typing import Any, Dict, Optional, Tuple, Union +from ...shared.types import AnyCallableT from .exceptions import GetParameterError, TransformParameterError DEFAULT_MAX_AGE_SECS = 5 @@ -34,14 +35,14 @@ def __init__(self): self.store = {} - def _has_not_expired(self, key: Tuple[str, Optional[str]]) -> bool: + def _has_not_expired(self, key: Tuple[str, Optional[Union[AnyCallableT, str]]]) -> bool: return key in self.store and self.store[key].ttl >= datetime.now() def get( self, name: str, max_age: int = DEFAULT_MAX_AGE_SECS, - transform: Optional[str] = None, + transform: Optional[Union[AnyCallableT, str]] = None, force_fetch: bool = False, **sdk_options, ) -> Union[str, list, dict, bytes]: @@ -112,7 +113,7 @@ def get_multiple( self, path: str, max_age: int = DEFAULT_MAX_AGE_SECS, - transform: Optional[str] = None, + transform: Optional[Union[AnyCallableT, str]] = None, raise_on_transform_error: bool = False, force_fetch: bool = False, **sdk_options, @@ -217,7 +218,11 @@ def get_transform_method(key: str, transform: Optional[str] = None) -> Optional[ return None -def transform_value(value: str, transform: str, raise_on_transform_error: bool = True) -> Union[dict, bytes, None]: +def transform_value( + value: str, + transform: Union[AnyCallableT, str], + raise_on_transform_error: bool = True, +) -> Union[dict, bytes, None]: """ Apply a transform to a value @@ -238,7 +243,9 @@ def transform_value(value: str, transform: str, raise_on_transform_error: bool = """ try: - if transform == TRANSFORM_METHOD_JSON: + if callable(transform): + return transform(value) + elif transform == TRANSFORM_METHOD_JSON: return json.loads(value) elif transform == TRANSFORM_METHOD_BINARY: return base64.b64decode(value) From dd6456e138df40fc0010cdb3c5f3b54751338662 Mon Sep 17 00:00:00 2001 From: Shane Spencer <305301+whardier@users.noreply.github.com> Date: Mon, 13 Dec 2021 19:47:17 +0000 Subject: [PATCH 2/9] initial whack at adding in callable transforms --- aws_lambda_powertools/utilities/feature_flags/appconfig.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/aws_lambda_powertools/utilities/feature_flags/appconfig.py b/aws_lambda_powertools/utilities/feature_flags/appconfig.py index dd581df9e22..3bdee3da7fa 100644 --- a/aws_lambda_powertools/utilities/feature_flags/appconfig.py +++ b/aws_lambda_powertools/utilities/feature_flags/appconfig.py @@ -3,6 +3,7 @@ from typing import Any, Dict, Optional, Union, cast from botocore.config import Config +from aws_lambda_powertools.shared.types import AnyCallableT from aws_lambda_powertools.utilities import jmespath_utils from aws_lambda_powertools.utilities.parameters import AppConfigProvider, GetParameterError, TransformParameterError @@ -22,6 +23,7 @@ def __init__( name: str, max_age: int = 5, sdk_config: Optional[Config] = None, + transform: Union[AnyCallableT, str] = TRANSFORM_TYPE, envelope: Optional[str] = "", jmespath_options: Optional[Dict] = None, logger: Optional[Union[logging.Logger, Logger]] = None, @@ -54,6 +56,7 @@ def __init__( self.name = name self.cache_seconds = max_age self.config = sdk_config + self.transform = transform self.envelope = envelope self.jmespath_options = jmespath_options self._conf_store = AppConfigProvider(environment=environment, application=application, config=sdk_config) @@ -70,7 +73,7 @@ def get_raw_configuration(self) -> Dict[str, Any]: dict, self._conf_store.get( name=self.name, - transform=TRANSFORM_TYPE, + transform=self.transform, max_age=self.cache_seconds, ), ) From 620812d2959ea8b41e9dda6fd9eaef9ce7050c91 Mon Sep 17 00:00:00 2001 From: Shane Spencer <305301+whardier@users.noreply.github.com> Date: Mon, 13 Dec 2021 20:17:49 +0000 Subject: [PATCH 3/9] add in tests for callable transforms for parameters --- tests/functional/test_utilities_parameters.py | 118 ++++++++++++++++++ 1 file changed, 118 insertions(+) diff --git a/tests/functional/test_utilities_parameters.py b/tests/functional/test_utilities_parameters.py index 47fc5a0e982..41aab0ec5ec 100644 --- a/tests/functional/test_utilities_parameters.py +++ b/tests/functional/test_utilities_parameters.py @@ -1138,6 +1138,52 @@ def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: assert "Incorrect padding" in str(excinfo) +def test_base_provider_get_transform_callable(mock_name, mock_value): + """ + Test BaseProvider.get() with a callable transform + """ + + mock_data = str(mock_value) + + class TestProvider(BaseProvider): + def _get(self, name: str, **kwargs) -> str: + assert name == mock_name + return mock_data + + def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + raise NotImplementedError() + + provider = TestProvider() + + value = provider.get(mock_name, transform=str) + + assert isinstance(value, str) + assert value == mock_data + + +def test_base_provider_get_transform_callable_exception(mock_name): + """ + Test BaseProvider.get() with a callable transform that raises an exception + """ + + mock_data = str(mock_value) + "a" + + class TestProvider(BaseProvider): + def _get(self, name: str, **kwargs) -> str: + assert name == mock_name + return mock_data + + def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + raise NotImplementedError() + + provider = TestProvider() + + with pytest.raises(parameters.TransformParameterError) as excinfo: + provider.get(mock_name, transform=int) + + assert "invalid literal for int() with base 10" in str(excinfo) + + def test_base_provider_get_multiple_transform_json(mock_name, mock_value): """ Test BaseProvider.get_multiple() with a json transform @@ -1281,6 +1327,78 @@ def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: assert "Incorrect padding" in str(excinfo) +def test_base_provider_get_multiple_transform_callable(mock_name, mock_value): + """ + Test BaseProvider.get_multiple() with a callable transform + """ + + mock_data = str(mock_value) + + class TestProvider(BaseProvider): + def _get(self, name: str, **kwargs) -> str: + raise NotImplementedError() + + def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + assert path == mock_name + return {"A": mock_data} + + provider = TestProvider() + + value = provider.get_multiple(mock_name, transform=str) + + assert isinstance(value, dict) + assert value["A"] == mock_data + + +def test_base_provider_get_multiple_transform_callable_partial_failure(mock_name, mock_value): + """ + Test BaseProvider.get_multiple() with a callable transform that contains a partial failure + """ + + mock_integer = 1234 + mock_data_a = str(mock_integer) + mock_data_b = str(mock_value) + "a" + + class TestProvider(BaseProvider): + def _get(self, name: str, **kwargs) -> str: + raise NotImplementedError() + + def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + assert path == mock_name + return {"A": mock_data_a, "B": mock_data_b} + + provider = TestProvider() + + value = provider.get_multiple(mock_name, transform=int) + + assert isinstance(value, dict) + assert value["A"] == mock_integer + assert value["B"] is None + + +def test_base_provider_get_multiple_transform_callable_exception(mock_name, mock_value): + """ + Test BaseProvider.get_multiple() with a callable transform that raises an exception + """ + + mock_data = str(mock_value) + "a" + + class TestProvider(BaseProvider): + def _get(self, name: str, **kwargs) -> str: + raise NotImplementedError() + + def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + assert path == mock_name + return {"A": mock_data} + + provider = TestProvider() + + with pytest.raises(parameters.TransformParameterError) as excinfo: + provider.get_multiple(mock_name, transform=int, raise_on_transform_error=True) + + assert "invalid literal for int() with base 10" in str(excinfo) + + def test_base_provider_get_multiple_cached(mock_name, mock_value): """ Test BaseProvider.get_multiple() with cached values From 097e9fb3a26941332c5e7fdd9185070ae94a2cd4 Mon Sep 17 00:00:00 2001 From: Shane Spencer <305301+whardier@users.noreply.github.com> Date: Mon, 13 Dec 2021 20:36:46 +0000 Subject: [PATCH 4/9] add in tests for callable transforms for parameters --- tests/functional/test_utilities_parameters.py | 43 +++++++++++-------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/tests/functional/test_utilities_parameters.py b/tests/functional/test_utilities_parameters.py index 41aab0ec5ec..541151d8b7c 100644 --- a/tests/functional/test_utilities_parameters.py +++ b/tests/functional/test_utilities_parameters.py @@ -1,9 +1,11 @@ import base64 import json +import pickle import random import string from datetime import datetime, timedelta from io import BytesIO +from tkinter import E from typing import Dict import pytest @@ -1143,7 +1145,8 @@ def test_base_provider_get_transform_callable(mock_name, mock_value): Test BaseProvider.get() with a callable transform """ - mock_data = str(mock_value) + mock_binary = mock_value.encode() + mock_data = base64.b16encode(mock_binary).decode() class TestProvider(BaseProvider): def _get(self, name: str, **kwargs) -> str: @@ -1155,10 +1158,10 @@ def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: provider = TestProvider() - value = provider.get(mock_name, transform=str) + value = provider.get(mock_name, transform=base64.b16decode) - assert isinstance(value, str) - assert value == mock_data + assert isinstance(value, bytes) + assert value == mock_binary def test_base_provider_get_transform_callable_exception(mock_name): @@ -1166,7 +1169,8 @@ def test_base_provider_get_transform_callable_exception(mock_name): Test BaseProvider.get() with a callable transform that raises an exception """ - mock_data = str(mock_value) + "a" + mock_data = "qw" + print(mock_data) class TestProvider(BaseProvider): def _get(self, name: str, **kwargs) -> str: @@ -1179,9 +1183,9 @@ def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: provider = TestProvider() with pytest.raises(parameters.TransformParameterError) as excinfo: - provider.get(mock_name, transform=int) + provider.get(mock_name, transform=base64.b16decode) - assert "invalid literal for int() with base 10" in str(excinfo) + assert "Non-base16 digit found" in str(excinfo) def test_base_provider_get_multiple_transform_json(mock_name, mock_value): @@ -1332,7 +1336,8 @@ def test_base_provider_get_multiple_transform_callable(mock_name, mock_value): Test BaseProvider.get_multiple() with a callable transform """ - mock_data = str(mock_value) + mock_binary = mock_value.encode() + mock_data = base64.b16encode(mock_binary).decode() class TestProvider(BaseProvider): def _get(self, name: str, **kwargs) -> str: @@ -1344,10 +1349,10 @@ def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: provider = TestProvider() - value = provider.get_multiple(mock_name, transform=str) + value = provider.get_multiple(mock_name, transform=base64.b16decode) assert isinstance(value, dict) - assert value["A"] == mock_data + assert value["A"] == mock_binary def test_base_provider_get_multiple_transform_callable_partial_failure(mock_name, mock_value): @@ -1355,9 +1360,9 @@ def test_base_provider_get_multiple_transform_callable_partial_failure(mock_name Test BaseProvider.get_multiple() with a callable transform that contains a partial failure """ - mock_integer = 1234 - mock_data_a = str(mock_integer) - mock_data_b = str(mock_value) + "a" + mock_binary = mock_value.encode() + mock_data_a = base64.b16encode(mock_binary).decode() + mock_data_b = "qw" class TestProvider(BaseProvider): def _get(self, name: str, **kwargs) -> str: @@ -1369,19 +1374,19 @@ def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: provider = TestProvider() - value = provider.get_multiple(mock_name, transform=int) + value = provider.get_multiple(mock_name, transform=base64.b16decode) assert isinstance(value, dict) - assert value["A"] == mock_integer + assert value["A"] == mock_binary assert value["B"] is None -def test_base_provider_get_multiple_transform_callable_exception(mock_name, mock_value): +def test_base_provider_get_multiple_transform_callable_exception(mock_name): """ Test BaseProvider.get_multiple() with a callable transform that raises an exception """ - mock_data = str(mock_value) + "a" + mock_data = "qw" class TestProvider(BaseProvider): def _get(self, name: str, **kwargs) -> str: @@ -1394,9 +1399,9 @@ def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: provider = TestProvider() with pytest.raises(parameters.TransformParameterError) as excinfo: - provider.get_multiple(mock_name, transform=int, raise_on_transform_error=True) + provider.get_multiple(mock_name, transform=base64.b16decode, raise_on_transform_error=True) - assert "invalid literal for int() with base 10" in str(excinfo) + assert "Non-base16 digit found" in str(excinfo) def test_base_provider_get_multiple_cached(mock_name, mock_value): From 4884b706fdf22c702e73a1cfb293ba86489608db Mon Sep 17 00:00:00 2001 From: Shane Spencer <305301+whardier@users.noreply.github.com> Date: Mon, 13 Dec 2021 20:43:11 +0000 Subject: [PATCH 5/9] add in tests for callable transforms for parameters --- tests/functional/test_utilities_parameters.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/functional/test_utilities_parameters.py b/tests/functional/test_utilities_parameters.py index 541151d8b7c..c38ab01dfe9 100644 --- a/tests/functional/test_utilities_parameters.py +++ b/tests/functional/test_utilities_parameters.py @@ -1,6 +1,5 @@ import base64 import json -import pickle import random import string from datetime import datetime, timedelta From 771e0b00ea5c75de9ca9633e402413eb1f702af3 Mon Sep 17 00:00:00 2001 From: Shane Spencer <305301+whardier@users.noreply.github.com> Date: Mon, 13 Dec 2021 20:59:21 +0000 Subject: [PATCH 6/9] "properly?" type the optional transform in feature flags appconfig and start thinking about what to do here. --- aws_lambda_powertools/utilities/feature_flags/appconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda_powertools/utilities/feature_flags/appconfig.py b/aws_lambda_powertools/utilities/feature_flags/appconfig.py index 3bdee3da7fa..be6e19d5873 100644 --- a/aws_lambda_powertools/utilities/feature_flags/appconfig.py +++ b/aws_lambda_powertools/utilities/feature_flags/appconfig.py @@ -23,7 +23,7 @@ def __init__( name: str, max_age: int = 5, sdk_config: Optional[Config] = None, - transform: Union[AnyCallableT, str] = TRANSFORM_TYPE, + transform: Optional[Union[AnyCallableT, str]] = TRANSFORM_TYPE, envelope: Optional[str] = "", jmespath_options: Optional[Dict] = None, logger: Optional[Union[logging.Logger, Logger]] = None, From d4483077a60db2f430e3cae22a1311ba70df08d7 Mon Sep 17 00:00:00 2001 From: Shane Spencer <305301+whardier@users.noreply.github.com> Date: Mon, 13 Dec 2021 21:31:59 +0000 Subject: [PATCH 7/9] add in feature flags test.. naming in this module is kind of wonky --- .../feature_flags/test_feature_flags.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/functional/feature_flags/test_feature_flags.py b/tests/functional/feature_flags/test_feature_flags.py index 8381dc6bf1d..82aceb235a9 100644 --- a/tests/functional/feature_flags/test_feature_flags.py +++ b/tests/functional/feature_flags/test_feature_flags.py @@ -1,3 +1,4 @@ +import json from typing import Dict, List, Optional import pytest @@ -1197,3 +1198,30 @@ def test_flags_greater_than_or_equal_match_2(mocker, config): default=False, ) assert toggle == expected_value + + +def test_get_feature_toggle_appconfig_store_callable_transform(mocker, config): + + mock_schema = { + "ten_percent_off_campaign": { + "default": False, + }, + } + + mock_value = json.dumps(mock_schema) + + mocked__get_conf = mocker.patch("aws_lambda_powertools.utilities.parameters.AppConfigProvider._get") + mocked__get_conf.return_value = mock_value + + app_conf_fetcher = AppConfigStore( + environment="test_env", + application="test_app", + name="test_conf_name", + max_age=600, + sdk_config=config, + transform=json.loads, + ) + + feature_flags: FeatureFlags = FeatureFlags(store=app_conf_fetcher) + + assert feature_flags.get_configuration() == mock_schema From 67bc3e5a090232405aecb75a1b5d41b4d31a7233 Mon Sep 17 00:00:00 2001 From: Shane Spencer <305301+whardier@users.noreply.github.com> Date: Mon, 13 Dec 2021 21:33:00 +0000 Subject: [PATCH 8/9] unify styling --- aws_lambda_powertools/utilities/feature_flags/appconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda_powertools/utilities/feature_flags/appconfig.py b/aws_lambda_powertools/utilities/feature_flags/appconfig.py index be6e19d5873..9871df7314f 100644 --- a/aws_lambda_powertools/utilities/feature_flags/appconfig.py +++ b/aws_lambda_powertools/utilities/feature_flags/appconfig.py @@ -3,12 +3,12 @@ from typing import Any, Dict, Optional, Union, cast from botocore.config import Config -from aws_lambda_powertools.shared.types import AnyCallableT from aws_lambda_powertools.utilities import jmespath_utils from aws_lambda_powertools.utilities.parameters import AppConfigProvider, GetParameterError, TransformParameterError from ... import Logger +from ...shared.types import AnyCallableT from .base import StoreProvider from .exceptions import ConfigurationStoreError, StoreClientError From 2040d0fcd0ac6721d2b1fde36c97cd0d11370110 Mon Sep 17 00:00:00 2001 From: Shane Spencer <305301+whardier@users.noreply.github.com> Date: Mon, 13 Dec 2021 21:34:37 +0000 Subject: [PATCH 9/9] remove wonky autoimported module from parameters unit test --- tests/functional/test_utilities_parameters.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/functional/test_utilities_parameters.py b/tests/functional/test_utilities_parameters.py index c38ab01dfe9..3cf59ea15ef 100644 --- a/tests/functional/test_utilities_parameters.py +++ b/tests/functional/test_utilities_parameters.py @@ -4,7 +4,6 @@ import string from datetime import datetime, timedelta from io import BytesIO -from tkinter import E from typing import Dict import pytest