From 96cbdc1910aa4bdcdb47efef587c04a17f75436e Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Sun, 22 Aug 2021 08:28:22 +0200 Subject: [PATCH 1/9] fix(idempotency): sorting keys before hashing --- .../utilities/idempotency/persistence/base.py | 4 +-- tests/functional/idempotency/conftest.py | 14 +++++--- .../idempotency/test_idempotency.py | 34 ++++++++++++++----- 3 files changed, 37 insertions(+), 15 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/base.py b/aws_lambda_powertools/utilities/idempotency/persistence/base.py index 2f5dd512ac6..4901e9f9f75 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/base.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/base.py @@ -223,7 +223,7 @@ def _generate_hash(self, data: Any) -> str: """ data = getattr(data, "raw_event", data) # could be a data class depending on decorator order - hashed_data = self.hash_function(json.dumps(data, cls=Encoder).encode()) + hashed_data = self.hash_function(json.dumps(data, cls=Encoder, sort_keys=True).encode()) return hashed_data.hexdigest() def _validate_payload(self, data: Dict[str, Any], data_record: DataRecord) -> None: @@ -310,7 +310,7 @@ def save_success(self, data: Dict[str, Any], result: dict) -> None: result: dict The response from function """ - response_data = json.dumps(result, cls=Encoder) + response_data = json.dumps(result, cls=Encoder, sort_keys=True) data_record = DataRecord( idempotency_key=self._get_hashed_idempotency_key(data=data), diff --git a/tests/functional/idempotency/conftest.py b/tests/functional/idempotency/conftest.py index e613bb85e60..2c528cafc50 100644 --- a/tests/functional/idempotency/conftest.py +++ b/tests/functional/idempotency/conftest.py @@ -21,6 +21,10 @@ TABLE_NAME = "TEST_TABLE" +def serialize(data): + return json.dumps(data, sort_keys=True, cls=Encoder) + + @pytest.fixture(scope="module") def config() -> Config: return Config(region_name="us-east-1") @@ -62,12 +66,12 @@ def lambda_response(): @pytest.fixture(scope="module") def serialized_lambda_response(lambda_response): - return json.dumps(lambda_response, cls=Encoder) + return serialize(lambda_response) @pytest.fixture(scope="module") def deserialized_lambda_response(lambda_response): - return json.loads(json.dumps(lambda_response, cls=Encoder)) + return json.loads(serialize(lambda_response)) @pytest.fixture @@ -144,7 +148,7 @@ def expected_params_put_item_with_validation(hashed_idempotency_key, hashed_vali def hashed_idempotency_key(lambda_apigw_event, default_jmespath, lambda_context): compiled_jmespath = jmespath.compile(default_jmespath) data = compiled_jmespath.search(lambda_apigw_event) - return "test-func#" + hashlib.md5(json.dumps(data).encode()).hexdigest() + return "test-func#" + hashlib.md5(serialize(data).encode()).hexdigest() @pytest.fixture @@ -152,12 +156,12 @@ def hashed_idempotency_key_with_envelope(lambda_apigw_event): event = extract_data_from_envelope( data=lambda_apigw_event, envelope=envelopes.API_GATEWAY_HTTP, jmespath_options={} ) - return "test-func#" + hashlib.md5(json.dumps(event).encode()).hexdigest() + return "test-func#" + hashlib.md5(serialize(event).encode()).hexdigest() @pytest.fixture def hashed_validation_key(lambda_apigw_event): - return hashlib.md5(json.dumps(lambda_apigw_event["requestContext"]).encode()).hexdigest() + return hashlib.md5(serialize(lambda_apigw_event["requestContext"]).encode()).hexdigest() @pytest.fixture diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/test_idempotency.py index 5505a7dc5c9..cb0d43ae6fa 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/test_idempotency.py @@ -21,6 +21,7 @@ from aws_lambda_powertools.utilities.idempotency.idempotency import idempotent, idempotent_function from aws_lambda_powertools.utilities.idempotency.persistence.base import BasePersistenceLayer, DataRecord from aws_lambda_powertools.utilities.validation import envelopes, validator +from tests.functional.idempotency.conftest import serialize from tests.functional.utils import load_event TABLE_NAME = "TEST_TABLE" @@ -741,7 +742,7 @@ def test_default_no_raise_on_missing_idempotency_key( hashed_key = persistence_store._get_hashed_idempotency_key({}) # THEN return the hash of None - expected_value = "test-func#" + md5(json.dumps(None).encode()).hexdigest() + expected_value = "test-func#" + md5(serialize(None).encode()).hexdigest() assert expected_value == hashed_key @@ -785,7 +786,7 @@ def test_jmespath_with_powertools_json( expected_value = [sub_attr_value, key_attr_value] api_gateway_proxy_event = { "requestContext": {"authorizer": {"claims": {"sub": sub_attr_value}}}, - "body": json.dumps({"id": key_attr_value}), + "body": serialize({"id": key_attr_value}), } # WHEN calling _get_hashed_idempotency_key @@ -869,7 +870,7 @@ def _delete_record(self, data_record: DataRecord) -> None: def test_idempotent_lambda_event_source(lambda_context): # Scenario to validate that we can use the event_source decorator before or after the idempotent decorator mock_event = load_event("apiGatewayProxyV2Event.json") - persistence_layer = MockPersistenceLayer("test-func#" + hashlib.md5(json.dumps(mock_event).encode()).hexdigest()) + persistence_layer = MockPersistenceLayer("test-func#" + hashlib.md5(serialize(mock_event).encode()).hexdigest()) expected_result = {"message": "Foo"} # GIVEN an event_source decorator @@ -889,7 +890,7 @@ def lambda_handler(event, _): def test_idempotent_function(): # Scenario to validate we can use idempotent_function with any function mock_event = {"data": "value"} - persistence_layer = MockPersistenceLayer("test-func#" + hashlib.md5(json.dumps(mock_event).encode()).hexdigest()) + persistence_layer = MockPersistenceLayer("test-func#" + hashlib.md5(serialize(mock_event).encode()).hexdigest()) expected_result = {"message": "Foo"} @idempotent_function(persistence_store=persistence_layer, data_keyword_argument="record") @@ -906,7 +907,7 @@ def test_idempotent_function_arbitrary_args_kwargs(): # Scenario to validate we can use idempotent_function with a function # with an arbitrary number of args and kwargs mock_event = {"data": "value"} - persistence_layer = MockPersistenceLayer("test-func#" + hashlib.md5(json.dumps(mock_event).encode()).hexdigest()) + persistence_layer = MockPersistenceLayer("test-func#" + hashlib.md5(serialize(mock_event).encode()).hexdigest()) expected_result = {"message": "Foo"} @idempotent_function(persistence_store=persistence_layer, data_keyword_argument="record") @@ -921,7 +922,7 @@ def record_handler(arg_one, arg_two, record, is_record): def test_idempotent_function_invalid_data_kwarg(): mock_event = {"data": "value"} - persistence_layer = MockPersistenceLayer("test-func#" + hashlib.md5(json.dumps(mock_event).encode()).hexdigest()) + persistence_layer = MockPersistenceLayer("test-func#" + hashlib.md5(serialize(mock_event).encode()).hexdigest()) expected_result = {"message": "Foo"} keyword_argument = "payload" @@ -938,7 +939,7 @@ def record_handler(record): def test_idempotent_function_arg_instead_of_kwarg(): mock_event = {"data": "value"} - persistence_layer = MockPersistenceLayer("test-func#" + hashlib.md5(json.dumps(mock_event).encode()).hexdigest()) + persistence_layer = MockPersistenceLayer("test-func#" + hashlib.md5(serialize(mock_event).encode()).hexdigest()) expected_result = {"message": "Foo"} keyword_argument = "record" @@ -956,7 +957,7 @@ def record_handler(record): def test_idempotent_function_and_lambda_handler(lambda_context): # Scenario to validate we can use both idempotent_function and idempotent decorators mock_event = {"data": "value"} - persistence_layer = MockPersistenceLayer("test-func#" + hashlib.md5(json.dumps(mock_event).encode()).hexdigest()) + persistence_layer = MockPersistenceLayer("test-func#" + hashlib.md5(serialize(mock_event).encode()).hexdigest()) expected_result = {"message": "Foo"} @idempotent_function(persistence_store=persistence_layer, data_keyword_argument="record") @@ -976,3 +977,20 @@ def lambda_handler(event, _): # THEN we expect the function and lambda handler to execute successfully assert fn_result == expected_result assert handler_result == expected_result + + +def test_idempotent_data_sorting(): + # Scenario to validate same data in different order hashes to the same idempotency key + data_one = {"data": "test message 1", "more_data": "more data 1"} + data_two = {"more_data": "more data 1", "data": "test message 1"} + + # Assertion will happen in MockPersistenceLayer + persistence_layer = MockPersistenceLayer("test-func#" + hashlib.md5(json.dumps(data_one).encode()).hexdigest()) + + # GIVEN + @idempotent_function(data_keyword_argument="payload", persistence_store=persistence_layer) + def dummy(payload): + return {"message": "hello"} + + # WHEN + dummy(payload=data_two) From 4d7cf6b8e99f5050a136ecce111fd304d763483f Mon Sep 17 00:00:00 2001 From: "Gerald W. Lester" Date: Tue, 28 Sep 2021 16:26:22 -0500 Subject: [PATCH 2/9] Add logger to class inits. --- .../utilities/feature_flags/appconfig.py | 7 +++- .../utilities/feature_flags/feature_flags.py | 41 ++++++++++--------- .../utilities/feature_flags/schema.py | 21 +++++----- 3 files changed, 37 insertions(+), 32 deletions(-) diff --git a/aws_lambda_powertools/utilities/feature_flags/appconfig.py b/aws_lambda_powertools/utilities/feature_flags/appconfig.py index 2e0edc3b9b1..90b73064214 100644 --- a/aws_lambda_powertools/utilities/feature_flags/appconfig.py +++ b/aws_lambda_powertools/utilities/feature_flags/appconfig.py @@ -10,8 +10,6 @@ from .base import StoreProvider from .exceptions import ConfigurationStoreError, StoreClientError -logger = logging.getLogger(__name__) - TRANSFORM_TYPE = "json" @@ -25,6 +23,7 @@ def __init__( sdk_config: Optional[Config] = None, envelope: Optional[str] = "", jmespath_options: Optional[Dict] = None, + logger=None, ): """This class fetches JSON schemas from AWS AppConfig @@ -46,6 +45,10 @@ def __init__( Alternative JMESPath options to be included when filtering expr """ super().__init__() + if logger == None: + self.logger = logging.getLogger(__name__) + else: + self.logger = logger self.environment = environment self.application = application self.name = name diff --git a/aws_lambda_powertools/utilities/feature_flags/feature_flags.py b/aws_lambda_powertools/utilities/feature_flags/feature_flags.py index d04e74ff293..7b79dc63cce 100644 --- a/aws_lambda_powertools/utilities/feature_flags/feature_flags.py +++ b/aws_lambda_powertools/utilities/feature_flags/feature_flags.py @@ -5,11 +5,9 @@ from .base import StoreProvider from .exceptions import ConfigurationStoreError -logger = logging.getLogger(__name__) - class FeatureFlags: - def __init__(self, store: StoreProvider): + def __init__(self, store: StoreProvider, logger=None): """Evaluates whether feature flags should be enabled based on a given context. It uses the provided store to fetch feature flag rules before evaluating them. @@ -37,9 +35,12 @@ def __init__(self, store: StoreProvider): Store to use to fetch feature flag schema configuration. """ self._store = store + if logger == None: + self.logger = logging.getLogger(__name__) + else: + self.logger = logger - @staticmethod - def _match_by_action(action: str, condition_value: Any, context_value: Any) -> bool: + def _match_by_action(self, action: str, condition_value: Any, context_value: Any) -> bool: if not context_value: return False mapping_by_action = { @@ -54,7 +55,7 @@ def _match_by_action(action: str, condition_value: Any, context_value: Any) -> b func = mapping_by_action.get(action, lambda a, b: False) return func(context_value, condition_value) except Exception as exc: - logger.debug(f"caught exception while matching action: action={action}, exception={str(exc)}") + self.loggerdebug(f"caught exception while matching action: action={action}, exception={str(exc)}") return False def _evaluate_conditions( @@ -65,7 +66,7 @@ def _evaluate_conditions( conditions = cast(List[Dict], rule.get(schema.CONDITIONS_KEY)) if not conditions: - logger.debug( + self.loggerdebug( f"rule did not match, no conditions to match, rule_name={rule_name}, rule_value={rule_match_value}, " f"name={feature_name} " ) @@ -77,13 +78,13 @@ def _evaluate_conditions( cond_value = condition.get(schema.CONDITION_VALUE) if not self._match_by_action(action=cond_action, condition_value=cond_value, context_value=context_value): - logger.debug( + self.loggerdebug( f"rule did not match action, rule_name={rule_name}, rule_value={rule_match_value}, " f"name={feature_name}, context_value={str(context_value)} " ) return False # context doesn't match condition - logger.debug(f"rule matched, rule_name={rule_name}, rule_value={rule_match_value}, name={feature_name}") + self.loggerdebug(f"rule matched, rule_name={rule_name}, rule_value={rule_match_value}, name={feature_name}") return True def _evaluate_rules( @@ -94,12 +95,12 @@ def _evaluate_rules( rule_match_value = rule.get(schema.RULE_MATCH_VALUE) # Context might contain PII data; do not log its value - logger.debug(f"Evaluating rule matching, rule={rule_name}, feature={feature_name}, default={feat_default}") + self.loggerdebug(f"Evaluating rule matching, rule={rule_name}, feature={feature_name}, default={feat_default}") if self._evaluate_conditions(rule_name=rule_name, feature_name=feature_name, rule=rule, context=context): return bool(rule_match_value) # no rule matched, return default value of feature - logger.debug(f"no rule matched, returning feature default, default={feat_default}, name={feature_name}") + self.loggerdebug(f"no rule matched, returning feature default, default={feat_default}, name={feature_name}") return feat_default return False @@ -146,7 +147,7 @@ def get_configuration(self) -> Union[Dict[str, Dict], Dict]: ``` """ # parse result conf as JSON, keep in cache for max age defined in store - logger.debug(f"Fetching schema from registered store, store={self._store}") + self.loggerdebug(f"Fetching schema from registered store, store={self._store}") config = self._store.get_configuration() validator = schema.SchemaValidator(schema=config) validator.validate() @@ -190,21 +191,21 @@ def evaluate(self, *, name: str, context: Optional[Dict[str, Any]] = None, defau try: features = self.get_configuration() except ConfigurationStoreError as err: - logger.debug(f"Failed to fetch feature flags from store, returning default provided, reason={err}") + self.loggerdebug(f"Failed to fetch feature flags from store, returning default provided, reason={err}") return default feature = features.get(name) if feature is None: - logger.debug(f"Feature not found; returning default provided, name={name}, default={default}") + self.loggerdebug(f"Feature not found; returning default provided, name={name}, default={default}") return default rules = feature.get(schema.RULES_KEY) feat_default = feature.get(schema.FEATURE_DEFAULT_VAL_KEY) if not rules: - logger.debug(f"no rules found, returning feature default, name={name}, default={feat_default}") + self.loggerdebug(f"no rules found, returning feature default, name={name}, default={feat_default}") return bool(feat_default) - logger.debug(f"looking for rule match, name={name}, default={feat_default}") + self.loggerdebug(f"looking for rule match, name={name}, default={feat_default}") return self._evaluate_rules(feature_name=name, context=context, feat_default=bool(feat_default), rules=rules) def get_enabled_features(self, *, context: Optional[Dict[str, Any]] = None) -> List[str]: @@ -241,20 +242,20 @@ def get_enabled_features(self, *, context: Optional[Dict[str, Any]] = None) -> L try: features: Dict[str, Any] = self.get_configuration() except ConfigurationStoreError as err: - logger.debug(f"Failed to fetch feature flags from store, returning empty list, reason={err}") + self.loggerdebug(f"Failed to fetch feature flags from store, returning empty list, reason={err}") return features_enabled - logger.debug("Evaluating all features") + self.loggerdebug("Evaluating all features") for name, feature in features.items(): rules = feature.get(schema.RULES_KEY, {}) feature_default_value = feature.get(schema.FEATURE_DEFAULT_VAL_KEY) if feature_default_value and not rules: - logger.debug(f"feature is enabled by default and has no defined rules, name={name}") + self.loggerdebug(f"feature is enabled by default and has no defined rules, name={name}") features_enabled.append(name) elif self._evaluate_rules( feature_name=name, context=context, feat_default=feature_default_value, rules=rules ): - logger.debug(f"feature's calculated value is True, name={name}") + self.loggerdebug(f"feature's calculated value is True, name={name}") features_enabled.append(name) return features_enabled diff --git a/aws_lambda_powertools/utilities/feature_flags/schema.py b/aws_lambda_powertools/utilities/feature_flags/schema.py index efce82018db..daf6152ebb2 100644 --- a/aws_lambda_powertools/utilities/feature_flags/schema.py +++ b/aws_lambda_powertools/utilities/feature_flags/schema.py @@ -5,8 +5,6 @@ from .base import BaseValidator from .exceptions import SchemaValidationError -logger = logging.getLogger(__name__) - RULES_KEY = "rules" FEATURE_DEFAULT_VAL_KEY = "default" CONDITIONS_KEY = "conditions" @@ -105,11 +103,15 @@ class SchemaValidator(BaseValidator): ``` """ - def __init__(self, schema: Dict[str, Any]): + def __init__(self, schema: Dict[str, Any], logger=None): self.schema = schema + if logger == None: + self.logger = logging.getLogger(__name__) + else: + self.logger = logger def validate(self) -> None: - logger.debug("Validating schema") + self.loggerdebug("Validating schema") if not isinstance(self.schema, dict): raise SchemaValidationError(f"Features must be a dictionary, schema={str(self.schema)}") @@ -125,7 +127,7 @@ def __init__(self, schema: Dict): def validate(self): for name, feature in self.schema.items(): - logger.debug(f"Attempting to validate feature '{name}'") + self.loggerdebug(f"Attempting to validate feature '{name}'") self.validate_feature(name, feature) rules = RulesValidator(feature=feature) rules.validate() @@ -150,14 +152,14 @@ def __init__(self, feature: Dict[str, Any]): def validate(self): if not self.rules: - logger.debug("Rules are empty, ignoring validation") + self.loggerdebug("Rules are empty, ignoring validation") return if not isinstance(self.rules, dict): raise SchemaValidationError(f"Feature rules must be a dictionary, feature={self.feature_name}") for rule_name, rule in self.rules.items(): - logger.debug(f"Attempting to validate rule '{rule_name}'") + self.loggerdebug(f"Attempting to validate rule '{rule_name}'") self.validate_rule(rule=rule, rule_name=rule_name, feature_name=self.feature_name) conditions = ConditionsValidator(rule=rule, rule_name=rule_name) conditions.validate() @@ -194,13 +196,12 @@ def validate(self): for condition in self.conditions: self.validate_condition(rule_name=self.rule_name, condition=condition) - @staticmethod - def validate_condition(rule_name: str, condition: Dict[str, str]) -> None: + def validate_condition(self,rule_name: str, condition: Dict[str, str]) -> None: if not condition or not isinstance(condition, dict): raise SchemaValidationError(f"Feature rule condition must be a dictionary, rule={rule_name}") # Condition can contain PII data; do not log condition value - logger.debug(f"Attempting to validate condition for '{rule_name}'") + self.loggerdebug(f"Attempting to validate condition for '{rule_name}'") ConditionsValidator.validate_condition_action(condition=condition, rule_name=rule_name) ConditionsValidator.validate_condition_key(condition=condition, rule_name=rule_name) ConditionsValidator.validate_condition_value(condition=condition, rule_name=rule_name) From dd6f67fec139ee1fe02b657ac398f8705dc2c174 Mon Sep 17 00:00:00 2001 From: "Gerald W. Lester" Date: Tue, 28 Sep 2021 16:49:32 -0500 Subject: [PATCH 3/9] Updated documentation --- aws_lambda_powertools/utilities/feature_flags/appconfig.py | 2 ++ aws_lambda_powertools/utilities/feature_flags/feature_flags.py | 2 ++ docs/utilities/feature_flags.md | 1 + 3 files changed, 5 insertions(+) diff --git a/aws_lambda_powertools/utilities/feature_flags/appconfig.py b/aws_lambda_powertools/utilities/feature_flags/appconfig.py index 90b73064214..a0e016dda7b 100644 --- a/aws_lambda_powertools/utilities/feature_flags/appconfig.py +++ b/aws_lambda_powertools/utilities/feature_flags/appconfig.py @@ -43,6 +43,8 @@ def __init__( JMESPath expression to pluck feature flags data from config jmespath_options : Optional[Dict] Alternative JMESPath options to be included when filtering expr + logger: A logging object + Used to log messages. If None is supplied, one will be created. """ super().__init__() if logger == None: diff --git a/aws_lambda_powertools/utilities/feature_flags/feature_flags.py b/aws_lambda_powertools/utilities/feature_flags/feature_flags.py index 7b79dc63cce..6abddb21fc6 100644 --- a/aws_lambda_powertools/utilities/feature_flags/feature_flags.py +++ b/aws_lambda_powertools/utilities/feature_flags/feature_flags.py @@ -33,6 +33,8 @@ def __init__(self, store: StoreProvider, logger=None): ---------- store: StoreProvider Store to use to fetch feature flag schema configuration. + logger: A logging object + Used to log messages. If None is supplied, one will be created. """ self._store = store if logger == None: diff --git a/docs/utilities/feature_flags.md b/docs/utilities/feature_flags.md index d22f9c03296..f5f899e3d11 100644 --- a/docs/utilities/feature_flags.md +++ b/docs/utilities/feature_flags.md @@ -548,6 +548,7 @@ Parameter | Default | Description **max_age** | `5` | Number of seconds to cache feature flags configuration fetched from AWS AppConfig **sdk_config** | `None` | [Botocore Config object](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html){target="_blank"} **jmespath_options** | `None` | For advanced use cases when you want to bring your own [JMESPath functions](https://github.com/jmespath/jmespath.py#custom-functions){target="_blank"} +**logger** | `None` | Logger to use. If None is supplied, then a logger will be created. === "appconfig_store_example.py" From 97d5f96ffc29da16c0cd695b2394aaa8d8905fd9 Mon Sep 17 00:00:00 2001 From: Gerald Leter Date: Thu, 30 Sep 2021 16:59:13 -0500 Subject: [PATCH 4/9] Update aws_lambda_powertools/utilities/feature_flags/appconfig.py Co-authored-by: Dani Comnea --- aws_lambda_powertools/utilities/feature_flags/appconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda_powertools/utilities/feature_flags/appconfig.py b/aws_lambda_powertools/utilities/feature_flags/appconfig.py index a0e016dda7b..103d4d87726 100644 --- a/aws_lambda_powertools/utilities/feature_flags/appconfig.py +++ b/aws_lambda_powertools/utilities/feature_flags/appconfig.py @@ -44,7 +44,7 @@ def __init__( jmespath_options : Optional[Dict] Alternative JMESPath options to be included when filtering expr logger: A logging object - Used to log messages. If None is supplied, one will be created. + Used to log messages. If None is supplied, one will be created. """ super().__init__() if logger == None: From 29633dc93efd9b99e8896c8b9d04f2821f407b2b Mon Sep 17 00:00:00 2001 From: Gerald Leter Date: Thu, 30 Sep 2021 16:59:22 -0500 Subject: [PATCH 5/9] Update aws_lambda_powertools/utilities/feature_flags/feature_flags.py Co-authored-by: Dani Comnea --- aws_lambda_powertools/utilities/feature_flags/feature_flags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda_powertools/utilities/feature_flags/feature_flags.py b/aws_lambda_powertools/utilities/feature_flags/feature_flags.py index 6abddb21fc6..e2f1d227891 100644 --- a/aws_lambda_powertools/utilities/feature_flags/feature_flags.py +++ b/aws_lambda_powertools/utilities/feature_flags/feature_flags.py @@ -34,7 +34,7 @@ def __init__(self, store: StoreProvider, logger=None): store: StoreProvider Store to use to fetch feature flag schema configuration. logger: A logging object - Used to log messages. If None is supplied, one will be created. + Used to log messages. If None is supplied, one will be created. """ self._store = store if logger == None: From 12339663f6e7e008893fa956806154faadf18c64 Mon Sep 17 00:00:00 2001 From: "Gerald W. Lester" Date: Fri, 1 Oct 2021 08:26:18 -0500 Subject: [PATCH 6/9] Add missing period to logger.debug --- .../utilities/feature_flags/feature_flags.py | 30 +++++++++---------- .../utilities/feature_flags/schema.py | 10 +++---- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/aws_lambda_powertools/utilities/feature_flags/feature_flags.py b/aws_lambda_powertools/utilities/feature_flags/feature_flags.py index e2f1d227891..346a114a518 100644 --- a/aws_lambda_powertools/utilities/feature_flags/feature_flags.py +++ b/aws_lambda_powertools/utilities/feature_flags/feature_flags.py @@ -57,7 +57,7 @@ def _match_by_action(self, action: str, condition_value: Any, context_value: Any func = mapping_by_action.get(action, lambda a, b: False) return func(context_value, condition_value) except Exception as exc: - self.loggerdebug(f"caught exception while matching action: action={action}, exception={str(exc)}") + self.logger.debug(f"caught exception while matching action: action={action}, exception={str(exc)}") return False def _evaluate_conditions( @@ -68,7 +68,7 @@ def _evaluate_conditions( conditions = cast(List[Dict], rule.get(schema.CONDITIONS_KEY)) if not conditions: - self.loggerdebug( + self.logger.debug( f"rule did not match, no conditions to match, rule_name={rule_name}, rule_value={rule_match_value}, " f"name={feature_name} " ) @@ -80,13 +80,13 @@ def _evaluate_conditions( cond_value = condition.get(schema.CONDITION_VALUE) if not self._match_by_action(action=cond_action, condition_value=cond_value, context_value=context_value): - self.loggerdebug( + self.logger.debug( f"rule did not match action, rule_name={rule_name}, rule_value={rule_match_value}, " f"name={feature_name}, context_value={str(context_value)} " ) return False # context doesn't match condition - self.loggerdebug(f"rule matched, rule_name={rule_name}, rule_value={rule_match_value}, name={feature_name}") + self.logger.debug(f"rule matched, rule_name={rule_name}, rule_value={rule_match_value}, name={feature_name}") return True def _evaluate_rules( @@ -97,12 +97,12 @@ def _evaluate_rules( rule_match_value = rule.get(schema.RULE_MATCH_VALUE) # Context might contain PII data; do not log its value - self.loggerdebug(f"Evaluating rule matching, rule={rule_name}, feature={feature_name}, default={feat_default}") + self.logger.debug(f"Evaluating rule matching, rule={rule_name}, feature={feature_name}, default={feat_default}") if self._evaluate_conditions(rule_name=rule_name, feature_name=feature_name, rule=rule, context=context): return bool(rule_match_value) # no rule matched, return default value of feature - self.loggerdebug(f"no rule matched, returning feature default, default={feat_default}, name={feature_name}") + self.logger.debug(f"no rule matched, returning feature default, default={feat_default}, name={feature_name}") return feat_default return False @@ -149,7 +149,7 @@ def get_configuration(self) -> Union[Dict[str, Dict], Dict]: ``` """ # parse result conf as JSON, keep in cache for max age defined in store - self.loggerdebug(f"Fetching schema from registered store, store={self._store}") + self.logger.debug(f"Fetching schema from registered store, store={self._store}") config = self._store.get_configuration() validator = schema.SchemaValidator(schema=config) validator.validate() @@ -193,21 +193,21 @@ def evaluate(self, *, name: str, context: Optional[Dict[str, Any]] = None, defau try: features = self.get_configuration() except ConfigurationStoreError as err: - self.loggerdebug(f"Failed to fetch feature flags from store, returning default provided, reason={err}") + self.logger.debug(f"Failed to fetch feature flags from store, returning default provided, reason={err}") return default feature = features.get(name) if feature is None: - self.loggerdebug(f"Feature not found; returning default provided, name={name}, default={default}") + self.logger.debug(f"Feature not found; returning default provided, name={name}, default={default}") return default rules = feature.get(schema.RULES_KEY) feat_default = feature.get(schema.FEATURE_DEFAULT_VAL_KEY) if not rules: - self.loggerdebug(f"no rules found, returning feature default, name={name}, default={feat_default}") + self.logger.debug(f"no rules found, returning feature default, name={name}, default={feat_default}") return bool(feat_default) - self.loggerdebug(f"looking for rule match, name={name}, default={feat_default}") + self.logger.debug(f"looking for rule match, name={name}, default={feat_default}") return self._evaluate_rules(feature_name=name, context=context, feat_default=bool(feat_default), rules=rules) def get_enabled_features(self, *, context: Optional[Dict[str, Any]] = None) -> List[str]: @@ -244,20 +244,20 @@ def get_enabled_features(self, *, context: Optional[Dict[str, Any]] = None) -> L try: features: Dict[str, Any] = self.get_configuration() except ConfigurationStoreError as err: - self.loggerdebug(f"Failed to fetch feature flags from store, returning empty list, reason={err}") + self.logger.debug(f"Failed to fetch feature flags from store, returning empty list, reason={err}") return features_enabled - self.loggerdebug("Evaluating all features") + self.logger.debug("Evaluating all features") for name, feature in features.items(): rules = feature.get(schema.RULES_KEY, {}) feature_default_value = feature.get(schema.FEATURE_DEFAULT_VAL_KEY) if feature_default_value and not rules: - self.loggerdebug(f"feature is enabled by default and has no defined rules, name={name}") + self.logger.debug(f"feature is enabled by default and has no defined rules, name={name}") features_enabled.append(name) elif self._evaluate_rules( feature_name=name, context=context, feat_default=feature_default_value, rules=rules ): - self.loggerdebug(f"feature's calculated value is True, name={name}") + self.logger.debug(f"feature's calculated value is True, name={name}") features_enabled.append(name) return features_enabled diff --git a/aws_lambda_powertools/utilities/feature_flags/schema.py b/aws_lambda_powertools/utilities/feature_flags/schema.py index daf6152ebb2..8f64db0dfce 100644 --- a/aws_lambda_powertools/utilities/feature_flags/schema.py +++ b/aws_lambda_powertools/utilities/feature_flags/schema.py @@ -111,7 +111,7 @@ def __init__(self, schema: Dict[str, Any], logger=None): self.logger = logger def validate(self) -> None: - self.loggerdebug("Validating schema") + self.logger.debug("Validating schema") if not isinstance(self.schema, dict): raise SchemaValidationError(f"Features must be a dictionary, schema={str(self.schema)}") @@ -127,7 +127,7 @@ def __init__(self, schema: Dict): def validate(self): for name, feature in self.schema.items(): - self.loggerdebug(f"Attempting to validate feature '{name}'") + self.logger.debug(f"Attempting to validate feature '{name}'") self.validate_feature(name, feature) rules = RulesValidator(feature=feature) rules.validate() @@ -152,14 +152,14 @@ def __init__(self, feature: Dict[str, Any]): def validate(self): if not self.rules: - self.loggerdebug("Rules are empty, ignoring validation") + self.logger.debug("Rules are empty, ignoring validation") return if not isinstance(self.rules, dict): raise SchemaValidationError(f"Feature rules must be a dictionary, feature={self.feature_name}") for rule_name, rule in self.rules.items(): - self.loggerdebug(f"Attempting to validate rule '{rule_name}'") + self.logger.debug(f"Attempting to validate rule '{rule_name}'") self.validate_rule(rule=rule, rule_name=rule_name, feature_name=self.feature_name) conditions = ConditionsValidator(rule=rule, rule_name=rule_name) conditions.validate() @@ -201,7 +201,7 @@ def validate_condition(self,rule_name: str, condition: Dict[str, str]) -> None: raise SchemaValidationError(f"Feature rule condition must be a dictionary, rule={rule_name}") # Condition can contain PII data; do not log condition value - self.loggerdebug(f"Attempting to validate condition for '{rule_name}'") + self.logger.debug(f"Attempting to validate condition for '{rule_name}'") ConditionsValidator.validate_condition_action(condition=condition, rule_name=rule_name) ConditionsValidator.validate_condition_key(condition=condition, rule_name=rule_name) ConditionsValidator.validate_condition_value(condition=condition, rule_name=rule_name) From 3d4305b91d05112ac25aae0947bcc764c191e503 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Fri, 1 Oct 2021 16:09:30 +0200 Subject: [PATCH 7/9] feat: add get_raw_configuration property in store; expose store --- .../utilities/feature_flags/appconfig.py | 48 +++++++++++-------- .../utilities/feature_flags/base.py | 13 ++++- .../utilities/feature_flags/feature_flags.py | 10 ++-- .../feature_flags/test_feature_flags.py | 12 +++++ 4 files changed, 56 insertions(+), 27 deletions(-) diff --git a/aws_lambda_powertools/utilities/feature_flags/appconfig.py b/aws_lambda_powertools/utilities/feature_flags/appconfig.py index 2e0edc3b9b1..df3f83c47aa 100644 --- a/aws_lambda_powertools/utilities/feature_flags/appconfig.py +++ b/aws_lambda_powertools/utilities/feature_flags/appconfig.py @@ -55,9 +55,31 @@ def __init__( self.jmespath_options = jmespath_options self._conf_store = AppConfigProvider(environment=environment, application=application, config=sdk_config) + @property + def get_raw_configuration(self) -> Dict[str, Any]: + """Fetch feature schema configuration from AWS AppConfig""" + try: + # parse result conf as JSON, keep in cache for self.max_age seconds + return cast( + dict, + self._conf_store.get( + name=self.name, + transform=TRANSFORM_TYPE, + max_age=self.cache_seconds, + ), + ) + except (GetParameterError, TransformParameterError) as exc: + err_msg = traceback.format_exc() + if "AccessDenied" in err_msg: + raise StoreClientError(err_msg) from exc + raise ConfigurationStoreError("Unable to get AWS AppConfig configuration file") from exc + def get_configuration(self) -> Dict[str, Any]: """Fetch feature schema configuration from AWS AppConfig + If envelope is set, it'll extract and return feature flags from configuration, + otherwise it'll return the entire configuration fetched from AWS AppConfig. + Raises ------ ConfigurationStoreError @@ -68,25 +90,11 @@ def get_configuration(self) -> Dict[str, Any]: Dict[str, Any] parsed JSON dictionary """ - try: - # parse result conf as JSON, keep in cache for self.max_age seconds - config = cast( - dict, - self._conf_store.get( - name=self.name, - transform=TRANSFORM_TYPE, - max_age=self.cache_seconds, - ), - ) + config = self.get_raw_configuration - if self.envelope: - config = jmespath_utils.extract_data_from_envelope( - data=config, envelope=self.envelope, jmespath_options=self.jmespath_options - ) + if self.envelope: + config = jmespath_utils.extract_data_from_envelope( + data=config, envelope=self.envelope, jmespath_options=self.jmespath_options + ) - return config - except (GetParameterError, TransformParameterError) as exc: - err_msg = traceback.format_exc() - if "AccessDenied" in err_msg: - raise StoreClientError(err_msg) from exc - raise ConfigurationStoreError("Unable to get AWS AppConfig configuration file") from exc + return config diff --git a/aws_lambda_powertools/utilities/feature_flags/base.py b/aws_lambda_powertools/utilities/feature_flags/base.py index edb94c4f45d..e323f32d8b1 100644 --- a/aws_lambda_powertools/utilities/feature_flags/base.py +++ b/aws_lambda_powertools/utilities/feature_flags/base.py @@ -3,10 +3,19 @@ class StoreProvider(ABC): + @property + @abstractmethod + def get_raw_configuration(self) -> Dict[str, Any]: + """Get configuration from any store and return the parsed JSON dictionary""" + raise NotImplementedError() # pragma: no cover + @abstractmethod def get_configuration(self) -> Dict[str, Any]: """Get configuration from any store and return the parsed JSON dictionary + If envelope is set, it'll extract and return feature flags from configuration, + otherwise it'll return the entire configuration fetched from the store. + Raises ------ ConfigurationStoreError @@ -42,10 +51,10 @@ def get_configuration(self) -> Dict[str, Any]: } ``` """ - return NotImplemented # pragma: no cover + raise NotImplementedError() # pragma: no cover class BaseValidator(ABC): @abstractmethod def validate(self): - return NotImplemented # pragma: no cover + raise NotImplementedError() # pragma: no cover diff --git a/aws_lambda_powertools/utilities/feature_flags/feature_flags.py b/aws_lambda_powertools/utilities/feature_flags/feature_flags.py index d04e74ff293..d26144a262a 100644 --- a/aws_lambda_powertools/utilities/feature_flags/feature_flags.py +++ b/aws_lambda_powertools/utilities/feature_flags/feature_flags.py @@ -1,5 +1,5 @@ import logging -from typing import Any, Dict, List, Optional, Union, cast +from typing import Any, Dict, List, Optional, cast from . import schema from .base import StoreProvider @@ -36,7 +36,7 @@ def __init__(self, store: StoreProvider): store: StoreProvider Store to use to fetch feature flag schema configuration. """ - self._store = store + self.store = store @staticmethod def _match_by_action(action: str, condition_value: Any, context_value: Any) -> bool: @@ -103,7 +103,7 @@ def _evaluate_rules( return feat_default return False - def get_configuration(self) -> Union[Dict[str, Dict], Dict]: + def get_configuration(self) -> Dict: """Get validated feature flag schema from configured store. Largely used to aid testing, since it's called by `evaluate` and `get_enabled_features` methods. @@ -146,8 +146,8 @@ def get_configuration(self) -> Union[Dict[str, Dict], Dict]: ``` """ # parse result conf as JSON, keep in cache for max age defined in store - logger.debug(f"Fetching schema from registered store, store={self._store}") - config = self._store.get_configuration() + logger.debug(f"Fetching schema from registered store, store={self.store}") + config: Dict = self.store.get_configuration() validator = schema.SchemaValidator(schema=config) validator.validate() diff --git a/tests/functional/feature_flags/test_feature_flags.py b/tests/functional/feature_flags/test_feature_flags.py index 5342105da3d..8b6698a8179 100644 --- a/tests/functional/feature_flags/test_feature_flags.py +++ b/tests/functional/feature_flags/test_feature_flags.py @@ -587,3 +587,15 @@ def test_get_feature_toggle_propagates_access_denied_error(mocker, config): # THEN raise StoreClientError error with pytest.raises(StoreClientError, match="AccessDeniedException") as err: feature_flags.evaluate(name="Foo", default=False) + + +def test_get_configuration_with_envelope_and_raw(mocker, config): + expected_value = True + mocked_app_config_schema = {"log_level": "INFO", "features": {"my_feature": {"default": expected_value}}} + feature_flags = init_feature_flags(mocker, mocked_app_config_schema, config, envelope="features") + + features_config = feature_flags.get_configuration() + config = feature_flags.store.get_raw_configuration + + assert "log_level" in config + assert "log_level" not in features_config From 93f8a5ce912cc80a545837cb837caa8f29c7b6ea Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Fri, 1 Oct 2021 18:18:16 +0200 Subject: [PATCH 8/9] fix: type annotation, clarify logger in docs --- .../utilities/feature_flags/appconfig.py | 9 +++++++-- .../utilities/feature_flags/feature_flags.py | 5 +++-- .../utilities/feature_flags/schema.py | 14 +++++++++----- docs/utilities/feature_flags.md | 2 +- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/aws_lambda_powertools/utilities/feature_flags/appconfig.py b/aws_lambda_powertools/utilities/feature_flags/appconfig.py index e9d9e29a1fa..ff688dc6be5 100644 --- a/aws_lambda_powertools/utilities/feature_flags/appconfig.py +++ b/aws_lambda_powertools/utilities/feature_flags/appconfig.py @@ -1,11 +1,12 @@ import logging import traceback -from typing import Any, Dict, Optional, cast +from typing import Any, Dict, Optional, Union, cast from botocore.config import Config from aws_lambda_powertools.utilities.parameters import AppConfigProvider, GetParameterError, TransformParameterError +from ... import Logger from ...shared import jmespath_utils from .base import StoreProvider from .exceptions import ConfigurationStoreError, StoreClientError @@ -23,7 +24,7 @@ def __init__( sdk_config: Optional[Config] = None, envelope: Optional[str] = "", jmespath_options: Optional[Dict] = None, - logger=None, + logger: Optional[Union[logging.Logger, Logger]] = None, ): """This class fetches JSON schemas from AWS AppConfig @@ -62,6 +63,9 @@ def get_raw_configuration(self) -> Dict[str, Any]: """Fetch feature schema configuration from AWS AppConfig""" try: # parse result conf as JSON, keep in cache for self.max_age seconds + self.logger.debug( + "Fetching configuration from the store", extra={"param_name": self.name, "max_age": self.cache_seconds} + ) return cast( dict, self._conf_store.get( @@ -95,6 +99,7 @@ def get_configuration(self) -> Dict[str, Any]: config = self.get_raw_configuration if self.envelope: + self.logger.debug("Envelope enabled; extracting data from config", extra={"envelope": self.envelope}) config = jmespath_utils.extract_data_from_envelope( data=config, envelope=self.envelope, jmespath_options=self.jmespath_options ) diff --git a/aws_lambda_powertools/utilities/feature_flags/feature_flags.py b/aws_lambda_powertools/utilities/feature_flags/feature_flags.py index 5bf5f33f3d5..01d3ce13639 100644 --- a/aws_lambda_powertools/utilities/feature_flags/feature_flags.py +++ b/aws_lambda_powertools/utilities/feature_flags/feature_flags.py @@ -1,13 +1,14 @@ import logging -from typing import Any, Dict, List, Optional, cast +from typing import Any, Dict, List, Optional, Union, cast +from ... import Logger from . import schema from .base import StoreProvider from .exceptions import ConfigurationStoreError class FeatureFlags: - def __init__(self, store: StoreProvider, logger=None): + def __init__(self, store: StoreProvider, logger: Optional[Union[logging.Logger, Logger]] = None): """Evaluates whether feature flags should be enabled based on a given context. It uses the provided store to fetch feature flag rules before evaluating them. diff --git a/aws_lambda_powertools/utilities/feature_flags/schema.py b/aws_lambda_powertools/utilities/feature_flags/schema.py index f5111a02cd6..5c19a967067 100644 --- a/aws_lambda_powertools/utilities/feature_flags/schema.py +++ b/aws_lambda_powertools/utilities/feature_flags/schema.py @@ -1,7 +1,8 @@ import logging from enum import Enum -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Union +from ... import Logger from .base import BaseValidator from .exceptions import SchemaValidationError @@ -109,7 +110,7 @@ class SchemaValidator(BaseValidator): ``` """ - def __init__(self, schema: Dict[str, Any], logger=None): + def __init__(self, schema: Dict[str, Any], logger: Optional[Union[logging.Logger, Logger]] = None): self.schema = schema self.logger = logger or logging.getLogger(__name__) @@ -125,8 +126,9 @@ def validate(self) -> None: class FeaturesValidator(BaseValidator): """Validates each feature and calls RulesValidator to validate its rules""" - def __init__(self, schema: Dict): + def __init__(self, schema: Dict, logger: Optional[Union[logging.Logger, Logger]] = None): self.schema = schema + self.logger = logger or logging.getLogger(__name__) def validate(self): for name, feature in self.schema.items(): @@ -148,10 +150,11 @@ def validate_feature(name, feature): class RulesValidator(BaseValidator): """Validates each rule and calls ConditionsValidator to validate each rule's conditions""" - def __init__(self, feature: Dict[str, Any]): + def __init__(self, feature: Dict[str, Any], logger: Optional[Union[logging.Logger, Logger]] = None): self.feature = feature self.feature_name = next(iter(self.feature)) self.rules: Optional[Dict] = self.feature.get(RULES_KEY) + self.logger = logger or logging.getLogger(__name__) def validate(self): if not self.rules: @@ -188,9 +191,10 @@ def validate_rule_default_value(rule: Dict, rule_name: str): class ConditionsValidator(BaseValidator): - def __init__(self, rule: Dict[str, Any], rule_name: str): + def __init__(self, rule: Dict[str, Any], rule_name: str, logger: Optional[Union[logging.Logger, Logger]] = None): self.conditions: List[Dict[str, Any]] = rule.get(CONDITIONS_KEY, {}) self.rule_name = rule_name + self.logger = logger or logging.getLogger(__name__) def validate(self): if not self.conditions or not isinstance(self.conditions, list): diff --git a/docs/utilities/feature_flags.md b/docs/utilities/feature_flags.md index 37a50ae293c..7e08cc358dd 100644 --- a/docs/utilities/feature_flags.md +++ b/docs/utilities/feature_flags.md @@ -580,7 +580,7 @@ Parameter | Default | Description **max_age** | `5` | Number of seconds to cache feature flags configuration fetched from AWS AppConfig **sdk_config** | `None` | [Botocore Config object](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html){target="_blank"} **jmespath_options** | `None` | For advanced use cases when you want to bring your own [JMESPath functions](https://github.com/jmespath/jmespath.py#custom-functions){target="_blank"} -**logger** | `None` | Logger to use. If None is supplied, then a logger will be created. +**logger** | `logging.Logger` | Logger to use for debug. You can optionally supply an instance of Powertools Logger. === "appconfig_store_example.py" From 370d9ca3d0af94c0c6565ad2d38291884f3772ab Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Fri, 1 Oct 2021 18:24:50 +0200 Subject: [PATCH 9/9] fix: remove logger from staticmethod --- aws_lambda_powertools/utilities/feature_flags/schema.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/aws_lambda_powertools/utilities/feature_flags/schema.py b/aws_lambda_powertools/utilities/feature_flags/schema.py index 5c19a967067..fc745342750 100644 --- a/aws_lambda_powertools/utilities/feature_flags/schema.py +++ b/aws_lambda_powertools/utilities/feature_flags/schema.py @@ -201,14 +201,15 @@ def validate(self): raise SchemaValidationError(f"Invalid condition, rule={self.rule_name}") for condition in self.conditions: + # Condition can contain PII data; do not log condition value + self.logger.debug(f"Attempting to validate condition for '{self.rule_name}'") self.validate_condition(rule_name=self.rule_name, condition=condition) - def validate_condition(self, rule_name: str, condition: Dict[str, str]) -> None: + @staticmethod + def validate_condition(rule_name: str, condition: Dict[str, str]) -> None: if not condition or not isinstance(condition, dict): raise SchemaValidationError(f"Feature rule condition must be a dictionary, rule={rule_name}") - # Condition can contain PII data; do not log condition value - self.logger.debug(f"Attempting to validate condition for '{rule_name}'") ConditionsValidator.validate_condition_action(condition=condition, rule_name=rule_name) ConditionsValidator.validate_condition_key(condition=condition, rule_name=rule_name) ConditionsValidator.validate_condition_value(condition=condition, rule_name=rule_name)