Add a DISABLE_METRIC_EXTRACTION config option (#55)

* Add a DISABLE_METRIC_EXTRACTION config option

This adds a new configuration option `DISABLE_METRIC_EXTRACTION` which
removes the `_aws` metadata from the EMF log records, thus disabling the
extraction of metrics by CloudWatch Metrics. The remaining part of the
log records is still getting logged.

Author: Dunedan

README.md

@@ -250,6 +250,21 @@ Config.namespace = "MyApplication"
 AWS_EMF_NAMESPACE = MyApplication
 ```
 
+**DISABLE_METRIC_EXTRACTION**: Disables extraction of metrics by CloudWatch, by omitting EMF
+metadata from serialized log records.
+
+Example:
+
+```py
+# in process
+from aws_embedded_metrics.config import get_config
+Config = get_config()
+Config.disable_metric_extraction = True
+
+# environment
+AWS_EMF_DISABLE_METRIC_EXTRACTION = true
+```
+
 ## Examples
 
 Check out the [examples](https://github.com/awslabs/aws-embedded-metrics-python/tree/master/examples) directory to get started.

aws_embedded_metrics/config/configuration.py

@@ -23,6 +23,7 @@ def __init__(
         agent_endpoint: str,
         ec2_metadata_endpoint: str = None,
         namespace: str = None,
+        disable_metric_extraction: bool = False,
     ):
         self.debug_logging_enabled = debug_logging_enabled
         self.service_name = service_name
@@ -32,3 +33,4 @@ def __init__(
         self.agent_endpoint = agent_endpoint
         self.ec2_metadata_endpoint = ec2_metadata_endpoint
         self.namespace = namespace
+        self.disable_metric_extraction = disable_metric_extraction

aws_embedded_metrics/config/environment_configuration_provider.py

@@ -24,6 +24,7 @@
 AGENT_ENDPOINT = "AGENT_ENDPOINT"
 EC2_METADATA_ENDPOINT = "EC2_METADATA_ENDPOINT"
 NAMESPACE = "NAMESPACE"
+DISABLE_METRIC_EXTRACTION = "DISABLE_METRIC_EXTRACTION"
 
 
 class EnvironmentConfigurationProvider:
@@ -41,6 +42,7 @@ def get_configuration(self) -> Configuration:
             self.__get_env_var(AGENT_ENDPOINT),
             self.__get_env_var(EC2_METADATA_ENDPOINT),
             self.__get_env_var(NAMESPACE),
+            self.__get_bool_env_var(DISABLE_METRIC_EXTRACTION),
         )
 
     @staticmethod

aws_embedded_metrics/serializers/log_serializer.py

@@ -11,6 +11,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from aws_embedded_metrics.config import get_config
 from aws_embedded_metrics.logger.metrics_context import MetricsContext
 from aws_embedded_metrics.serializers import Serializer
 from aws_embedded_metrics.constants import MAX_DIMENSIONS, MAX_METRICS_PER_EVENT
@@ -21,6 +22,8 @@
 class LogSerializer(Serializer):
     @staticmethod
     def serialize(context: MetricsContext) -> List[str]:
+        config = get_config()
+
         dimension_keys = []
         dimensions_properties: Dict[str, str] = {}
 
@@ -30,10 +33,12 @@ def serialize(context: MetricsContext) -> List[str]:
             dimensions_properties = {**dimensions_properties, **dimension_set}
 
         def create_body() -> Dict[str, Any]:
-            return {
+            body: Dict[str, Any] = {
                 **dimensions_properties,
                 **context.properties,
-                "_aws": {
+            }
+            if not config.disable_metric_extraction:
+                body["_aws"] = {
                     **context.meta,
                     "CloudWatchMetrics": [
                         {
@@ -42,11 +47,12 @@ def create_body() -> Dict[str, Any]:
                             "Namespace": context.namespace,
                         },
                     ],
-                },
-            }
+                }
+            return body
 
         current_body: Dict[str, Any] = create_body()
         event_batches: List[str] = []
+        num_metrics_in_current_body = 0
 
         for metric_name, metric in context.metrics.items():
 
@@ -55,14 +61,18 @@ def create_body() -> Dict[str, Any]:
             else:
                 current_body[metric_name] = metric.values
 
-            current_body["_aws"]["CloudWatchMetrics"][0]["Metrics"].append({"Name": metric_name, "Unit": metric.unit})
+            if not config.disable_metric_extraction:
+                current_body["_aws"]["CloudWatchMetrics"][0]["Metrics"].append({"Name": metric_name, "Unit": metric.unit})
+
+            num_metrics_in_current_body += 1
 
-            should_serialize: bool = len(current_body["_aws"]["CloudWatchMetrics"][0]["Metrics"]) == MAX_METRICS_PER_EVENT
+            should_serialize: bool = num_metrics_in_current_body == MAX_METRICS_PER_EVENT
             if should_serialize:
                 event_batches.append(json.dumps(current_body))
                 current_body = create_body()
+                num_metrics_in_current_body = 0
 
-        if not event_batches or current_body["_aws"]["CloudWatchMetrics"][0]["Metrics"]:
+        if not event_batches or num_metrics_in_current_body > 0:
             event_batches.append(json.dumps(current_body))
 
         return event_batches

tests/config/test_config.py

@@ -23,6 +23,7 @@ def test_can_get_config_from_environment(monkeypatch):
     agent_endpoint = fake.word()
     ec2_metadata_endpoint = fake.word()
     namespace = fake.word()
+    disable_metric_extraction = True
 
     monkeypatch.setenv("AWS_EMF_ENABLE_DEBUG_LOGGING", str(debug_enabled))
     monkeypatch.setenv("AWS_EMF_SERVICE_NAME", service_name)
@@ -32,6 +33,7 @@ def test_can_get_config_from_environment(monkeypatch):
     monkeypatch.setenv("AWS_EMF_AGENT_ENDPOINT", agent_endpoint)
     monkeypatch.setenv("AWS_EMF_EC2_METADATA_ENDPOINT", ec2_metadata_endpoint)
     monkeypatch.setenv("AWS_EMF_NAMESPACE", namespace)
+    monkeypatch.setenv("AWS_EMF_DISABLE_METRIC_EXTRACTION", str(disable_metric_extraction))
 
     # act
     result = get_config()
@@ -45,6 +47,7 @@ def test_can_get_config_from_environment(monkeypatch):
     assert result.agent_endpoint == agent_endpoint
     assert result.ec2_metadata_endpoint == ec2_metadata_endpoint
     assert result.namespace == namespace
+    assert result.disable_metric_extraction == disable_metric_extraction
 
 
 def test_can_override_config(monkeypatch):
@@ -57,6 +60,7 @@ def test_can_override_config(monkeypatch):
     monkeypatch.setenv("AWS_EMF_AGENT_ENDPOINT", fake.word())
     monkeypatch.setenv("AWS_EMF_EC2_METADATA_ENDPOINT", fake.word())
     monkeypatch.setenv("AWS_EMF_NAMESPACE", fake.word())
+    monkeypatch.setenv("AWS_EMF_DISABLE_METRIC_EXTRACTION", str(True))
 
     config = get_config()
 
@@ -68,6 +72,7 @@ def test_can_override_config(monkeypatch):
     agent_endpoint = fake.word()
     ec2_metadata_endpoint = fake.word()
     namespace = fake.word()
+    disable_metric_extraction = False
 
     # act
     config.debug_logging_enabled = debug_enabled
@@ -78,6 +83,7 @@ def test_can_override_config(monkeypatch):
     config.agent_endpoint = agent_endpoint
     config.ec2_metadata_endpoint = ec2_metadata_endpoint
     config.namespace = namespace
+    config.disable_metric_extraction = disable_metric_extraction
 
     # assert
     assert config.debug_logging_enabled == debug_enabled
@@ -88,3 +94,4 @@ def test_can_override_config(monkeypatch):
     assert config.agent_endpoint == agent_endpoint
     assert config.ec2_metadata_endpoint == ec2_metadata_endpoint
     assert config.namespace == namespace
+    assert config.disable_metric_extraction == disable_metric_extraction

tests/serializer/test_log_serializer.py

@@ -1,3 +1,4 @@
+from aws_embedded_metrics.config import get_config
 from aws_embedded_metrics.logger.metrics_context import MetricsContext
 from aws_embedded_metrics.serializers.log_serializer import LogSerializer
 from faker import Faker
@@ -177,6 +178,27 @@ def test_serialize_metrics_with_multiple_datapoints():
     assert results == [json.dumps(expected)]
 
 
+def test_serialize_metrics_with_aggregation_disabled():
+    """Test log records don't contain metadata when aggregation is disabled."""
+    # arrange
+    config = get_config()
+    config.disable_metric_extraction = True
+
+    expected_key = fake.word()
+    expected_value = fake.random.randrange(0, 100)
+
+    expected = {expected_key: expected_value}
+
+    context = get_context()
+    context.put_metric(expected_key, expected_value)
+
+    # act
+    result_json = serializer.serialize(context)[0]
+
+    # assert
+    assert_json_equality(result_json, expected)
+
+
 # Test utility method
 
 

tests/sinks/test_lambda_sink.py

@@ -1,3 +1,6 @@
+from importlib import reload
+
+from aws_embedded_metrics import config
 from aws_embedded_metrics.sinks.lambda_sink import LambdaSink
 from aws_embedded_metrics.logger.metrics_context import MetricsContext
 from faker import Faker
@@ -9,6 +12,8 @@
 
 def test_accept_writes_to_stdout(capfd):
     # arrange
+    reload(config)
+
     sink = LambdaSink()
     context = MetricsContext.empty()
     context.meta["Timestamp"] = 1