Updated metric publishing to address internal review comments.

Also fixed an issue where multiple parts of the SDK were ignoring the "alternate" request ID.

Author: millems

core/aws-core/src/main/java/software/amazon/awssdk/awscore/eventstream/EventStreamAsyncResponseTransformer.java

@@ -17,6 +17,7 @@
 
 import static java.util.Collections.singletonList;
 import static software.amazon.awssdk.core.http.HttpResponseHandler.X_AMZN_REQUEST_ID_HEADER;
+import static software.amazon.awssdk.core.http.HttpResponseHandler.X_AMZN_REQUEST_ID_HEADERS;
 import static software.amazon.awssdk.core.http.HttpResponseHandler.X_AMZ_ID_2_HEADER;
 import static software.amazon.awssdk.utils.FunctionalUtils.runAndLogError;
 
@@ -49,6 +50,7 @@
 import software.amazon.awssdk.http.SdkCancellationException;
 import software.amazon.awssdk.http.SdkHttpFullResponse;
 import software.amazon.awssdk.utils.BinaryUtils;
+import software.amazon.awssdk.utils.http.SdkHttpUtils;
 import software.amazon.eventstream.Message;
 import software.amazon.eventstream.MessageDecoder;
 
@@ -193,9 +195,10 @@ public CompletableFuture<Void> prepare() {
     @Override
     public void onResponse(SdkResponse response) {
         if (response != null && response.sdkHttpResponse() != null) {
-            this.requestId = response.sdkHttpResponse()
-                                     .firstMatchingHeader(X_AMZN_REQUEST_ID_HEADER)
-                                     .orElse(null);
+            this.requestId = SdkHttpUtils.firstMatchingHeaderFromCollection(response.sdkHttpResponse().headers(),
+                                                                            X_AMZN_REQUEST_ID_HEADERS)
+                                         .orElse(null);
+
             this.extendedRequestId = response.sdkHttpResponse()
                                              .firstMatchingHeader(X_AMZ_ID_2_HEADER)
                                              .orElse(null);

core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/LoggingMetricPublisher.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.metrics;
+
+import software.amazon.awssdk.annotations.SdkPublicApi;
+import software.amazon.awssdk.utils.Logger;
+
+/**
+ * An implementation of {@link MetricPublisher} that writes all published metrics to the logs at the INFO level under the
+ * {@code software.amazon.awssdk.metrics.LoggingMetricPublisher} namespace.
+ */
+@SdkPublicApi
+public final class LoggingMetricPublisher implements MetricPublisher {
+    private static final Logger LOGGER = Logger.loggerFor(LoggingMetricPublisher.class);
+
+    private LoggingMetricPublisher() {
+    }
+
+    public static LoggingMetricPublisher create() {
+        return new LoggingMetricPublisher();
+    }
+
+    @Override
+    public void publish(MetricCollection metricCollection) {
+        LOGGER.info(() -> "Metrics published: " + metricCollection);
+    }
+
+    @Override
+    public void close() {
+    }
+}

core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/MetricCategory.java

@@ -25,33 +25,29 @@
  */
 @SdkPublicApi
 public enum MetricCategory {
-
     /**
-     * All metrics defined by the SDK are classified under this category at a minimum. If the metrics feature is enabled
-     * but the category to collect is not, only metrics that are classified under this category are collected by the SDK
+     * Metrics collected by the core SDK are classified under this category.
      */
-    DEFAULT("Default"),
+    CORE("Core"),
 
     /**
      * Metrics collected at the http client level are classified under this category.
      */
     HTTP_CLIENT("HttpClient"),
 
     /**
-     * Metrics specific to streaming, eventStream APIs are classified under this category.
+     * Metrics specified by the customer should be classified under this category.
      */
-    STREAMING("Streaming"),
+    CUSTOM("Custom"),
 
     /**
      * This is an umbrella category (provided for convenience) that records metrics belonging to every category
      * defined in this enum. Clients who wish to collect lot of SDK metrics data should use this.
      * <p>
-     * Note: Enabling this option is verbose and can be expensive based on the platform the metrics are uploaded to.
-     * Please make sure you need all this data before using this category.
+     * Note: Enabling this option along with {@link MetricLevel#TRACE} is verbose and can be expensive based on the platform
+     * the metrics are uploaded to. Please make sure you need all this data before using this category.
      */
-    ALL("All")
-
-    ;
+    ALL("All");
 
     private final String value;
 

core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/MetricCollection.java

@@ -17,6 +17,8 @@
 
 import java.time.Instant;
 import java.util.List;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
 import software.amazon.awssdk.annotations.SdkPublicApi;
 
 /**
@@ -29,6 +31,13 @@ public interface MetricCollection extends Iterable<MetricRecord<?>> {
      */
     String name();
 
+    /**
+     * Return a stream of records in this collection.
+     */
+    default Stream<MetricRecord<?>> stream() {
+        return StreamSupport.stream(spliterator(), false);
+    }
+
     /**
      * Return all the values of the given metric.
      *
@@ -43,6 +52,16 @@ public interface MetricCollection extends Iterable<MetricRecord<?>> {
      */
     List<MetricCollection> children();
 
+    /**
+     * Return all of the {@link #children()} with a specific name.
+     *
+     * @param name The name by which we will filter {@link #children()}.
+     * @return The child metric collections that have the provided name.
+     */
+    default Stream<MetricCollection> childrenWithName(String name) {
+        return children().stream().filter(c -> c.name().equals(name));
+    }
+
     /**
      * @return The time at which this collection was created.
      */

core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/MetricLevel.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.metrics;
+
+import software.amazon.awssdk.annotations.SdkPublicApi;
+
+/**
+ * The {@code MetricLevel} associated with a {@link SdkMetric}, similar to log levels, defines the 'scenario' in which the metric
+ * is useful. This makes it easy to reduce the cost of metric publishing (e.g. by setting it to {@link #INFO}), and then increase
+ * it when additional data level is needed for debugging purposes (e.g. by setting it to {@link #TRACE}.
+ */
+@SdkPublicApi
+public enum MetricLevel {
+    /**
+     * The metric level that includes every other metric level, as well as some highly-technical metrics that may only be useful
+     * in very specific performance or failure scenarios.
+     */
+    TRACE,
+
+    /**
+     * The "default" metric level that includes metrics that are useful for identifying <i>why</i> errors or performance issues
+     * are occurring within the SDK. This excludes technical metrics that are only useful in very specific performance or failure
+     * scenarios.
+     */
+    INFO,
+
+    /**
+     * Includes metrics that report <i>when</i> API call errors are occurring within the SDK. This <b>does not</b> include all
+     * of the information that may be generally useful when debugging <i>why</i> errors are occurring (e.g. latency).
+     */
+    ERROR;
+
+    public boolean includesLevel(MetricLevel level) {
+        return this.compareTo(level) <= 0;
+    }
+}

core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/NoOpMetricCollector.java

@@ -16,14 +16,12 @@
 package software.amazon.awssdk.metrics;
 
 import software.amazon.awssdk.annotations.SdkPublicApi;
-import software.amazon.awssdk.utils.Logger;
 
 /**
  * A metric collector that doesn't do anything.
  */
 @SdkPublicApi
 public final class NoOpMetricCollector implements MetricCollector {
-    private static final Logger log = Logger.loggerFor(NoOpMetricCollector.class);
     private static final NoOpMetricCollector INSTANCE = new NoOpMetricCollector();
 
     private NoOpMetricCollector() {
@@ -36,7 +34,6 @@ public String name() {
 
     @Override
     public <T> void reportMetric(SdkMetric<T> metric, T data) {
-        log.trace(() -> "Metrics reported: " + data);
     }
 
     @Override

core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/SdkMetric.java

@@ -37,6 +37,11 @@ public interface SdkMetric<T> {
      */
     Set<MetricCategory> categories();
 
+    /**
+     * @return The level of this metric.
+     */
+    MetricLevel level();
+
     /**
      * @return The class of the value associated with this metric.
      */
@@ -52,20 +57,6 @@ public interface SdkMetric<T> {
      */
     T convertValue(Object o);
 
-    /**
-     * Create a new metric under the {@link MetricCategory#DEFAULT} category.
-     *
-     * @param name The name of this metric.
-     * @param clzz The class of the object containing the associated value for this metric.
-     * @param <T> The type of the object containing the associated value for this metric.
-     * @return The created metric.
-     *
-     * @throws IllegalArgumentException If a metric of the same name has already been created.
-     */
-    static <T> SdkMetric<T> create(String name, Class<T> clzz) {
-        return DefaultSdkMetric.create(name, clzz, MetricCategory.DEFAULT);
-    }
-
     /**
      * Create a new metric.
      *
@@ -78,8 +69,8 @@ static <T> SdkMetric<T> create(String name, Class<T> clzz) {
      *
      * @throws IllegalArgumentException If a metric of the same name has already been created.
      */
-    static <T> SdkMetric<T> create(String name, Class<T> clzz, MetricCategory c1, MetricCategory... cn) {
-        return DefaultSdkMetric.create(name, clzz, c1, cn);
+    static <T> SdkMetric<T> create(String name, Class<T> clzz, MetricLevel level, MetricCategory c1, MetricCategory... cn) {
+        return DefaultSdkMetric.create(name, clzz, level, c1, cn);
     }
 
     /**
@@ -93,7 +84,7 @@ static <T> SdkMetric<T> create(String name, Class<T> clzz, MetricCategory c1, Me
      *
      * @throws IllegalArgumentException If a metric of the same name has already been created.
      */
-    static <T> SdkMetric<T> create(String name, Class<T> clzz, Set<MetricCategory> categories) {
-        return DefaultSdkMetric.create(name, clzz, categories);
+    static <T> SdkMetric<T> create(String name, Class<T> clzz, MetricLevel level, Set<MetricCategory> categories) {
+        return DefaultSdkMetric.create(name, clzz, level, categories);
     }
 }

core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/internal/DefaultSdkMetric.java

@@ -24,6 +24,7 @@
 import software.amazon.awssdk.annotations.SdkInternalApi;
 import software.amazon.awssdk.annotations.SdkTestInternalApi;
 import software.amazon.awssdk.metrics.MetricCategory;
+import software.amazon.awssdk.metrics.MetricLevel;
 import software.amazon.awssdk.metrics.SdkMetric;
 import software.amazon.awssdk.utils.AttributeMap;
 import software.amazon.awssdk.utils.ToString;
@@ -36,32 +37,42 @@ public final class DefaultSdkMetric<T> extends AttributeMap.Key<T> implements Sd
     private final String name;
     private final Class<T> clzz;
     private final Set<MetricCategory> categories;
+    private final MetricLevel level;
 
-    private DefaultSdkMetric(String name, Class<T> clzz, Set<MetricCategory> categories) {
+    private DefaultSdkMetric(String name, Class<T> clzz, MetricLevel level, Set<MetricCategory> categories) {
         super(clzz);
         this.name = Validate.notBlank(name, "name must not be blank");
         this.clzz = Validate.notNull(clzz, "clzz must not be null");
+        this.level = Validate.notNull(level, "level must not be null");
         Validate.notEmpty(categories, "categories must not be empty");
         this.categories = EnumSet.copyOf(categories);
     }
 
     /**
      * @return The name of this event.
      */
+    @Override
     public String name() {
         return name;
     }
 
     /**
      * @return The categories of this event.
      */
+    @Override
     public Set<MetricCategory> categories() {
         return Collections.unmodifiableSet(categories);
     }
 
+    @Override
+    public MetricLevel level() {
+        return level;
+    }
+
     /**
      * @return The class of the value associated with this event.
      */
+    @Override
     public Class<T> valueClass() {
         return clzz;
     }
@@ -106,13 +117,14 @@ public String toString() {
      *
      * @throws IllegalArgumentException If a metric of the same name has already been created.
      */
-    public static <T> SdkMetric<T> create(String name, Class<T> clzz, MetricCategory c1, MetricCategory... cn) {
+    public static <T> SdkMetric<T> create(String name, Class<T> clzz, MetricLevel level,
+                                          MetricCategory c1, MetricCategory... cn) {
         Stream<MetricCategory> categoryStream = Stream.of(c1);
         if (cn != null) {
             categoryStream = Stream.concat(categoryStream, Stream.of(cn));
         }
         Set<MetricCategory> categories = categoryStream.collect(Collectors.toSet());
-        return create(name, clzz, categories);
+        return create(name, clzz, level, categories);
     }
 
     /**
@@ -126,9 +138,9 @@ public static <T> SdkMetric<T> create(String name, Class<T> clzz, MetricCategory
      *
      * @throws IllegalArgumentException If a metric of the same name has already been created.
      */
-    public static <T> SdkMetric<T> create(String name, Class<T> clzz, Set<MetricCategory> categories) {
+    public static <T> SdkMetric<T> create(String name, Class<T> clzz, MetricLevel level, Set<MetricCategory> categories) {
         Validate.noNullElements(categories, "categories must not contain null elements");
-        SdkMetric<T> event = new DefaultSdkMetric<>(name, clzz, categories);
+        SdkMetric<T> event = new DefaultSdkMetric<>(name, clzz, level, categories);
         if (SDK_METRICS.putIfAbsent(event, Boolean.TRUE) != null) {
             throw new IllegalArgumentException("Metric with name " + name + " has already been created");
         }