diff --git a/core/aws-core/src/main/java/software/amazon/awssdk/awscore/eventstream/EventStreamAsyncResponseTransformer.java b/core/aws-core/src/main/java/software/amazon/awssdk/awscore/eventstream/EventStreamAsyncResponseTransformer.java index d801bb9c19e6..d8437707427e 100644 --- a/core/aws-core/src/main/java/software/amazon/awssdk/awscore/eventstream/EventStreamAsyncResponseTransformer.java +++ b/core/aws-core/src/main/java/software/amazon/awssdk/awscore/eventstream/EventStreamAsyncResponseTransformer.java @@ -17,6 +17,7 @@ import static java.util.Collections.singletonList; import static software.amazon.awssdk.core.http.HttpResponseHandler.X_AMZN_REQUEST_ID_HEADER; +import static software.amazon.awssdk.core.http.HttpResponseHandler.X_AMZN_REQUEST_ID_HEADERS; import static software.amazon.awssdk.core.http.HttpResponseHandler.X_AMZ_ID_2_HEADER; import static software.amazon.awssdk.utils.FunctionalUtils.runAndLogError; @@ -49,6 +50,7 @@ import software.amazon.awssdk.http.SdkCancellationException; import software.amazon.awssdk.http.SdkHttpFullResponse; import software.amazon.awssdk.utils.BinaryUtils; +import software.amazon.awssdk.utils.http.SdkHttpUtils; import software.amazon.eventstream.Message; import software.amazon.eventstream.MessageDecoder; @@ -193,9 +195,10 @@ public CompletableFuture prepare() { @Override public void onResponse(SdkResponse response) { if (response != null && response.sdkHttpResponse() != null) { - this.requestId = response.sdkHttpResponse() - .firstMatchingHeader(X_AMZN_REQUEST_ID_HEADER) - .orElse(null); + this.requestId = SdkHttpUtils.firstMatchingHeaderFromCollection(response.sdkHttpResponse().headers(), + X_AMZN_REQUEST_ID_HEADERS) + .orElse(null); + this.extendedRequestId = response.sdkHttpResponse() .firstMatchingHeader(X_AMZ_ID_2_HEADER) .orElse(null); diff --git a/core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/LoggingMetricPublisher.java b/core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/LoggingMetricPublisher.java new file mode 100644 index 000000000000..004f41c63e81 --- /dev/null +++ b/core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/LoggingMetricPublisher.java @@ -0,0 +1,44 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.metrics; + +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.utils.Logger; + +/** + * An implementation of {@link MetricPublisher} that writes all published metrics to the logs at the INFO level under the + * {@code software.amazon.awssdk.metrics.LoggingMetricPublisher} namespace. + */ +@SdkPublicApi +public final class LoggingMetricPublisher implements MetricPublisher { + private static final Logger LOGGER = Logger.loggerFor(LoggingMetricPublisher.class); + + private LoggingMetricPublisher() { + } + + public static LoggingMetricPublisher create() { + return new LoggingMetricPublisher(); + } + + @Override + public void publish(MetricCollection metricCollection) { + LOGGER.info(() -> "Metrics published: " + metricCollection); + } + + @Override + public void close() { + } +} diff --git a/core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/MetricCategory.java b/core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/MetricCategory.java index 867d73426556..f034c2184997 100644 --- a/core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/MetricCategory.java +++ b/core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/MetricCategory.java @@ -25,12 +25,10 @@ */ @SdkPublicApi public enum MetricCategory { - /** - * All metrics defined by the SDK are classified under this category at a minimum. If the metrics feature is enabled - * but the category to collect is not, only metrics that are classified under this category are collected by the SDK + * Metrics collected by the core SDK are classified under this category. */ - DEFAULT("Default"), + CORE("Core"), /** * Metrics collected at the http client level are classified under this category. @@ -38,20 +36,18 @@ public enum MetricCategory { HTTP_CLIENT("HttpClient"), /** - * Metrics specific to streaming, eventStream APIs are classified under this category. + * Metrics specified by the customer should be classified under this category. */ - STREAMING("Streaming"), + CUSTOM("Custom"), /** * This is an umbrella category (provided for convenience) that records metrics belonging to every category * defined in this enum. Clients who wish to collect lot of SDK metrics data should use this. *

- * Note: Enabling this option is verbose and can be expensive based on the platform the metrics are uploaded to. - * Please make sure you need all this data before using this category. + * Note: Enabling this option along with {@link MetricLevel#TRACE} is verbose and can be expensive based on the platform + * the metrics are uploaded to. Please make sure you need all this data before using this category. */ - ALL("All") - - ; + ALL("All"); private final String value; diff --git a/core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/MetricCollection.java b/core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/MetricCollection.java index 904ae992f4bc..5eb4a031de98 100644 --- a/core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/MetricCollection.java +++ b/core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/MetricCollection.java @@ -17,6 +17,8 @@ import java.time.Instant; import java.util.List; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; import software.amazon.awssdk.annotations.SdkPublicApi; /** @@ -29,6 +31,13 @@ public interface MetricCollection extends Iterable> { */ String name(); + /** + * Return a stream of records in this collection. + */ + default Stream> stream() { + return StreamSupport.stream(spliterator(), false); + } + /** * Return all the values of the given metric. * @@ -43,6 +52,16 @@ public interface MetricCollection extends Iterable> { */ List children(); + /** + * Return all of the {@link #children()} with a specific name. + * + * @param name The name by which we will filter {@link #children()}. + * @return The child metric collections that have the provided name. + */ + default Stream childrenWithName(String name) { + return children().stream().filter(c -> c.name().equals(name)); + } + /** * @return The time at which this collection was created. */ diff --git a/core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/MetricLevel.java b/core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/MetricLevel.java new file mode 100644 index 000000000000..5c87d9805a68 --- /dev/null +++ b/core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/MetricLevel.java @@ -0,0 +1,49 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.metrics; + +import software.amazon.awssdk.annotations.SdkPublicApi; + +/** + * The {@code MetricLevel} associated with a {@link SdkMetric}, similar to log levels, defines the 'scenario' in which the metric + * is useful. This makes it easy to reduce the cost of metric publishing (e.g. by setting it to {@link #INFO}), and then increase + * it when additional data level is needed for debugging purposes (e.g. by setting it to {@link #TRACE}. + */ +@SdkPublicApi +public enum MetricLevel { + /** + * The metric level that includes every other metric level, as well as some highly-technical metrics that may only be useful + * in very specific performance or failure scenarios. + */ + TRACE, + + /** + * The "default" metric level that includes metrics that are useful for identifying why errors or performance issues + * are occurring within the SDK. This excludes technical metrics that are only useful in very specific performance or failure + * scenarios. + */ + INFO, + + /** + * Includes metrics that report when API call errors are occurring within the SDK. This does not include all + * of the information that may be generally useful when debugging why errors are occurring (e.g. latency). + */ + ERROR; + + public boolean includesLevel(MetricLevel level) { + return this.compareTo(level) <= 0; + } +} diff --git a/core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/NoOpMetricCollector.java b/core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/NoOpMetricCollector.java index 4c3266c8c626..c6bbb5e88997 100644 --- a/core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/NoOpMetricCollector.java +++ b/core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/NoOpMetricCollector.java @@ -16,14 +16,12 @@ package software.amazon.awssdk.metrics; import software.amazon.awssdk.annotations.SdkPublicApi; -import software.amazon.awssdk.utils.Logger; /** * A metric collector that doesn't do anything. */ @SdkPublicApi public final class NoOpMetricCollector implements MetricCollector { - private static final Logger log = Logger.loggerFor(NoOpMetricCollector.class); private static final NoOpMetricCollector INSTANCE = new NoOpMetricCollector(); private NoOpMetricCollector() { @@ -36,7 +34,6 @@ public String name() { @Override public void reportMetric(SdkMetric metric, T data) { - log.trace(() -> "Metrics reported: " + data); } @Override diff --git a/core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/SdkMetric.java b/core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/SdkMetric.java index 270488389652..0b555243dca4 100644 --- a/core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/SdkMetric.java +++ b/core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/SdkMetric.java @@ -37,6 +37,11 @@ public interface SdkMetric { */ Set categories(); + /** + * @return The level of this metric. + */ + MetricLevel level(); + /** * @return The class of the value associated with this metric. */ @@ -52,20 +57,6 @@ public interface SdkMetric { */ T convertValue(Object o); - /** - * Create a new metric under the {@link MetricCategory#DEFAULT} category. - * - * @param name The name of this metric. - * @param clzz The class of the object containing the associated value for this metric. - * @param The type of the object containing the associated value for this metric. - * @return The created metric. - * - * @throws IllegalArgumentException If a metric of the same name has already been created. - */ - static SdkMetric create(String name, Class clzz) { - return DefaultSdkMetric.create(name, clzz, MetricCategory.DEFAULT); - } - /** * Create a new metric. * @@ -78,8 +69,8 @@ static SdkMetric create(String name, Class clzz) { * * @throws IllegalArgumentException If a metric of the same name has already been created. */ - static SdkMetric create(String name, Class clzz, MetricCategory c1, MetricCategory... cn) { - return DefaultSdkMetric.create(name, clzz, c1, cn); + static SdkMetric create(String name, Class clzz, MetricLevel level, MetricCategory c1, MetricCategory... cn) { + return DefaultSdkMetric.create(name, clzz, level, c1, cn); } /** @@ -93,7 +84,7 @@ static SdkMetric create(String name, Class clzz, MetricCategory c1, Me * * @throws IllegalArgumentException If a metric of the same name has already been created. */ - static SdkMetric create(String name, Class clzz, Set categories) { - return DefaultSdkMetric.create(name, clzz, categories); + static SdkMetric create(String name, Class clzz, MetricLevel level, Set categories) { + return DefaultSdkMetric.create(name, clzz, level, categories); } } diff --git a/core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/internal/DefaultSdkMetric.java b/core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/internal/DefaultSdkMetric.java index 56615403d28d..461307f31e70 100644 --- a/core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/internal/DefaultSdkMetric.java +++ b/core/metrics-spi/src/main/java/software/amazon/awssdk/metrics/internal/DefaultSdkMetric.java @@ -24,6 +24,7 @@ import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.annotations.SdkTestInternalApi; import software.amazon.awssdk.metrics.MetricCategory; +import software.amazon.awssdk.metrics.MetricLevel; import software.amazon.awssdk.metrics.SdkMetric; import software.amazon.awssdk.utils.AttributeMap; import software.amazon.awssdk.utils.ToString; @@ -36,11 +37,13 @@ public final class DefaultSdkMetric extends AttributeMap.Key implements Sd private final String name; private final Class clzz; private final Set categories; + private final MetricLevel level; - private DefaultSdkMetric(String name, Class clzz, Set categories) { + private DefaultSdkMetric(String name, Class clzz, MetricLevel level, Set categories) { super(clzz); this.name = Validate.notBlank(name, "name must not be blank"); this.clzz = Validate.notNull(clzz, "clzz must not be null"); + this.level = Validate.notNull(level, "level must not be null"); Validate.notEmpty(categories, "categories must not be empty"); this.categories = EnumSet.copyOf(categories); } @@ -48,6 +51,7 @@ private DefaultSdkMetric(String name, Class clzz, Set categor /** * @return The name of this event. */ + @Override public String name() { return name; } @@ -55,13 +59,20 @@ public String name() { /** * @return The categories of this event. */ + @Override public Set categories() { return Collections.unmodifiableSet(categories); } + @Override + public MetricLevel level() { + return level; + } + /** * @return The class of the value associated with this event. */ + @Override public Class valueClass() { return clzz; } @@ -106,13 +117,14 @@ public String toString() { * * @throws IllegalArgumentException If a metric of the same name has already been created. */ - public static SdkMetric create(String name, Class clzz, MetricCategory c1, MetricCategory... cn) { + public static SdkMetric create(String name, Class clzz, MetricLevel level, + MetricCategory c1, MetricCategory... cn) { Stream categoryStream = Stream.of(c1); if (cn != null) { categoryStream = Stream.concat(categoryStream, Stream.of(cn)); } Set categories = categoryStream.collect(Collectors.toSet()); - return create(name, clzz, categories); + return create(name, clzz, level, categories); } /** @@ -126,9 +138,9 @@ public static SdkMetric create(String name, Class clzz, MetricCategory * * @throws IllegalArgumentException If a metric of the same name has already been created. */ - public static SdkMetric create(String name, Class clzz, Set categories) { + public static SdkMetric create(String name, Class clzz, MetricLevel level, Set categories) { Validate.noNullElements(categories, "categories must not contain null elements"); - SdkMetric event = new DefaultSdkMetric<>(name, clzz, categories); + SdkMetric event = new DefaultSdkMetric<>(name, clzz, level, categories); if (SDK_METRICS.putIfAbsent(event, Boolean.TRUE) != null) { throw new IllegalArgumentException("Metric with name " + name + " has already been created"); } diff --git a/core/metrics-spi/src/test/java/software/amazon/awssdk/metrics/MetricLevelTest.java b/core/metrics-spi/src/test/java/software/amazon/awssdk/metrics/MetricLevelTest.java new file mode 100644 index 000000000000..317538e32b16 --- /dev/null +++ b/core/metrics-spi/src/test/java/software/amazon/awssdk/metrics/MetricLevelTest.java @@ -0,0 +1,43 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.metrics; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.Test; + +public class MetricLevelTest { + @Test + public void allLevelsAreCorrect() { + assertThat(MetricLevel.TRACE.includesLevel(MetricLevel.TRACE)).isTrue(); + assertThat(MetricLevel.TRACE.includesLevel(MetricLevel.INFO)).isTrue(); + assertThat(MetricLevel.TRACE.includesLevel(MetricLevel.ERROR)).isTrue(); + } + + @Test + public void infoLevelsAreCorrect() { + assertThat(MetricLevel.INFO.includesLevel(MetricLevel.TRACE)).isFalse(); + assertThat(MetricLevel.INFO.includesLevel(MetricLevel.INFO)).isTrue(); + assertThat(MetricLevel.INFO.includesLevel(MetricLevel.ERROR)).isTrue(); + } + + @Test + public void errorLevelsAreCorrect() { + assertThat(MetricLevel.ERROR.includesLevel(MetricLevel.TRACE)).isFalse(); + assertThat(MetricLevel.ERROR.includesLevel(MetricLevel.INFO)).isFalse(); + assertThat(MetricLevel.ERROR.includesLevel(MetricLevel.ERROR)).isTrue(); + } +} \ No newline at end of file diff --git a/core/metrics-spi/src/test/java/software/amazon/awssdk/metrics/internal/DefaultMetricCollectionTest.java b/core/metrics-spi/src/test/java/software/amazon/awssdk/metrics/internal/DefaultMetricCollectionTest.java index 0eb28a7e56b9..65d168b4e1a7 100644 --- a/core/metrics-spi/src/test/java/software/amazon/awssdk/metrics/internal/DefaultMetricCollectionTest.java +++ b/core/metrics-spi/src/test/java/software/amazon/awssdk/metrics/internal/DefaultMetricCollectionTest.java @@ -16,6 +16,7 @@ package software.amazon.awssdk.metrics.internal; import static org.assertj.core.api.Assertions.assertThat; + import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -27,11 +28,12 @@ import org.junit.AfterClass; import org.junit.Test; import software.amazon.awssdk.metrics.MetricCategory; +import software.amazon.awssdk.metrics.MetricLevel; import software.amazon.awssdk.metrics.MetricRecord; import software.amazon.awssdk.metrics.SdkMetric; public class DefaultMetricCollectionTest { - private static final SdkMetric M1 = SdkMetric.create("m1", Integer.class, MetricCategory.DEFAULT); + private static final SdkMetric M1 = SdkMetric.create("m1", Integer.class, MetricLevel.INFO, MetricCategory.CORE); @AfterClass public static void teardown() { diff --git a/core/metrics-spi/src/test/java/software/amazon/awssdk/metrics/internal/DefaultMetricCollectorTest.java b/core/metrics-spi/src/test/java/software/amazon/awssdk/metrics/internal/DefaultMetricCollectorTest.java index 854a55ea8977..d3f0682d6c8d 100644 --- a/core/metrics-spi/src/test/java/software/amazon/awssdk/metrics/internal/DefaultMetricCollectorTest.java +++ b/core/metrics-spi/src/test/java/software/amazon/awssdk/metrics/internal/DefaultMetricCollectorTest.java @@ -24,10 +24,11 @@ import software.amazon.awssdk.metrics.MetricCategory; import software.amazon.awssdk.metrics.MetricCollection; import software.amazon.awssdk.metrics.MetricCollector; +import software.amazon.awssdk.metrics.MetricLevel; import software.amazon.awssdk.metrics.SdkMetric; public class DefaultMetricCollectorTest { - private static final SdkMetric M1 = SdkMetric.create("m1", Integer.class, MetricCategory.DEFAULT); + private static final SdkMetric M1 = SdkMetric.create("m1", Integer.class, MetricLevel.INFO, MetricCategory.CORE); @Rule public ExpectedException thrown = ExpectedException.none(); diff --git a/core/metrics-spi/src/test/java/software/amazon/awssdk/metrics/internal/DefaultSdkMetricRecordTest.java b/core/metrics-spi/src/test/java/software/amazon/awssdk/metrics/internal/DefaultSdkMetricRecordTest.java index fc64e6b64190..a6a2fbbc18d6 100644 --- a/core/metrics-spi/src/test/java/software/amazon/awssdk/metrics/internal/DefaultSdkMetricRecordTest.java +++ b/core/metrics-spi/src/test/java/software/amazon/awssdk/metrics/internal/DefaultSdkMetricRecordTest.java @@ -18,6 +18,7 @@ import static org.assertj.core.api.Assertions.assertThat; import org.junit.Test; import software.amazon.awssdk.metrics.MetricCategory; +import software.amazon.awssdk.metrics.MetricLevel; import software.amazon.awssdk.metrics.SdkMetric; import software.amazon.awssdk.metrics.MetricRecord; @@ -27,7 +28,7 @@ public class DefaultSdkMetricRecordTest { @Test public void testGetters() { - SdkMetric event = SdkMetric.create("foo", Integer.class, MetricCategory.DEFAULT); + SdkMetric event = SdkMetric.create("foo", Integer.class, MetricLevel.INFO, MetricCategory.CORE); MetricRecord record = new DefaultMetricRecord<>(event, 2); diff --git a/core/metrics-spi/src/test/java/software/amazon/awssdk/metrics/internal/DefaultSdkMetricTest.java b/core/metrics-spi/src/test/java/software/amazon/awssdk/metrics/internal/DefaultSdkMetricTest.java index 003e7b716ffc..1fe8d4fbea1a 100644 --- a/core/metrics-spi/src/test/java/software/amazon/awssdk/metrics/internal/DefaultSdkMetricTest.java +++ b/core/metrics-spi/src/test/java/software/amazon/awssdk/metrics/internal/DefaultSdkMetricTest.java @@ -16,6 +16,7 @@ package software.amazon.awssdk.metrics.internal; import static org.assertj.core.api.Assertions.assertThat; + import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -24,6 +25,7 @@ import org.junit.Test; import org.junit.rules.ExpectedException; import software.amazon.awssdk.metrics.MetricCategory; +import software.amazon.awssdk.metrics.MetricLevel; import software.amazon.awssdk.metrics.SdkMetric; public class DefaultSdkMetricTest { @@ -37,19 +39,19 @@ public void methodSetup() { @Test public void testOf_variadicOverload_createdProperly() { - SdkMetric event = SdkMetric.create("event", Integer.class, MetricCategory.DEFAULT); + SdkMetric event = SdkMetric.create("event", Integer.class, MetricLevel.INFO, MetricCategory.CORE); - assertThat(event.categories()).containsExactly(MetricCategory.DEFAULT); + assertThat(event.categories()).containsExactly(MetricCategory.CORE); assertThat(event.name()).isEqualTo("event"); assertThat(event.valueClass()).isEqualTo(Integer.class); } @Test public void testOf_setOverload_createdProperly() { - SdkMetric event = SdkMetric.create("event", Integer.class, Stream.of(MetricCategory.DEFAULT) + SdkMetric event = SdkMetric.create("event", Integer.class, MetricLevel.INFO, Stream.of(MetricCategory.CORE) .collect(Collectors.toSet())); - assertThat(event.categories()).containsExactly(MetricCategory.DEFAULT); + assertThat(event.categories()).containsExactly(MetricCategory.CORE); assertThat(event.name()).isEqualTo("event"); assertThat(event.valueClass()).isEqualTo(Integer.class); } @@ -58,33 +60,33 @@ public void testOf_setOverload_createdProperly() { public void testOf_variadicOverload_c1Null_throws() { thrown.expect(IllegalArgumentException.class); thrown.expectMessage("must not contain null elements"); - SdkMetric.create("event", Integer.class, (MetricCategory) null); + SdkMetric.create("event", Integer.class, MetricLevel.INFO, (MetricCategory) null); } @Test public void testOf_variadicOverload_c1NotNull_cnNull_doesNotThrow() { - SdkMetric.create("event", Integer.class, MetricCategory.DEFAULT, null); + SdkMetric.create("event", Integer.class, MetricLevel.INFO, MetricCategory.CORE, null); } @Test public void testOf_variadicOverload_cnContainsNull_throws() { thrown.expect(IllegalArgumentException.class); thrown.expectMessage("must not contain null elements"); - SdkMetric.create("event", Integer.class, MetricCategory.DEFAULT, new MetricCategory[]{ null }); + SdkMetric.create("event", Integer.class, MetricLevel.INFO, MetricCategory.CORE, new MetricCategory[]{null }); } @Test public void testOf_setOverload_null_throws() { thrown.expect(NullPointerException.class); thrown.expectMessage("object is null"); - SdkMetric.create("event", Integer.class, (Set) null); + SdkMetric.create("event", Integer.class, MetricLevel.INFO, (Set) null); } @Test public void testOf_setOverload_nullElement_throws() { thrown.expect(IllegalArgumentException.class); thrown.expectMessage("categories must not contain null elements"); - SdkMetric.create("event", Integer.class, Stream.of((MetricCategory) null).collect(Collectors.toSet())); + SdkMetric.create("event", Integer.class, MetricLevel.INFO, Stream.of((MetricCategory) null).collect(Collectors.toSet())); } @Test @@ -94,8 +96,8 @@ public void testOf_namePreviouslyUsed_throws() { thrown.expect(IllegalArgumentException.class); thrown.expectMessage(fooName + " has already been created"); - SdkMetric.create(fooName, Integer.class, MetricCategory.DEFAULT); - SdkMetric.create(fooName, Integer.class, MetricCategory.DEFAULT); + SdkMetric.create(fooName, Integer.class, MetricLevel.INFO, MetricCategory.CORE); + SdkMetric.create(fooName, Integer.class, MetricLevel.INFO, MetricCategory.CORE); } @Test @@ -105,8 +107,8 @@ public void testOf_namePreviouslyUsed_differentArgs_throws() { thrown.expect(IllegalArgumentException.class); thrown.expectMessage(fooName + " has already been created"); - SdkMetric.create(fooName, Integer.class, MetricCategory.DEFAULT); - SdkMetric.create(fooName, Long.class, MetricCategory.STREAMING); + SdkMetric.create(fooName, Integer.class, MetricLevel.INFO, MetricCategory.CORE); + SdkMetric.create(fooName, Long.class, MetricLevel.INFO, MetricCategory.HTTP_CLIENT); } @Test @@ -116,9 +118,9 @@ public void testOf_namePreviouslyUsed_doesNotReplaceExisting() { thrown.expect(IllegalArgumentException.class); thrown.expectMessage(fooName + " has already been created"); - SdkMetric.create(fooName, Integer.class, MetricCategory.DEFAULT); + SdkMetric.create(fooName, Integer.class, MetricLevel.INFO, MetricCategory.CORE); try { - SdkMetric.create(fooName, Long.class, MetricCategory.STREAMING); + SdkMetric.create(fooName, Long.class, MetricLevel.INFO, MetricCategory.HTTP_CLIENT); } finally { SdkMetric fooMetric = DefaultSdkMetric.declaredEvents() .stream() @@ -128,7 +130,7 @@ public void testOf_namePreviouslyUsed_doesNotReplaceExisting() { assertThat(fooMetric.name()).isEqualTo(fooName); assertThat(fooMetric.valueClass()).isEqualTo(Integer.class); - assertThat(fooMetric.categories()).containsExactly(MetricCategory.DEFAULT); + assertThat(fooMetric.categories()).containsExactly(MetricCategory.CORE); } } } diff --git a/core/protocols/aws-json-protocol/src/main/java/software/amazon/awssdk/protocols/json/BaseAwsJsonProtocolFactory.java b/core/protocols/aws-json-protocol/src/main/java/software/amazon/awssdk/protocols/json/BaseAwsJsonProtocolFactory.java index b85f58246e43..1fd0a6b05670 100644 --- a/core/protocols/aws-json-protocol/src/main/java/software/amazon/awssdk/protocols/json/BaseAwsJsonProtocolFactory.java +++ b/core/protocols/aws-json-protocol/src/main/java/software/amazon/awssdk/protocols/json/BaseAwsJsonProtocolFactory.java @@ -31,6 +31,8 @@ import software.amazon.awssdk.core.client.config.SdkClientConfiguration; import software.amazon.awssdk.core.client.config.SdkClientOption; import software.amazon.awssdk.core.http.HttpResponseHandler; +import software.amazon.awssdk.core.http.MetricCollectingHttpResponseHandler; +import software.amazon.awssdk.core.metrics.CoreMetric; import software.amazon.awssdk.core.protocol.MarshallLocation; import software.amazon.awssdk.core.traits.TimestampFormatTrait; import software.amazon.awssdk.http.SdkHttpFullRequest; @@ -102,11 +104,12 @@ public final HttpResponseHandler createResponseHandler(Js public final HttpResponseHandler createResponseHandler( JsonOperationMetadata operationMetadata, Function pojoSupplier) { - return new AwsJsonResponseHandler<>( - new JsonResponseHandler<>(protocolUnmarshaller, - pojoSupplier, - operationMetadata.hasStreamingSuccessResponse(), - operationMetadata.isPayloadJson())); + return timeUnmarshalling( + new AwsJsonResponseHandler<>( + new JsonResponseHandler<>(protocolUnmarshaller, + pojoSupplier, + operationMetadata.hasStreamingSuccessResponse(), + operationMetadata.isPayloadJson()))); } /** @@ -114,7 +117,7 @@ public final HttpResponseHandler createResponseHandler( */ public final HttpResponseHandler createErrorResponseHandler( JsonOperationMetadata errorResponseMetadata) { - return AwsJsonProtocolErrorUnmarshaller + return timeUnmarshalling(AwsJsonProtocolErrorUnmarshaller .builder() .jsonProtocolUnmarshaller(protocolUnmarshaller) .exceptions(modeledExceptions) @@ -122,7 +125,11 @@ public final HttpResponseHandler createErrorResponseHandler .errorMessageParser(AwsJsonErrorMessageParser.DEFAULT_ERROR_MESSAGE_PARSER) .jsonFactory(getSdkFactory().getJsonFactory()) .defaultExceptionSupplier(defaultServiceExceptionSupplier) - .build(); + .build()); + } + + private MetricCollectingHttpResponseHandler timeUnmarshalling(HttpResponseHandler delegate) { + return MetricCollectingHttpResponseHandler.create(CoreMetric.UNMARSHALLING_DURATION, delegate); } private StructuredJsonGenerator createGenerator(OperationInfo operationInfo) { diff --git a/core/protocols/aws-json-protocol/src/main/java/software/amazon/awssdk/protocols/json/internal/unmarshall/AwsJsonProtocolErrorUnmarshaller.java b/core/protocols/aws-json-protocol/src/main/java/software/amazon/awssdk/protocols/json/internal/unmarshall/AwsJsonProtocolErrorUnmarshaller.java index 62f77d8f3804..ea832b762e8e 100644 --- a/core/protocols/aws-json-protocol/src/main/java/software/amazon/awssdk/protocols/json/internal/unmarshall/AwsJsonProtocolErrorUnmarshaller.java +++ b/core/protocols/aws-json-protocol/src/main/java/software/amazon/awssdk/protocols/json/internal/unmarshall/AwsJsonProtocolErrorUnmarshaller.java @@ -134,7 +134,7 @@ private AwsErrorDetails extractAwsErrorDetails(SdkHttpFullResponse response, } private String getRequestIdFromHeaders(Map> headers) { - return SdkHttpUtils.firstMatchingHeader(headers, X_AMZN_REQUEST_ID_HEADER).orElse(null); + return SdkHttpUtils.firstMatchingHeaderFromCollection(headers, X_AMZN_REQUEST_ID_HEADERS).orElse(null); } private String getExtendedRequestIdFromHeaders(Map> headers) { diff --git a/core/protocols/aws-json-protocol/src/main/java/software/amazon/awssdk/protocols/json/internal/unmarshall/AwsJsonResponseHandler.java b/core/protocols/aws-json-protocol/src/main/java/software/amazon/awssdk/protocols/json/internal/unmarshall/AwsJsonResponseHandler.java index 0c20eafc4d2f..7569bf3f8e36 100644 --- a/core/protocols/aws-json-protocol/src/main/java/software/amazon/awssdk/protocols/json/internal/unmarshall/AwsJsonResponseHandler.java +++ b/core/protocols/aws-json-protocol/src/main/java/software/amazon/awssdk/protocols/json/internal/unmarshall/AwsJsonResponseHandler.java @@ -27,6 +27,7 @@ import software.amazon.awssdk.core.interceptor.ExecutionAttributes; import software.amazon.awssdk.http.SdkHttpFullResponse; import software.amazon.awssdk.http.SdkHttpResponse; +import software.amazon.awssdk.utils.http.SdkHttpUtils; @SdkInternalApi public final class AwsJsonResponseHandler implements HttpResponseHandler { @@ -57,7 +58,9 @@ public T handle(SdkHttpFullResponse response, ExecutionAttributes executionAttri private AwsResponseMetadata generateResponseMetadata(SdkHttpResponse response) { Map metadata = new HashMap<>(); - metadata.put(AWS_REQUEST_ID, response.firstMatchingHeader(X_AMZN_REQUEST_ID_HEADER).orElse(null)); + metadata.put(AWS_REQUEST_ID, SdkHttpUtils.firstMatchingHeaderFromCollection(response.headers(), + X_AMZN_REQUEST_ID_HEADERS) + .orElse(null)); response.headers().forEach((key, value) -> metadata.put(key, value.get(0))); return DefaultAwsResponseMetadata.create(metadata); diff --git a/core/protocols/aws-query-protocol/src/main/java/software/amazon/awssdk/protocols/query/AwsQueryProtocolFactory.java b/core/protocols/aws-query-protocol/src/main/java/software/amazon/awssdk/protocols/query/AwsQueryProtocolFactory.java index ae2891123282..e7e791f555a9 100644 --- a/core/protocols/aws-query-protocol/src/main/java/software/amazon/awssdk/protocols/query/AwsQueryProtocolFactory.java +++ b/core/protocols/aws-query-protocol/src/main/java/software/amazon/awssdk/protocols/query/AwsQueryProtocolFactory.java @@ -28,6 +28,8 @@ import software.amazon.awssdk.core.client.config.SdkClientConfiguration; import software.amazon.awssdk.core.client.config.SdkClientOption; import software.amazon.awssdk.core.http.HttpResponseHandler; +import software.amazon.awssdk.core.http.MetricCollectingHttpResponseHandler; +import software.amazon.awssdk.core.metrics.CoreMetric; import software.amazon.awssdk.http.SdkHttpFullRequest; import software.amazon.awssdk.protocols.core.ExceptionMetadata; import software.amazon.awssdk.protocols.core.OperationInfo; @@ -47,20 +49,20 @@ public class AwsQueryProtocolFactory { private final SdkClientConfiguration clientConfiguration; private final List modeledExceptions; private final Supplier defaultServiceExceptionSupplier; - private final AwsXmlErrorProtocolUnmarshaller errorUnmarshaller; + private final MetricCollectingHttpResponseHandler errorUnmarshaller; AwsQueryProtocolFactory(Builder builder) { this.clientConfiguration = builder.clientConfiguration; this.modeledExceptions = unmodifiableList(builder.modeledExceptions); this.defaultServiceExceptionSupplier = builder.defaultServiceExceptionSupplier; - this.errorUnmarshaller = AwsXmlErrorProtocolUnmarshaller + this.errorUnmarshaller = timeUnmarshalling(AwsXmlErrorProtocolUnmarshaller .builder() .defaultExceptionSupplier(defaultServiceExceptionSupplier) .exceptions(modeledExceptions) // We don't set result wrapper since that's handled by the errorRootExtractor .errorUnmarshaller(QueryProtocolUnmarshaller.builder().build()) .errorRootExtractor(this::getErrorRoot) - .build(); + .build()); } /** @@ -86,10 +88,9 @@ public final ProtocolMarshaller createProtocolMarshaller( * @return New {@link HttpResponseHandler} for success responses. */ public final HttpResponseHandler createResponseHandler(Supplier pojoSupplier) { - return new AwsQueryResponseHandler<>(QueryProtocolUnmarshaller.builder() - .hasResultWrapper(!isEc2()) - .build(), - r -> pojoSupplier.get()); + return timeUnmarshalling(new AwsQueryResponseHandler<>(QueryProtocolUnmarshaller.builder() + .hasResultWrapper(!isEc2()) + .build(), r -> pojoSupplier.get())); } /** @@ -100,6 +101,10 @@ public final HttpResponseHandler createErrorResponseHandler return errorUnmarshaller; } + private MetricCollectingHttpResponseHandler timeUnmarshalling(HttpResponseHandler delegate) { + return MetricCollectingHttpResponseHandler.create(CoreMetric.UNMARSHALLING_DURATION, delegate); + } + /** * Extracts the element from the root XML document. Method is protected as EC2 has a slightly * different location. diff --git a/core/protocols/aws-query-protocol/src/main/java/software/amazon/awssdk/protocols/query/internal/unmarshall/AwsQueryResponseHandler.java b/core/protocols/aws-query-protocol/src/main/java/software/amazon/awssdk/protocols/query/internal/unmarshall/AwsQueryResponseHandler.java index 1df64688999b..bf0c23ad6fb9 100644 --- a/core/protocols/aws-query-protocol/src/main/java/software/amazon/awssdk/protocols/query/internal/unmarshall/AwsQueryResponseHandler.java +++ b/core/protocols/aws-query-protocol/src/main/java/software/amazon/awssdk/protocols/query/internal/unmarshall/AwsQueryResponseHandler.java @@ -32,6 +32,7 @@ import software.amazon.awssdk.http.SdkHttpResponse; import software.amazon.awssdk.utils.Logger; import software.amazon.awssdk.utils.Pair; +import software.amazon.awssdk.utils.http.SdkHttpUtils; /** * Response handler for AWS/Query services and Amazon EC2 which is a dialect of the Query protocol. @@ -84,7 +85,8 @@ private T unmarshallResponse(SdkHttpFullResponse response) throws Exception { private AwsResponseMetadata generateResponseMetadata(SdkHttpResponse response, Map metadata) { if (!metadata.containsKey(AWS_REQUEST_ID)) { metadata.put(AWS_REQUEST_ID, - response.firstMatchingHeader(X_AMZN_REQUEST_ID_HEADER).orElse(null)); + SdkHttpUtils.firstMatchingHeaderFromCollection(response.headers(), X_AMZN_REQUEST_ID_HEADERS) + .orElse(null)); } response.headers().forEach((key, value) -> metadata.put(key, value.get(0))); diff --git a/core/protocols/aws-xml-protocol/src/main/java/software/amazon/awssdk/protocols/xml/AwsXmlProtocolFactory.java b/core/protocols/aws-xml-protocol/src/main/java/software/amazon/awssdk/protocols/xml/AwsXmlProtocolFactory.java index c771e82f7193..296ba2483e77 100644 --- a/core/protocols/aws-xml-protocol/src/main/java/software/amazon/awssdk/protocols/xml/AwsXmlProtocolFactory.java +++ b/core/protocols/aws-xml-protocol/src/main/java/software/amazon/awssdk/protocols/xml/AwsXmlProtocolFactory.java @@ -30,7 +30,9 @@ import software.amazon.awssdk.core.client.config.SdkClientConfiguration; import software.amazon.awssdk.core.client.config.SdkClientOption; import software.amazon.awssdk.core.http.HttpResponseHandler; +import software.amazon.awssdk.core.http.MetricCollectingHttpResponseHandler; import software.amazon.awssdk.core.internal.http.CombinedResponseHandler; +import software.amazon.awssdk.core.metrics.CoreMetric; import software.amazon.awssdk.http.SdkHttpFullRequest; import software.amazon.awssdk.protocols.core.ExceptionMetadata; import software.amazon.awssdk.protocols.core.OperationInfo; @@ -72,20 +74,21 @@ public class AwsXmlProtocolFactory { private final List modeledExceptions; private final Supplier defaultServiceExceptionSupplier; - private final AwsXmlErrorProtocolUnmarshaller errorUnmarshaller; + private final HttpResponseHandler errorUnmarshaller; private final SdkClientConfiguration clientConfiguration; AwsXmlProtocolFactory(Builder builder) { this.modeledExceptions = unmodifiableList(builder.modeledExceptions); this.defaultServiceExceptionSupplier = builder.defaultServiceExceptionSupplier; this.clientConfiguration = builder.clientConfiguration; - this.errorUnmarshaller = AwsXmlErrorProtocolUnmarshaller - .builder() - .defaultExceptionSupplier(defaultServiceExceptionSupplier) - .exceptions(modeledExceptions) - .errorUnmarshaller(XML_PROTOCOL_UNMARSHALLER) - .errorRootExtractor(this::getErrorRoot) - .build(); + + this.errorUnmarshaller = timeUnmarshalling( + AwsXmlErrorProtocolUnmarshaller.builder() + .defaultExceptionSupplier(defaultServiceExceptionSupplier) + .exceptions(modeledExceptions) + .errorUnmarshaller(XML_PROTOCOL_UNMARSHALLER) + .errorRootExtractor(this::getErrorRoot) + .build()); } /** @@ -103,9 +106,8 @@ public ProtocolMarshaller createProtocolMarshaller(Operation public HttpResponseHandler createResponseHandler(Supplier pojoSupplier, XmlOperationMetadata staxOperationMetadata) { - return new AwsXmlResponseHandler<>( - XML_PROTOCOL_UNMARSHALLER, r -> pojoSupplier.get(), - staxOperationMetadata.isHasStreamingSuccessResponse()); + return timeUnmarshalling(new AwsXmlResponseHandler<>(XML_PROTOCOL_UNMARSHALLER, r -> pojoSupplier.get(), + staxOperationMetadata.isHasStreamingSuccessResponse())); } protected Function createResponseTransformer( @@ -127,6 +129,10 @@ public HttpResponseHandler createErrorResponseHandler() { return errorUnmarshaller; } + private MetricCollectingHttpResponseHandler timeUnmarshalling(HttpResponseHandler delegate) { + return MetricCollectingHttpResponseHandler.create(CoreMetric.UNMARSHALLING_DURATION, delegate); + } + public HttpResponseHandler> createCombinedResponseHandler( Supplier pojoSupplier, XmlOperationMetadata staxOperationMetadata) { diff --git a/core/protocols/aws-xml-protocol/src/main/java/software/amazon/awssdk/protocols/xml/internal/unmarshall/AwsXmlResponseHandler.java b/core/protocols/aws-xml-protocol/src/main/java/software/amazon/awssdk/protocols/xml/internal/unmarshall/AwsXmlResponseHandler.java index 71a8b7ae57bd..0a81b613928a 100644 --- a/core/protocols/aws-xml-protocol/src/main/java/software/amazon/awssdk/protocols/xml/internal/unmarshall/AwsXmlResponseHandler.java +++ b/core/protocols/aws-xml-protocol/src/main/java/software/amazon/awssdk/protocols/xml/internal/unmarshall/AwsXmlResponseHandler.java @@ -32,6 +32,7 @@ import software.amazon.awssdk.http.SdkHttpFullResponse; import software.amazon.awssdk.http.SdkHttpResponse; import software.amazon.awssdk.utils.Logger; +import software.amazon.awssdk.utils.http.SdkHttpUtils; /** * Response handler for REST-XML services (Cloudfront, Route53, and S3). @@ -92,7 +93,7 @@ private T unmarshallResponse(SdkHttpFullResponse response) throws Exception { private AwsResponseMetadata generateResponseMetadata(SdkHttpResponse response) { Map metadata = new HashMap<>(); metadata.put(AWS_REQUEST_ID, - response.firstMatchingHeader(X_AMZN_REQUEST_ID_HEADER).orElse(null)); + SdkHttpUtils.firstMatchingHeaderFromCollection(response.headers(), X_AMZN_REQUEST_ID_HEADERS).orElse(null)); response.headers().forEach((key, value) -> metadata.put(key, value.get(0))); return DefaultAwsResponseMetadata.create(metadata); diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/http/HttpResponseHandler.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/http/HttpResponseHandler.java index 494d261209c8..1bf32ee01615 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/http/HttpResponseHandler.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/http/HttpResponseHandler.java @@ -15,6 +15,10 @@ package software.amazon.awssdk.core.http; +import java.util.Collections; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; import software.amazon.awssdk.annotations.SdkProtectedApi; import software.amazon.awssdk.core.interceptor.ExecutionAttributes; import software.amazon.awssdk.http.SdkHttpFullResponse; @@ -30,9 +34,11 @@ @SdkProtectedApi @FunctionalInterface public interface HttpResponseHandler { - String X_AMZN_REQUEST_ID_HEADER = "x-amzn-RequestId"; - + String X_AMZN_REQUEST_ID_HEADER_ALTERNATE = "x-amz-request-id"; + Set X_AMZN_REQUEST_ID_HEADERS = Collections.unmodifiableSet(Stream.of(X_AMZN_REQUEST_ID_HEADER, + X_AMZN_REQUEST_ID_HEADER_ALTERNATE) + .collect(Collectors.toSet())); String X_AMZ_ID_2_HEADER = "x-amz-id-2"; /** diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/http/MetricCollectingHttpResponseHandler.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/http/MetricCollectingHttpResponseHandler.java new file mode 100644 index 000000000000..5a801c865bd4 --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/http/MetricCollectingHttpResponseHandler.java @@ -0,0 +1,71 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.http; + +import java.time.Duration; +import java.util.Optional; +import software.amazon.awssdk.annotations.SdkProtectedApi; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.SdkExecutionAttribute; +import software.amazon.awssdk.core.internal.util.MetricUtils; +import software.amazon.awssdk.http.SdkHttpFullResponse; +import software.amazon.awssdk.metrics.MetricCollector; +import software.amazon.awssdk.metrics.SdkMetric; +import software.amazon.awssdk.utils.Pair; + +/** + * An implementation of {@link HttpResponseHandler} that publishes the time it took to execute { + * @link #handle(SdkHttpFullResponse, ExecutionAttributes)} as the provieded duration metric to the + * {@link SdkExecutionAttribute#API_CALL_ATTEMPT_METRIC_COLLECTOR}. + */ +@SdkProtectedApi +public final class MetricCollectingHttpResponseHandler implements HttpResponseHandler { + public final SdkMetric metric; + public final HttpResponseHandler delegateToTime; + + private MetricCollectingHttpResponseHandler(SdkMetric durationMetric, + HttpResponseHandler delegateToTime) { + this.metric = durationMetric; + this.delegateToTime = delegateToTime; + } + + public static MetricCollectingHttpResponseHandler create(SdkMetric durationMetric, + HttpResponseHandler delegateToTime) { + return new MetricCollectingHttpResponseHandler<>(durationMetric, delegateToTime); + } + + @Override + public T handle(SdkHttpFullResponse response, ExecutionAttributes executionAttributes) throws Exception { + Pair result = MetricUtils.measureDurationUnsafe(() -> delegateToTime.handle(response, executionAttributes)); + + collector(executionAttributes).ifPresent(c -> c.reportMetric(metric, result.right())); + + return result.left(); + } + + private Optional collector(ExecutionAttributes attributes) { + if (attributes == null) { + return Optional.empty(); + } + + return Optional.ofNullable(attributes.getAttribute(SdkExecutionAttribute.API_CALL_ATTEMPT_METRIC_COLLECTOR)); + } + + @Override + public boolean needsConnectionLeftOpen() { + return delegateToTime.needsConnectionLeftOpen(); + } +} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/interceptor/SdkExecutionAttribute.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/interceptor/SdkExecutionAttribute.java index 37b83a6a893e..6aa85ae6c70f 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/interceptor/SdkExecutionAttribute.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/interceptor/SdkExecutionAttribute.java @@ -19,6 +19,7 @@ import software.amazon.awssdk.core.ClientType; import software.amazon.awssdk.core.ServiceConfiguration; import software.amazon.awssdk.core.signer.Signer; +import software.amazon.awssdk.metrics.MetricCollector; /** * Contains attributes attached to the execution. This information is available to {@link ExecutionInterceptor}s and @@ -46,6 +47,13 @@ public class SdkExecutionAttribute { public static final ExecutionAttribute OPERATION_NAME = new ExecutionAttribute<>("OperationName"); + /** + * The {@link MetricCollector} associated with the current, ongoing API call attempt. This is not set until the actual + * internal API call attempt starts. + */ + public static final ExecutionAttribute API_CALL_ATTEMPT_METRIC_COLLECTOR = + new ExecutionAttribute<>("ApiCallAttemptMetricCollector"); + /** * If true indicates that the configured endpoint of the client is a value that was supplied as an override and not * generated from regional metadata. diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/handler/BaseAsyncClientHandler.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/handler/BaseAsyncClientHandler.java index 925d2750a1b4..2006288de580 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/handler/BaseAsyncClientHandler.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/handler/BaseAsyncClientHandler.java @@ -20,6 +20,7 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.function.Function; +import java.util.function.Supplier; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.Response; import software.amazon.awssdk.core.SdkRequest; @@ -44,9 +45,11 @@ import software.amazon.awssdk.core.internal.http.async.AsyncStreamingResponseHandler; import software.amazon.awssdk.core.internal.http.async.CombinedResponseAsyncHttpResponseHandler; import software.amazon.awssdk.core.internal.util.ThrowableUtils; +import software.amazon.awssdk.core.metrics.CoreMetric; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.http.SdkHttpFullRequest; import software.amazon.awssdk.http.SdkHttpFullResponse; +import software.amazon.awssdk.metrics.MetricCollector; import software.amazon.awssdk.utils.CompletableFutureUtils; import software.amazon.awssdk.utils.Logger; @@ -69,21 +72,23 @@ protected BaseAsyncClientHandler(SdkClientConfiguration clientConfiguration, public CompletableFuture execute( ClientExecutionParams executionParams) { - validateExecutionParams(executionParams); - ExecutionContext executionContext = createExecutionContext(executionParams, createInitialExecutionAttributes()); - TransformingAsyncResponseHandler> combinedResponseHandler; - - /* Decorate and combine provided response handlers into a single decorated response handler */ - if (executionParams.getCombinedResponseHandler() == null) { - combinedResponseHandler = createDecoratedHandler(executionParams.getResponseHandler(), - executionParams.getErrorResponseHandler(), - executionContext); - } else { - combinedResponseHandler = createDecoratedHandler(executionParams.getCombinedResponseHandler(), - executionContext); - } + return measureApiCallSuccess(executionParams, () -> { + validateExecutionParams(executionParams); + ExecutionContext executionContext = createExecutionContext(executionParams, createInitialExecutionAttributes()); + TransformingAsyncResponseHandler> combinedResponseHandler; + + /* Decorate and combine provided response handlers into a single decorated response handler */ + if (executionParams.getCombinedResponseHandler() == null) { + combinedResponseHandler = createDecoratedHandler(executionParams.getResponseHandler(), + executionParams.getErrorResponseHandler(), + executionContext); + } else { + combinedResponseHandler = createDecoratedHandler(executionParams.getCombinedResponseHandler(), + executionContext); + } - return doExecute(executionParams, executionContext, combinedResponseHandler); + return doExecute(executionParams, executionContext, combinedResponseHandler); + }); } @Override @@ -91,46 +96,48 @@ public Complet ClientExecutionParams executionParams, AsyncResponseTransformer asyncResponseTransformer) { - validateExecutionParams(executionParams); + return measureApiCallSuccess(executionParams, () -> { + validateExecutionParams(executionParams); - if (executionParams.getCombinedResponseHandler() != null) { - // There is no support for catching errors in a body for streaming responses. Our codegen must never - // attempt to do this. - throw new IllegalArgumentException("A streaming 'asyncResponseTransformer' may not be used when a " - + "'combinedResponseHandler' has been specified in a " - + "ClientExecutionParams object."); - } + if (executionParams.getCombinedResponseHandler() != null) { + // There is no support for catching errors in a body for streaming responses. Our codegen must never + // attempt to do this. + throw new IllegalArgumentException("A streaming 'asyncResponseTransformer' may not be used when a " + + "'combinedResponseHandler' has been specified in a " + + "ClientExecutionParams object."); + } - ExecutionAttributes executionAttributes = createInitialExecutionAttributes(); + ExecutionAttributes executionAttributes = createInitialExecutionAttributes(); - AsyncStreamingResponseHandler asyncStreamingResponseHandler = - new AsyncStreamingResponseHandler<>(asyncResponseTransformer); + AsyncStreamingResponseHandler asyncStreamingResponseHandler = + new AsyncStreamingResponseHandler<>(asyncResponseTransformer); - // For streaming requests, prepare() should be called as early as possible to avoid NPE in client - // See https://github.com/aws/aws-sdk-java-v2/issues/1268. We do this with a wrapper that caches the prepare - // result until the execution attempt number changes. This guarantees that prepare is only called once per - // execution. - TransformingAsyncResponseHandler wrappedAsyncStreamingResponseHandler = - IdempotentAsyncResponseHandler.create( - asyncStreamingResponseHandler, - () -> executionAttributes.getAttribute(InternalCoreExecutionAttribute.EXECUTION_ATTEMPT), - Integer::equals); - wrappedAsyncStreamingResponseHandler.prepare(); + // For streaming requests, prepare() should be called as early as possible to avoid NPE in client + // See https://github.com/aws/aws-sdk-java-v2/issues/1268. We do this with a wrapper that caches the prepare + // result until the execution attempt number changes. This guarantees that prepare is only called once per + // execution. + TransformingAsyncResponseHandler wrappedAsyncStreamingResponseHandler = + IdempotentAsyncResponseHandler.create( + asyncStreamingResponseHandler, + () -> executionAttributes.getAttribute(InternalCoreExecutionAttribute.EXECUTION_ATTEMPT), + Integer::equals); + wrappedAsyncStreamingResponseHandler.prepare(); - ExecutionContext context = createExecutionContext(executionParams, executionAttributes); + ExecutionContext context = createExecutionContext(executionParams, executionAttributes); - HttpResponseHandler decoratedResponseHandlers = - decorateResponseHandlers(executionParams.getResponseHandler(), context); + HttpResponseHandler decoratedResponseHandlers = + decorateResponseHandlers(executionParams.getResponseHandler(), context); - asyncStreamingResponseHandler.responseHandler(decoratedResponseHandlers); + asyncStreamingResponseHandler.responseHandler(decoratedResponseHandlers); - TransformingAsyncResponseHandler errorHandler = - resolveErrorResponseHandler(executionParams.getErrorResponseHandler(), context, crc32Validator); + TransformingAsyncResponseHandler errorHandler = + resolveErrorResponseHandler(executionParams.getErrorResponseHandler(), context, crc32Validator); - TransformingAsyncResponseHandler> combinedResponseHandler = - new CombinedResponseAsyncHttpResponseHandler<>(wrappedAsyncStreamingResponseHandler, errorHandler); + TransformingAsyncResponseHandler> combinedResponseHandler = + new CombinedResponseAsyncHttpResponseHandler<>(wrappedAsyncStreamingResponseHandler, errorHandler); - return doExecute(executionParams, context, combinedResponseHandler); + return doExecute(executionParams, context, combinedResponseHandler); + }); } /** @@ -261,4 +268,28 @@ private CompletableFuture invoke( .executionContext(executionContext) .execute(responseHandler); } + + private CompletableFuture measureApiCallSuccess(ClientExecutionParams executionParams, + Supplier> apiCall) { + try { + CompletableFuture apiCallResult = apiCall.get(); + CompletableFuture outputFuture = + apiCallResult.whenComplete((r, t) -> reportApiCallSuccess(executionParams, t == null)); + + // Preserve cancellations on the output future, by passing cancellations of the output future to the api call future. + CompletableFutureUtils.forwardExceptionTo(outputFuture, apiCallResult); + + return outputFuture; + } catch (Exception e) { + reportApiCallSuccess(executionParams, false); + throw e; + } + } + + private void reportApiCallSuccess(ClientExecutionParams executionParams, boolean value) { + MetricCollector metricCollector = executionParams.getMetricCollector(); + if (metricCollector != null) { + metricCollector.reportMetric(CoreMetric.API_CALL_SUCCESSFUL, value); + } + } } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/handler/BaseSyncClientHandler.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/handler/BaseSyncClientHandler.java index 33b868042b9d..e035e844a753 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/handler/BaseSyncClientHandler.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/handler/BaseSyncClientHandler.java @@ -16,6 +16,7 @@ package software.amazon.awssdk.core.internal.handler; import java.util.Optional; +import java.util.function.Supplier; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.Response; import software.amazon.awssdk.core.SdkRequest; @@ -33,11 +34,13 @@ import software.amazon.awssdk.core.internal.http.AmazonSyncHttpClient; import software.amazon.awssdk.core.internal.http.CombinedResponseHandler; import software.amazon.awssdk.core.internal.http.InterruptMonitor; +import software.amazon.awssdk.core.metrics.CoreMetric; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.core.sync.ResponseTransformer; import software.amazon.awssdk.http.AbortableInputStream; import software.amazon.awssdk.http.SdkHttpFullRequest; import software.amazon.awssdk.http.SdkHttpFullResponse; +import software.amazon.awssdk.metrics.MetricCollector; @SdkInternalApi public abstract class BaseSyncClientHandler extends BaseClientHandler implements SyncClientHandler { @@ -56,49 +59,53 @@ public ReturnT ClientExecutionParams executionParams, ResponseTransformer responseTransformer) { - validateExecutionParams(executionParams); + return measureApiCallSuccess(executionParams, () -> { + validateExecutionParams(executionParams); - if (executionParams.getCombinedResponseHandler() != null) { - // There is no support for catching errors in a body for streaming responses - throw new IllegalArgumentException("A streaming 'responseTransformer' may not be used when a " - + "'combinedResponseHandler' has been specified in a " - + "ClientExecutionParams object."); - } + if (executionParams.getCombinedResponseHandler() != null) { + // There is no support for catching errors in a body for streaming responses + throw new IllegalArgumentException("A streaming 'responseTransformer' may not be used when a " + + "'combinedResponseHandler' has been specified in a " + + "ClientExecutionParams object."); + } - ExecutionContext executionContext = createExecutionContext(executionParams, createInitialExecutionAttributes()); + ExecutionContext executionContext = createExecutionContext(executionParams, createInitialExecutionAttributes()); - HttpResponseHandler decoratedResponseHandlers = - decorateResponseHandlers(executionParams.getResponseHandler(), executionContext); + HttpResponseHandler decoratedResponseHandlers = + decorateResponseHandlers(executionParams.getResponseHandler(), executionContext); - HttpResponseHandler httpResponseHandler = - new HttpResponseHandlerAdapter<>(decoratedResponseHandlers, responseTransformer); + HttpResponseHandler httpResponseHandler = + new HttpResponseHandlerAdapter<>(decoratedResponseHandlers, responseTransformer); - return doExecute( - executionParams, - executionContext, - new CombinedResponseHandler<>(httpResponseHandler, executionParams.getErrorResponseHandler())); + return doExecute( + executionParams, + executionContext, + new CombinedResponseHandler<>(httpResponseHandler, executionParams.getErrorResponseHandler())); + }); } @Override public OutputT execute( ClientExecutionParams executionParams) { - validateExecutionParams(executionParams); - ExecutionContext executionContext = createExecutionContext(executionParams, createInitialExecutionAttributes()); - HttpResponseHandler> combinedResponseHandler; + return measureApiCallSuccess(executionParams, () -> { + validateExecutionParams(executionParams); + ExecutionContext executionContext = createExecutionContext(executionParams, createInitialExecutionAttributes()); + HttpResponseHandler> combinedResponseHandler; - if (executionParams.getCombinedResponseHandler() != null) { - combinedResponseHandler = decorateSuccessResponseHandlers(executionParams.getCombinedResponseHandler(), - executionContext); - } else { - HttpResponseHandler decoratedResponseHandlers = - decorateResponseHandlers(executionParams.getResponseHandler(), executionContext); + if (executionParams.getCombinedResponseHandler() != null) { + combinedResponseHandler = decorateSuccessResponseHandlers(executionParams.getCombinedResponseHandler(), + executionContext); + } else { + HttpResponseHandler decoratedResponseHandlers = + decorateResponseHandlers(executionParams.getResponseHandler(), executionContext); - combinedResponseHandler = new CombinedResponseHandler<>(decoratedResponseHandlers, - executionParams.getErrorResponseHandler()); - } + combinedResponseHandler = new CombinedResponseHandler<>(decoratedResponseHandlers, + executionParams.getErrorResponseHandler()); + } - return doExecute(executionParams, executionContext, combinedResponseHandler); + return doExecute(executionParams, executionContext, combinedResponseHandler); + }); } @Override @@ -150,6 +157,24 @@ private ReturnT doExecute( responseHandler); } + private T measureApiCallSuccess(ClientExecutionParams executionParams, Supplier thingToMeasureSuccessOf) { + try { + T result = thingToMeasureSuccessOf.get(); + reportApiCallSuccess(executionParams, true); + return result; + } catch (Exception e) { + reportApiCallSuccess(executionParams, false); + throw e; + } + } + + private void reportApiCallSuccess(ClientExecutionParams executionParams, boolean value) { + MetricCollector metricCollector = executionParams.getMetricCollector(); + if (metricCollector != null) { + metricCollector.reportMetric(CoreMetric.API_CALL_SUCCESSFUL, value); + } + } + private static class HttpResponseHandlerAdapter implements HttpResponseHandler { diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/RequestExecutionContext.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/RequestExecutionContext.java index d49f0cb0754f..73d9f2ea94f4 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/RequestExecutionContext.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/RequestExecutionContext.java @@ -23,6 +23,7 @@ import software.amazon.awssdk.core.http.ExecutionContext; import software.amazon.awssdk.core.interceptor.ExecutionAttributes; import software.amazon.awssdk.core.interceptor.ExecutionInterceptorChain; +import software.amazon.awssdk.core.interceptor.SdkExecutionAttribute; import software.amazon.awssdk.core.internal.http.pipeline.RequestPipeline; import software.amazon.awssdk.core.internal.http.timers.TimeoutTracker; import software.amazon.awssdk.core.signer.Signer; @@ -42,7 +43,7 @@ public final class RequestExecutionContext { private final ExecutionContext executionContext; private TimeoutTracker apiCallTimeoutTracker; private TimeoutTracker apiCallAttemptTimeoutTracker; - private MetricCollector metricCollector; + private MetricCollector attemptMetricCollector; private RequestExecutionContext(Builder builder) { this.requestProvider = builder.requestProvider; @@ -117,12 +118,13 @@ public void apiCallAttemptTimeoutTracker(TimeoutTracker timeoutTracker) { this.apiCallAttemptTimeoutTracker = timeoutTracker; } - public MetricCollector metricCollector() { - return metricCollector; + public MetricCollector attemptMetricCollector() { + return attemptMetricCollector; } - public void metricCollector(MetricCollector metricCollector) { - this.metricCollector = metricCollector; + public void attemptMetricCollector(MetricCollector metricCollector) { + executionAttributes().putAttribute(SdkExecutionAttribute.API_CALL_ATTEMPT_METRIC_COLLECTOR, metricCollector); + this.attemptMetricCollector = metricCollector; } /** diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ApiCallAttemptMetricCollectionStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ApiCallAttemptMetricCollectionStage.java index c1a90987b3a7..329b302ccba2 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ApiCallAttemptMetricCollectionStage.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ApiCallAttemptMetricCollectionStage.java @@ -18,11 +18,13 @@ import static software.amazon.awssdk.core.internal.util.MetricUtils.collectHttpMetrics; import static software.amazon.awssdk.core.internal.util.MetricUtils.createAttemptMetricsCollector; +import java.time.Duration; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.Response; import software.amazon.awssdk.core.internal.http.RequestExecutionContext; import software.amazon.awssdk.core.internal.http.pipeline.RequestPipeline; import software.amazon.awssdk.core.internal.http.pipeline.RequestToResponsePipeline; +import software.amazon.awssdk.core.internal.http.pipeline.stages.utils.RetryableStageHelper; import software.amazon.awssdk.core.metrics.CoreMetric; import software.amazon.awssdk.http.SdkHttpFullRequest; import software.amazon.awssdk.metrics.MetricCollector; @@ -42,21 +44,20 @@ public ApiCallAttemptMetricCollectionStage(RequestPipeline execute(SdkHttpFullRequest input, RequestExecutionContext context) throws Exception { MetricCollector apiCallAttemptMetrics = createAttemptMetricsCollector(context); - context.metricCollector(apiCallAttemptMetrics); + context.attemptMetricCollector(apiCallAttemptMetrics); + reportBackoffDelay(context); - try { - Response response = wrapped.execute(input, context); + Response response = wrapped.execute(input, context); - collectHttpMetrics(apiCallAttemptMetrics, response.httpResponse()); + collectHttpMetrics(apiCallAttemptMetrics, response.httpResponse()); - if (!response.isSuccess() && response.exception() != null) { - apiCallAttemptMetrics.reportMetric(CoreMetric.EXCEPTION, response.exception()); - } + return response; + } - return response; - } catch (Throwable t) { - apiCallAttemptMetrics.reportMetric(CoreMetric.EXCEPTION, t); - throw t; + private void reportBackoffDelay(RequestExecutionContext context) { + Duration lastBackoffDelay = context.executionAttributes().getAttribute(RetryableStageHelper.LAST_BACKOFF_DELAY_DURATION); + if (lastBackoffDelay != null) { + context.attemptMetricCollector().reportMetric(CoreMetric.BACKOFF_DELAY_DURATION, lastBackoffDelay); } } } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AsyncApiCallAttemptMetricCollectionStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AsyncApiCallAttemptMetricCollectionStage.java index 7ec0022e0f9f..ddb1c66643e2 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AsyncApiCallAttemptMetricCollectionStage.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AsyncApiCallAttemptMetricCollectionStage.java @@ -18,11 +18,13 @@ import static software.amazon.awssdk.core.internal.util.MetricUtils.collectHttpMetrics; import static software.amazon.awssdk.core.internal.util.MetricUtils.createAttemptMetricsCollector; +import java.time.Duration; import java.util.concurrent.CompletableFuture; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.Response; import software.amazon.awssdk.core.internal.http.RequestExecutionContext; import software.amazon.awssdk.core.internal.http.pipeline.RequestPipeline; +import software.amazon.awssdk.core.internal.http.pipeline.stages.utils.RetryableStageHelper; import software.amazon.awssdk.core.metrics.CoreMetric; import software.amazon.awssdk.http.SdkHttpFullRequest; import software.amazon.awssdk.metrics.MetricCollector; @@ -46,21 +48,24 @@ public CompletableFuture> execute(SdkHttpFullRequest input, RequestExecutionContext context) throws Exception { MetricCollector apiCallAttemptMetrics = createAttemptMetricsCollector(context); - context.metricCollector(apiCallAttemptMetrics); + context.attemptMetricCollector(apiCallAttemptMetrics); + reportBackoffDelay(context); CompletableFuture> executeFuture = wrapped.execute(input, context); executeFuture.whenComplete((r, t) -> { - if (t != null) { - apiCallAttemptMetrics.reportMetric(CoreMetric.EXCEPTION, t); - } else { + if (t == null) { collectHttpMetrics(apiCallAttemptMetrics, r.httpResponse()); - if (!r.isSuccess()) { - apiCallAttemptMetrics.reportMetric(CoreMetric.EXCEPTION, r.exception()); - } } }); return executeFuture; } + + private void reportBackoffDelay(RequestExecutionContext context) { + Duration lastBackoffDelay = context.executionAttributes().getAttribute(RetryableStageHelper.LAST_BACKOFF_DELAY_DURATION); + if (lastBackoffDelay != null) { + context.attemptMetricCollector().reportMetric(CoreMetric.BACKOFF_DELAY_DURATION, lastBackoffDelay); + } + } } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java index e30d9fdd03fb..31a38f1f294d 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java @@ -52,6 +52,7 @@ import software.amazon.awssdk.http.async.SdkAsyncHttpClient; import software.amazon.awssdk.http.async.SdkHttpContentPublisher; import software.amazon.awssdk.metrics.MetricCollector; +import software.amazon.awssdk.utils.CompletableFutureUtils; import software.amazon.awssdk.utils.Logger; /** @@ -180,16 +181,20 @@ private CompletableFuture> executeHttpRequest(SdkHttpFullReque } private CompletableFuture doExecuteHttpRequest(RequestExecutionContext context, AsyncExecuteRequest executeRequest) { - MetricCollector metricCollector = context.metricCollector(); + MetricCollector metricCollector = context.attemptMetricCollector(); long callStart = System.nanoTime(); CompletableFuture httpClientFuture = sdkAsyncHttpClient.execute(executeRequest); // Offload the metrics reporting from this stage onto the future completion executor - httpClientFuture.whenCompleteAsync((r, t) -> { + CompletableFuture result = httpClientFuture.whenComplete((r, t) -> { long duration = System.nanoTime() - callStart; - metricCollector.reportMetric(CoreMetric.HTTP_REQUEST_ROUND_TRIP_TIME, Duration.ofNanos(duration)); - }, futureCompletionExecutor); - return httpClientFuture; + metricCollector.reportMetric(CoreMetric.SERVICE_CALL_DURATION, Duration.ofNanos(duration)); + }); + + // Make sure failures on the result future are forwarded to the http client future. + CompletableFutureUtils.forwardExceptionTo(result, httpClientFuture); + + return result; } private boolean isFullDuplex(ExecutionAttributes executionAttributes) { diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeHttpRequestStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeHttpRequestStage.java index e9bdbc02f629..d93bc087c079 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeHttpRequestStage.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeHttpRequestStage.java @@ -59,7 +59,7 @@ public Pair execute(SdkHttpFullRequest } private HttpExecuteResponse executeHttpRequest(SdkHttpFullRequest request, RequestExecutionContext context) throws Exception { - MetricCollector metricCollector = context.metricCollector(); + MetricCollector attemptMetricCollector = context.attemptMetricCollector(); MetricCollector httpMetricCollector = MetricUtils.createHttpMetricsCollector(context); @@ -75,7 +75,7 @@ private HttpExecuteResponse executeHttpRequest(SdkHttpFullRequest request, Reque Pair measuredExecute = MetricUtils.measureDurationUnsafe(requestCallable); - metricCollector.reportMetric(CoreMetric.HTTP_REQUEST_ROUND_TRIP_TIME, measuredExecute.right()); + attemptMetricCollector.reportMetric(CoreMetric.SERVICE_CALL_DURATION, measuredExecute.right()); return measuredExecute.left(); } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/SigningStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/SigningStage.java index bd07e207494a..4c7479238a88 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/SigningStage.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/SigningStage.java @@ -61,7 +61,7 @@ private SdkHttpFullRequest signRequest(SdkHttpFullRequest request, RequestExecut updateInterceptorContext(request, context.executionContext()); Signer signer = context.signer(); - MetricCollector metricCollector = context.metricCollector(); + MetricCollector metricCollector = context.attemptMetricCollector(); if (shouldSign(signer)) { adjustForClockSkew(context.executionAttributes()); diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/utils/RetryableStageHelper.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/utils/RetryableStageHelper.java index e04de518f1a5..e16032fd1278 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/utils/RetryableStageHelper.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/utils/RetryableStageHelper.java @@ -26,12 +26,14 @@ import software.amazon.awssdk.core.exception.NonRetryableException; import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.core.interceptor.ExecutionAttribute; import software.amazon.awssdk.core.internal.InternalCoreExecutionAttribute; import software.amazon.awssdk.core.internal.http.HttpClientDependencies; import software.amazon.awssdk.core.internal.http.RequestExecutionContext; import software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncRetryableStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage; import software.amazon.awssdk.core.internal.retry.ClockSkewAdjuster; +import software.amazon.awssdk.core.metrics.CoreMetric; import software.amazon.awssdk.core.retry.RetryPolicy; import software.amazon.awssdk.core.retry.RetryPolicyContext; import software.amazon.awssdk.core.retry.RetryUtils; @@ -45,6 +47,9 @@ */ @SdkInternalApi public class RetryableStageHelper { + public static final ExecutionAttribute LAST_BACKOFF_DELAY_DURATION = + new ExecutionAttribute<>("LastBackoffDuration"); + private final SdkHttpFullRequest request; private final RequestExecutionContext context; private final RetryPolicy retryPolicy; @@ -53,7 +58,6 @@ public class RetryableStageHelper { private int attemptNumber = 0; private SdkHttpResponse lastResponse = null; private SdkException lastException = null; - private Duration lastBackoffDelay = null; public RetryableStageHelper(SdkHttpFullRequest request, RequestExecutionContext context, @@ -99,6 +103,7 @@ public boolean retryPolicyAllowsRetry() { * Return the exception that should be thrown, because the retry policy did not allow the request to be retried. */ public SdkException retryPolicyDisallowedRetryException() { + context.executionContext().metricCollector().reportMetric(CoreMetric.RETRY_COUNT, retriesAttemptedSoFar(true)); return lastException; } @@ -118,7 +123,7 @@ public Duration getBackoffDelay() { result = retryPolicy.backoffStrategy().computeDelayBeforeNextRetry(context); } } - lastBackoffDelay = result; + context.executionAttributes().putAttribute(LAST_BACKOFF_DELAY_DURATION, result); return result; } @@ -139,7 +144,7 @@ public SdkHttpFullRequest requestToSend() { .map(TokenBucketRetryCondition.Capacity::capacityRemaining) .orElse(null); String headerValue = (attemptNumber - 1) + "/" + - lastBackoffDelay.toMillis() + "/" + + context.executionAttributes().getAttribute(LAST_BACKOFF_DELAY_DURATION).toMillis() + "/" + (availableRetryCapacity != null ? availableRetryCapacity : ""); return request.toBuilder() .putHeader(SDK_RETRY_INFO_HEADER, headerValue) @@ -169,6 +174,7 @@ public void adjustClockIfClockSkew(Response response) { */ public void attemptSucceeded() { retryPolicy.aggregateRetryCondition().requestSucceeded(retryPolicyContext(false)); + context.executionContext().metricCollector().reportMetric(CoreMetric.RETRY_COUNT, retriesAttemptedSoFar(false)); } /** diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/MetricUtils.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/MetricUtils.java index 77cab23ce4fd..0e26fb1c53c0 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/MetricUtils.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/MetricUtils.java @@ -15,7 +15,8 @@ package software.amazon.awssdk.core.internal.util; -import static software.amazon.awssdk.core.http.HttpResponseHandler.X_AMZN_REQUEST_ID_HEADER; +import static software.amazon.awssdk.core.http.HttpResponseHandler.X_AMZN_REQUEST_ID_HEADERS; +import static software.amazon.awssdk.core.http.HttpResponseHandler.X_AMZ_ID_2_HEADER; import java.time.Duration; import java.util.concurrent.Callable; @@ -23,10 +24,12 @@ import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.core.internal.http.RequestExecutionContext; import software.amazon.awssdk.core.metrics.CoreMetric; +import software.amazon.awssdk.http.HttpMetric; import software.amazon.awssdk.http.SdkHttpFullResponse; import software.amazon.awssdk.metrics.MetricCollector; import software.amazon.awssdk.metrics.NoOpMetricCollector; import software.amazon.awssdk.utils.Pair; +import software.amazon.awssdk.utils.http.SdkHttpUtils; /** * Utility methods for working with metrics. @@ -64,12 +67,10 @@ public static Pair measureDurationUnsafe(Callable c) throws } public static void collectHttpMetrics(MetricCollector metricCollector, SdkHttpFullResponse httpResponse) { - metricCollector.reportMetric(CoreMetric.HTTP_STATUS_CODE, httpResponse.statusCode()); - httpResponse.firstMatchingHeader("x-amz-request-id") - .ifPresent(v -> metricCollector.reportMetric(CoreMetric.AWS_REQUEST_ID, v)); - httpResponse.firstMatchingHeader(X_AMZN_REQUEST_ID_HEADER) - .ifPresent(v -> metricCollector.reportMetric(CoreMetric.AWS_REQUEST_ID, v)); - httpResponse.firstMatchingHeader("x-amz-id-2") + metricCollector.reportMetric(HttpMetric.HTTP_STATUS_CODE, httpResponse.statusCode()); + SdkHttpUtils.allMatchingHeadersFromCollection(httpResponse.headers(), X_AMZN_REQUEST_ID_HEADERS) + .forEach(v -> metricCollector.reportMetric(CoreMetric.AWS_REQUEST_ID, v)); + httpResponse.firstMatchingHeader(X_AMZ_ID_2_HEADER) .ifPresent(v -> metricCollector.reportMetric(CoreMetric.AWS_EXTENDED_REQUEST_ID, v)); } @@ -82,7 +83,7 @@ public static MetricCollector createAttemptMetricsCollector(RequestExecutionCont } public static MetricCollector createHttpMetricsCollector(RequestExecutionContext context) { - MetricCollector parentCollector = context.metricCollector(); + MetricCollector parentCollector = context.attemptMetricCollector(); if (parentCollector != null) { return parentCollector.createChild("HttpClient"); } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/metrics/CoreMetric.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/metrics/CoreMetric.java index 713ea5b24c35..f72cb37d5b96 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/metrics/CoreMetric.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/metrics/CoreMetric.java @@ -17,7 +17,9 @@ import java.time.Duration; import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.core.retry.RetryPolicy; import software.amazon.awssdk.metrics.MetricCategory; +import software.amazon.awssdk.metrics.MetricLevel; import software.amazon.awssdk.metrics.SdkMetric; @SdkPublicApi @@ -25,65 +27,96 @@ public final class CoreMetric { /** * The unique ID for the service. This is present for all API call metrics. */ - public static final SdkMetric SERVICE_ID = metric("ServiceId", String.class); + public static final SdkMetric SERVICE_ID = + metric("ServiceId", String.class, MetricLevel.ERROR); /** * The name of the service operation being invoked. This is present for all * API call metrics. */ - public static final SdkMetric OPERATION_NAME = metric("OperationName", String.class); + public static final SdkMetric OPERATION_NAME = + metric("OperationName", String.class, MetricLevel.ERROR); + + /** + * True if the API call succeeded, false otherwise. + */ + public static final SdkMetric API_CALL_SUCCESSFUL = + metric("ApiCallSuccessful", Boolean.class, MetricLevel.ERROR); + + /** + * The number of retries that the SDK performed in the execution of the request. 0 implies that the request worked the first + * time, and no retries were attempted. + */ + public static final SdkMetric RETRY_COUNT = + metric("RetryCount", Integer.class, MetricLevel.ERROR); /** * The duration of the API call. This includes all call attempts made. + * + *

{@code API_CALL_DURATION = CREDENTIALS_FETCH_DURATION + MARSHALLING_DURATION + SUM_ALL(BACKOFF_DELAY_DURATION) + + * SUM_ALL(SIGNING_DURATION) + SUM_ALL(SERVICE_CALL_DURATION) + SUM_ALL(UNMARSHALLING_DURATION)} */ - public static final SdkMetric API_CALL_DURATION = metric("ApiCallDuration", Duration.class); + public static final SdkMetric API_CALL_DURATION = + metric("ApiCallDuration", Duration.class, MetricLevel.INFO); /** - * The duration of time taken to marshall the SDK request to an HTTP - * request. + * The duration of time taken to fetch signing credentials for the API call. */ - public static final SdkMetric MARSHALLING_DURATION = metric("MarshallingDuration", Duration.class); + public static final SdkMetric CREDENTIALS_FETCH_DURATION = + metric("CredentialsFetchDuration", Duration.class, MetricLevel.INFO); + /** - * The duration of time taken to fetch signing credentials for the request. + * The duration of time that the SDK has waited before this API call attempt, based on the + * {@link RetryPolicy#backoffStrategy()}. */ - public static final SdkMetric CREDENTIALS_FETCH_DURATION = metric("CredentialsFetchDuration", Duration.class); + public static final SdkMetric BACKOFF_DELAY_DURATION = + metric("BackoffDelayDuration", Duration.class, MetricLevel.INFO); /** - * The duration fo time taken to sign the HTTP request. + * The duration of time taken to marshall the SDK request to an HTTP request. */ - public static final SdkMetric SIGNING_DURATION = metric("SigningDuration", Duration.class); + public static final SdkMetric MARSHALLING_DURATION = + metric("MarshallingDuration", Duration.class, MetricLevel.INFO); /** - * The total time take to send a HTTP request and receive the response. + * The duration of time taken to sign the HTTP request. */ - public static final SdkMetric HTTP_REQUEST_ROUND_TRIP_TIME = metric("HttpRequestRoundTripTime", Duration.class); + public static final SdkMetric SIGNING_DURATION = + metric("SigningDuration", Duration.class, MetricLevel.INFO); /** - * The status code of the HTTP response. + * The duration of time taken to connect to the service (or acquire a connection from the connection pool), send the + * serialized request and receive the initial response (e.g. HTTP status code and headers). This DOES NOT include the time + * taken to read the entire response from the service. */ - public static final SdkMetric HTTP_STATUS_CODE = metric("HttpStatusCode", Integer.class); + public static final SdkMetric SERVICE_CALL_DURATION = + metric("ServiceCallDuration", Duration.class, MetricLevel.INFO); /** - * The request ID of the service request. + * The duration of time taken to unmarshall the HTTP response to an SDK response. + * + *

Note: For streaming operations, this does not include the time to read the response payload. */ - public static final SdkMetric AWS_REQUEST_ID = metric("AwsRequestId", String.class); + public static final SdkMetric UNMARSHALLING_DURATION = + metric("UnmarshallingDuration", Duration.class, MetricLevel.INFO); /** - * The extended request ID of the service request. + * The request ID of the service request. */ - public static final SdkMetric AWS_EXTENDED_REQUEST_ID = metric("AwsExtendedRequestId", String.class); + public static final SdkMetric AWS_REQUEST_ID = + metric("AwsRequestId", String.class, MetricLevel.INFO); /** - * The exception thrown during request execution. Note this may be a service - * error that has been unmarshalled, or a client side exception. + * The extended request ID of the service request. */ - public static final SdkMetric EXCEPTION = metric("Exception", Throwable.class); + public static final SdkMetric AWS_EXTENDED_REQUEST_ID = + metric("AwsExtendedRequestId", String.class, MetricLevel.INFO); private CoreMetric() { } - private static SdkMetric metric(String name, Class clzz) { - return SdkMetric.create(name, clzz, MetricCategory.DEFAULT); + private static SdkMetric metric(String name, Class clzz, MetricLevel level) { + return SdkMetric.create(name, clzz, level, MetricCategory.CORE); } } diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStageTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStageTest.java index 42364c1cd87e..e1e0e26eeea5 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStageTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStageTest.java @@ -127,7 +127,7 @@ public void testExecute_contextContainsMetricCollector_addsChildToExecuteRequest .executionContext(executionContext) .build(); - context.metricCollector(mockCollector); + context.attemptMetricCollector(mockCollector); try { stage.execute(sdkHttpRequest, context); diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeHttpRequestStageTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeHttpRequestStageTest.java index 02fc52e31206..5636852bf5a3 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeHttpRequestStageTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeHttpRequestStageTest.java @@ -31,6 +31,7 @@ import org.mockito.runners.MockitoJUnitRunner; import software.amazon.awssdk.core.client.config.SdkClientConfiguration; import software.amazon.awssdk.core.http.ExecutionContext; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; import software.amazon.awssdk.core.internal.http.HttpClientDependencies; import software.amazon.awssdk.core.internal.http.RequestExecutionContext; import software.amazon.awssdk.core.internal.http.timers.TimeoutTracker; @@ -68,14 +69,16 @@ public void testExecute_contextContainsMetricCollector_addsChildToExecuteRequest when(mockCollector.createChild(any(String.class))).thenReturn(childCollector); - ExecutionContext executionContext = ExecutionContext.builder().build(); + ExecutionContext executionContext = ExecutionContext.builder() + .executionAttributes(new ExecutionAttributes()) + .build(); RequestExecutionContext context = RequestExecutionContext.builder() .originalRequest(ValidSdkObjects.sdkRequest()) .executionContext(executionContext) .build(); - context.metricCollector(mockCollector); + context.attemptMetricCollector(mockCollector); context.apiCallAttemptTimeoutTracker(mock(TimeoutTracker.class)); context.apiCallTimeoutTracker(mock(TimeoutTracker.class)); diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/util/MetricUtilsTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/util/MetricUtilsTest.java index 9fead321d00a..b32a543a7404 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/util/MetricUtilsTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/util/MetricUtilsTest.java @@ -26,6 +26,7 @@ import org.junit.rules.ExpectedException; import software.amazon.awssdk.core.http.HttpResponseHandler; import software.amazon.awssdk.core.metrics.CoreMetric; +import software.amazon.awssdk.http.HttpMetric; import software.amazon.awssdk.http.SdkHttpFullResponse; import software.amazon.awssdk.metrics.MetricCollector; import software.amazon.awssdk.utils.Pair; @@ -107,10 +108,9 @@ public void testCollectHttpMetrics_collectsAllExpectedMetrics() { MetricUtils.collectHttpMetrics(mockCollector, response); - verify(mockCollector).reportMetric(CoreMetric.HTTP_STATUS_CODE, statusCode); + verify(mockCollector).reportMetric(HttpMetric.HTTP_STATUS_CODE, statusCode); verify(mockCollector).reportMetric(CoreMetric.AWS_REQUEST_ID, requestId); verify(mockCollector).reportMetric(CoreMetric.AWS_REQUEST_ID, amznRequestId); verify(mockCollector).reportMetric(CoreMetric.AWS_EXTENDED_REQUEST_ID, requestId2); - } } diff --git a/http-client-spi/src/main/java/software/amazon/awssdk/http/Http2Metric.java b/http-client-spi/src/main/java/software/amazon/awssdk/http/Http2Metric.java index f34e687ce1b1..659c2db78446 100644 --- a/http-client-spi/src/main/java/software/amazon/awssdk/http/Http2Metric.java +++ b/http-client-spi/src/main/java/software/amazon/awssdk/http/Http2Metric.java @@ -17,6 +17,7 @@ import software.amazon.awssdk.annotations.SdkPublicApi; import software.amazon.awssdk.metrics.MetricCategory; +import software.amazon.awssdk.metrics.MetricLevel; import software.amazon.awssdk.metrics.SdkMetric; /** @@ -30,19 +31,21 @@ public final class Http2Metric { * *

See https://http2.github.io/http2-spec/#FlowControl for more information on HTTP/2 window sizes. */ - public static final SdkMetric LOCAL_STREAM_WINDOW_SIZE_IN_BYTES = metric("LocalStreamWindowSize", Integer.class); + public static final SdkMetric LOCAL_STREAM_WINDOW_SIZE_IN_BYTES = + metric("LocalStreamWindowSize", Integer.class, MetricLevel.TRACE); /** * The remote HTTP/2 window size in bytes for the stream that this request was executed on. * *

See https://http2.github.io/http2-spec/#FlowControl for more information on HTTP/2 window sizes. */ - public static final SdkMetric REMOTE_STREAM_WINDOW_SIZE_IN_BYTES = metric("RemoteStreamWindowSize", Integer.class); + public static final SdkMetric REMOTE_STREAM_WINDOW_SIZE_IN_BYTES = + metric("RemoteStreamWindowSize", Integer.class, MetricLevel.TRACE); private Http2Metric() { } - private static SdkMetric metric(String name, Class clzz) { - return SdkMetric.create(name, clzz, MetricCategory.DEFAULT, MetricCategory.HTTP_CLIENT); + private static SdkMetric metric(String name, Class clzz, MetricLevel level) { + return SdkMetric.create(name, clzz, level, MetricCategory.CORE, MetricCategory.HTTP_CLIENT); } } diff --git a/http-client-spi/src/main/java/software/amazon/awssdk/http/HttpMetric.java b/http-client-spi/src/main/java/software/amazon/awssdk/http/HttpMetric.java index e3cf44460328..ede491622824 100644 --- a/http-client-spi/src/main/java/software/amazon/awssdk/http/HttpMetric.java +++ b/http-client-spi/src/main/java/software/amazon/awssdk/http/HttpMetric.java @@ -17,6 +17,7 @@ import software.amazon.awssdk.annotations.SdkPublicApi; import software.amazon.awssdk.metrics.MetricCategory; +import software.amazon.awssdk.metrics.MetricLevel; import software.amazon.awssdk.metrics.SdkMetric; /** @@ -28,7 +29,8 @@ public final class HttpMetric { /** * The name of the HTTP client. */ - public static final SdkMetric HTTP_CLIENT_NAME = metric("HttpClientName", String.class); + public static final SdkMetric HTTP_CLIENT_NAME = + metric("HttpClientName", String.class, MetricLevel.INFO); /** * The maximum number of concurrent requests that is supported by the HTTP client. @@ -42,7 +44,8 @@ public final class HttpMetric { * individual HTTP client instance, and does not include concurrency that may be available in other HTTP clients running * within the same JVM. */ - public static final SdkMetric MAX_CONCURRENCY = metric("MaxConcurrency", Integer.class); + public static final SdkMetric MAX_CONCURRENCY = + metric("MaxConcurrency", Integer.class, MetricLevel.INFO); /** * The number of additional concurrent requests that can be supported by the HTTP client without needing to establish @@ -58,7 +61,8 @@ public final class HttpMetric { * individual HTTP client instance, and does not include concurrency that may be available in other HTTP clients running * within the same JVM. */ - public static final SdkMetric AVAILABLE_CONCURRENCY = metric("AvailableConcurrency", Integer.class); + public static final SdkMetric AVAILABLE_CONCURRENCY = + metric("AvailableConcurrency", Integer.class, MetricLevel.INFO); /** * The number of requests that are currently being executed by the HTTP client. @@ -73,7 +77,8 @@ public final class HttpMetric { * individual HTTP client instance, and does not include concurrency that may be available in other HTTP clients running * within the same JVM. */ - public static final SdkMetric LEASED_CONCURRENCY = metric("LeasedConcurrency", Integer.class); + public static final SdkMetric LEASED_CONCURRENCY = + metric("LeasedConcurrency", Integer.class, MetricLevel.INFO); /** * The number of requests that are awaiting concurrency to be made available from the HTTP client. @@ -88,12 +93,21 @@ public final class HttpMetric { * individual HTTP client instance, and does not include concurrency that may be available in other HTTP clients running * within the same JVM. */ - public static final SdkMetric PENDING_CONCURRENCY_ACQUIRES = metric("PendingConcurrencyAcquires", Integer.class); + public static final SdkMetric PENDING_CONCURRENCY_ACQUIRES = + metric("PendingConcurrencyAcquires", Integer.class, MetricLevel.INFO); + + /** + * The status code of the HTTP response. + * + * @implSpec This is reported by the SDK core, and should not be reported by an individual HTTP client implementation. + */ + public static final SdkMetric HTTP_STATUS_CODE = + metric("HttpStatusCode", Integer.class, MetricLevel.TRACE); private HttpMetric() { } - private static SdkMetric metric(String name, Class clzz) { - return SdkMetric.create(name, clzz, MetricCategory.DEFAULT, MetricCategory.HTTP_CLIENT); + private static SdkMetric metric(String name, Class clzz, MetricLevel level) { + return SdkMetric.create(name, clzz, level, MetricCategory.CORE, MetricCategory.HTTP_CLIENT); } } diff --git a/metric-publishers/cloudwatch-metric-publisher/pom.xml b/metric-publishers/cloudwatch-metric-publisher/pom.xml index 1a54f64e7dff..235e89194b9a 100644 --- a/metric-publishers/cloudwatch-metric-publisher/pom.xml +++ b/metric-publishers/cloudwatch-metric-publisher/pom.xml @@ -40,6 +40,11 @@ sdk-core ${awsjavasdk.version} + + software.amazon.awssdk + aws-core + ${awsjavasdk.version} + software.amazon.awssdk http-client-spi diff --git a/metric-publishers/cloudwatch-metric-publisher/src/main/java/software/amazon/awssdk/metrics/publishers/cloudwatch/CloudWatchMetricPublisher.java b/metric-publishers/cloudwatch-metric-publisher/src/main/java/software/amazon/awssdk/metrics/publishers/cloudwatch/CloudWatchMetricPublisher.java index fc4cde7c4891..3d38770c2914 100644 --- a/metric-publishers/cloudwatch-metric-publisher/src/main/java/software/amazon/awssdk/metrics/publishers/cloudwatch/CloudWatchMetricPublisher.java +++ b/metric-publishers/cloudwatch-metric-publisher/src/main/java/software/amazon/awssdk/metrics/publishers/cloudwatch/CloudWatchMetricPublisher.java @@ -42,6 +42,7 @@ import software.amazon.awssdk.metrics.MetricCategory; import software.amazon.awssdk.metrics.MetricCollection; import software.amazon.awssdk.metrics.MetricCollector; +import software.amazon.awssdk.metrics.MetricLevel; import software.amazon.awssdk.metrics.MetricPublisher; import software.amazon.awssdk.metrics.SdkMetric; import software.amazon.awssdk.metrics.publishers.cloudwatch.internal.MetricUploader; @@ -92,7 +93,7 @@ * * Step 1: Define which metrics you wish to collect * - *

Metrics are described using the {@link SdkMetric#create(String, Class)} method. When you describe your metric, you specify + *

Metrics are described using the {@link SdkMetric#create} method. When you describe your metric, you specify * the name that will appear in CloudWatch and the Java data-type of the metric. The metric should be described once for your * entire application. * @@ -103,11 +104,11 @@ * public static final class MyMethodMetrics { * // The number of times "myMethod" has been called. * private static final SdkMetric<Integer> MY_METHOD_CALL_COUNT = - * SdkMetric.create("MyMethodCallCount", Integer.class); + * SdkMetric.create("MyMethodCallCount", Integer.class, MetricLevel.INFO, MetricCategory.CUSTOM); * * // The amount of time that "myMethod" took to execute. * private static final SdkMetric<Duration> MY_METHOD_LATENCY = - * SdkMetric.create("MyMethodLatency", Duration.class); + * SdkMetric.create("MyMethodLatency", Duration.class, MetricLevel.INFO, MetricCategory.CUSTOM); * } * * @@ -172,9 +173,8 @@ public final class CloudWatchMetricPublisher implements MetricPublisher { private static final Set> DEFAULT_DIMENSIONS = Stream.of(CoreMetric.SERVICE_ID, CoreMetric.OPERATION_NAME) .collect(Collectors.toSet()); - private static final Set DEFAULT_METRIC_CATEGORIES = Stream.of(MetricCategory.DEFAULT, - MetricCategory.HTTP_CLIENT) - .collect(Collectors.toSet()); + private static final Set DEFAULT_METRIC_CATEGORIES = Collections.singleton(MetricCategory.ALL); + private static final MetricLevel DEFAULT_METRIC_LEVEL = MetricLevel.INFO; private static final Set> DEFAULT_DETAILED_METRICS = Collections.emptySet(); /** @@ -218,6 +218,7 @@ private CloudWatchMetricPublisher(Builder builder) { this.metricAggregator = new MetricCollectionAggregator(resolveNamespace(builder), resolveDimensions(builder), resolveMetricCategories(builder), + resolveMetricLevel(builder), resolveDetailedMetrics(builder)); this.metricUploader = new MetricUploader(resolveClient(builder)); this.maximumCallsPerUpload = resolveMaximumCallsPerUpload(builder); @@ -239,6 +240,10 @@ private Set resolveMetricCategories(Builder builder) { return builder.metricCategories == null ? DEFAULT_METRIC_CATEGORIES : new HashSet<>(builder.metricCategories); } + private MetricLevel resolveMetricLevel(Builder builder) { + return builder.metricLevel == null ? DEFAULT_METRIC_LEVEL : builder.metricLevel; + } + private Set> resolveDetailedMetrics(Builder builder) { return builder.detailedMetrics == null ? DEFAULT_DETAILED_METRICS : new HashSet<>(builder.detailedMetrics); } @@ -353,6 +358,7 @@ public static final class Builder { private Integer maximumCallsPerUpload; private Collection> dimensions; private Collection metricCategories; + private MetricLevel metricLevel; private Collection> detailedMetrics; private Builder() { @@ -465,7 +471,7 @@ public final Builder dimensions(SdkMetric... dimensions) { /** * Configure the {@link MetricCategory}s that should be uploaded to CloudWatch. * - *

If this is not specified, {@link MetricCategory#DEFAULT} and {@link MetricCategory#HTTP_CLIENT} are used. + *

If this is not specified, {@link MetricCategory#ALL} is used. * *

All {@link SdkMetric}s are associated with at least one {@code MetricCategory}. This setting determines which * category of metrics uploaded to CloudWatch. Any metrics {@link #publish(MetricCollection)}ed that do not fall under @@ -487,6 +493,24 @@ public Builder metricCategories(MetricCategory... metricCategories) { return metricCategories(Arrays.asList(metricCategories)); } + /** + * Configure the {@link MetricLevel} that should be uploaded to CloudWatch. + * + *

If this is not specified, {@link MetricLevel#INFO} is used. + * + *

All {@link SdkMetric}s are associated with one {@code MetricLevel}. This setting determines which level of metrics + * uploaded to CloudWatch. Any metrics {@link #publish(MetricCollection)}ed that do not fall under these configured + * categories are ignored. + * + *

Note: If there are {@link #dimensions(Collection)} configured that do not fall under this {@code MetricLevel} + * values, the dimensions will NOT be ignored. In other words, the metric category configuration only affects which + * metrics are uploaded to CloudWatch, not which values can be used for {@code dimensions}. + */ + public Builder metricLevel(MetricLevel metricLevel) { + this.metricLevel = metricLevel; + return this; + } + /** * Configure the set of metrics for which detailed values and counts are uploaded to CloudWatch, instead of summaries. * diff --git a/metric-publishers/cloudwatch-metric-publisher/src/main/java/software/amazon/awssdk/metrics/publishers/cloudwatch/internal/transform/MetricCollectionAggregator.java b/metric-publishers/cloudwatch-metric-publisher/src/main/java/software/amazon/awssdk/metrics/publishers/cloudwatch/internal/transform/MetricCollectionAggregator.java index fe13bd9da5af..9a00b2d8fa04 100644 --- a/metric-publishers/cloudwatch-metric-publisher/src/main/java/software/amazon/awssdk/metrics/publishers/cloudwatch/internal/transform/MetricCollectionAggregator.java +++ b/metric-publishers/cloudwatch-metric-publisher/src/main/java/software/amazon/awssdk/metrics/publishers/cloudwatch/internal/transform/MetricCollectionAggregator.java @@ -24,8 +24,10 @@ import java.util.stream.Stream; import software.amazon.awssdk.annotations.NotThreadSafe; import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.ApiName; import software.amazon.awssdk.metrics.MetricCategory; import software.amazon.awssdk.metrics.MetricCollection; +import software.amazon.awssdk.metrics.MetricLevel; import software.amazon.awssdk.metrics.SdkMetric; import software.amazon.awssdk.metrics.publishers.cloudwatch.internal.transform.DetailedMetricAggregator.DetailedMetrics; import software.amazon.awssdk.services.cloudwatch.model.MetricDatum; @@ -57,6 +59,11 @@ public class MetricCollectionAggregator { */ public static final int MAX_VALUES_PER_REQUEST = 300; + /** + * The API name to include in the user agent for all {@link PutMetricDataRequest}s generated by this aggregator. + */ + private static final ApiName API_NAME = ApiName.builder().name("hll").version("cw-mp").build(); + /** * The {@link PutMetricDataRequest#namespace()} that should be used for all {@link PutMetricDataRequest}s returned from * {@link #getRequests()}. @@ -72,9 +79,10 @@ public class MetricCollectionAggregator { public MetricCollectionAggregator(String namespace, Set> dimensions, Set metricCategories, + MetricLevel metricLevel, Set> detailedMetrics) { this.namespace = namespace; - this.timeBucketedMetrics = new TimeBucketedMetrics(dimensions, metricCategories, detailedMetrics); + this.timeBucketedMetrics = new TimeBucketedMetrics(dimensions, metricCategories, metricLevel, detailedMetrics); } /** @@ -183,6 +191,7 @@ private MetricDatum summaryMetricDatum(Instant timeBucket, private PutMetricDataRequest newPutRequest(List metricData) { return PutMetricDataRequest.builder() + .overrideConfiguration(r -> r.addApiName(API_NAME)) .namespace(namespace) .metricData(metricData) .build(); diff --git a/metric-publishers/cloudwatch-metric-publisher/src/main/java/software/amazon/awssdk/metrics/publishers/cloudwatch/internal/transform/TimeBucketedMetrics.java b/metric-publishers/cloudwatch-metric-publisher/src/main/java/software/amazon/awssdk/metrics/publishers/cloudwatch/internal/transform/TimeBucketedMetrics.java index f6ac04032c49..949f16a01504 100644 --- a/metric-publishers/cloudwatch-metric-publisher/src/main/java/software/amazon/awssdk/metrics/publishers/cloudwatch/internal/transform/TimeBucketedMetrics.java +++ b/metric-publishers/cloudwatch-metric-publisher/src/main/java/software/amazon/awssdk/metrics/publishers/cloudwatch/internal/transform/TimeBucketedMetrics.java @@ -31,6 +31,7 @@ import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.metrics.MetricCategory; import software.amazon.awssdk.metrics.MetricCollection; +import software.amazon.awssdk.metrics.MetricLevel; import software.amazon.awssdk.metrics.MetricRecord; import software.amazon.awssdk.metrics.SdkMetric; import software.amazon.awssdk.services.cloudwatch.model.Dimension; @@ -67,17 +68,27 @@ class TimeBucketedMetrics { */ private final Set metricCategories; + /** + * The metric levels for which we should aggregate values. Any categories at a more "verbose" level than this one will have + * their values ignored/dropped. + */ + private final MetricLevel metricLevel; + /** * True, when the {@link #metricCategories} contains {@link MetricCategory#ALL}. */ private final boolean metricCategoriesContainsAll; + + TimeBucketedMetrics(Set> dimensions, Set metricCategories, + MetricLevel metricLevel, Set> detailedMetrics) { this.dimensions = dimensions; this.detailedMetrics = detailedMetrics; this.metricCategories = metricCategories; + this.metricLevel = metricLevel; this.metricCategoriesContainsAll = metricCategories.contains(MetricCategory.ALL); } @@ -171,11 +182,12 @@ private StandardUnit unitFor(SdkMetric metric) { if (Duration.class.isAssignableFrom(metricType)) { return StandardUnit.MILLISECONDS; } + return StandardUnit.NONE; } private Optional valueFor(MetricRecord metricRecord) { - if (!hasReportedCategory(metricRecord)) { + if (!shouldReport(metricRecord)) { return Optional.empty(); } @@ -188,16 +200,27 @@ private Optional valueFor(MetricRecord metricRecord) { } else if (Number.class.isAssignableFrom(metricType)) { Number numberMetricValue = (Number) metricRecord.value(); return Optional.of(numberMetricValue.doubleValue()); + } else if (Boolean.class.isAssignableFrom(metricType)) { + Boolean booleanMetricValue = (Boolean) metricRecord.value(); + return Optional.of(booleanMetricValue ? 1.0 : 0.0); } return Optional.empty(); } - private boolean hasReportedCategory(MetricRecord metricRecord) { + private boolean shouldReport(MetricRecord metricRecord) { + return isSupportedCategory(metricRecord) && isSupportedLevel(metricRecord); + } + + private boolean isSupportedCategory(MetricRecord metricRecord) { return metricCategoriesContainsAll || metricRecord.metric() .categories() .stream() .anyMatch(metricCategories::contains); } + + private boolean isSupportedLevel(MetricRecord metricRecord) { + return metricLevel.includesLevel(metricRecord.metric().level()); + } } diff --git a/metric-publishers/cloudwatch-metric-publisher/src/test/java/software/amazon/awssdk/metrics/publishers/cloudwatch/CloudWatchMetricPublisherTest.java b/metric-publishers/cloudwatch-metric-publisher/src/test/java/software/amazon/awssdk/metrics/publishers/cloudwatch/CloudWatchMetricPublisherTest.java index f593633af24d..316c9762a919 100644 --- a/metric-publishers/cloudwatch-metric-publisher/src/test/java/software/amazon/awssdk/metrics/publishers/cloudwatch/CloudWatchMetricPublisherTest.java +++ b/metric-publishers/cloudwatch-metric-publisher/src/test/java/software/amazon/awssdk/metrics/publishers/cloudwatch/CloudWatchMetricPublisherTest.java @@ -30,6 +30,7 @@ import software.amazon.awssdk.http.HttpMetric; import software.amazon.awssdk.metrics.MetricCategory; import software.amazon.awssdk.metrics.MetricCollector; +import software.amazon.awssdk.metrics.MetricLevel; import software.amazon.awssdk.metrics.SdkMetric; import software.amazon.awssdk.metrics.publishers.cloudwatch.internal.transform.MetricCollectionAggregator; import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient; @@ -149,13 +150,14 @@ public void metricCategoriesSettingIsHonored() { try (CloudWatchMetricPublisher publisher = publisherBuilder.metricCategories(MetricCategory.HTTP_CLIENT).build()) { MetricCollector collector = newCollector(); collector.reportMetric(CoreMetric.SERVICE_ID, "ServiceId"); - collector.reportMetric(CoreMetric.HTTP_STATUS_CODE, 404); + collector.reportMetric(CoreMetric.API_CALL_SUCCESSFUL, true); collector.reportMetric(HttpMetric.AVAILABLE_CONCURRENCY, 5); publisher.publish(new FixedTimeMetricCollection(collector.collect())); } PutMetricDataRequest call = getPutMetricCall(); MetricDatum metric = call.metricData().get(0); + assertThat(call.metricData()).hasSize(1); assertThat(metric.dimensions()).containsExactly(Dimension.builder() .name(CoreMetric.SERVICE_ID.name()) .value("ServiceId") @@ -163,6 +165,26 @@ public void metricCategoriesSettingIsHonored() { assertThat(metric.metricName()).isEqualTo(HttpMetric.AVAILABLE_CONCURRENCY.name()); } + @Test + public void metricLevelSettingIsHonored() { + try (CloudWatchMetricPublisher publisher = publisherBuilder.metricLevel(MetricLevel.INFO).build()) { + MetricCollector collector = newCollector(); + collector.reportMetric(CoreMetric.SERVICE_ID, "ServiceId"); + collector.reportMetric(CoreMetric.API_CALL_SUCCESSFUL, true); + collector.reportMetric(HttpMetric.HTTP_STATUS_CODE, 404); + publisher.publish(new FixedTimeMetricCollection(collector.collect())); + } + + PutMetricDataRequest call = getPutMetricCall(); + MetricDatum metric = call.metricData().get(0); + assertThat(call.metricData()).hasSize(1); + assertThat(metric.dimensions()).containsExactly(Dimension.builder() + .name(CoreMetric.SERVICE_ID.name()) + .value("ServiceId") + .build()); + assertThat(metric.metricName()).isEqualTo(CoreMetric.API_CALL_SUCCESSFUL.name()); + } + @Test public void maximumCallsPerPublishSettingIsHonored() { try (CloudWatchMetricPublisher publisher = publisherBuilder.maximumCallsPerUpload(1) diff --git a/metric-publishers/cloudwatch-metric-publisher/src/test/java/software/amazon/awssdk/metrics/publishers/cloudwatch/internal/transform/MetricCollectionAggregatorTest.java b/metric-publishers/cloudwatch-metric-publisher/src/test/java/software/amazon/awssdk/metrics/publishers/cloudwatch/internal/transform/MetricCollectionAggregatorTest.java index 7f9a9d09d27e..e2d537853811 100644 --- a/metric-publishers/cloudwatch-metric-publisher/src/test/java/software/amazon/awssdk/metrics/publishers/cloudwatch/internal/transform/MetricCollectionAggregatorTest.java +++ b/metric-publishers/cloudwatch-metric-publisher/src/test/java/software/amazon/awssdk/metrics/publishers/cloudwatch/internal/transform/MetricCollectionAggregatorTest.java @@ -20,7 +20,6 @@ import java.time.Duration; import java.time.Instant; -import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Set; @@ -34,6 +33,7 @@ import software.amazon.awssdk.metrics.MetricCategory; import software.amazon.awssdk.metrics.MetricCollection; import software.amazon.awssdk.metrics.MetricCollector; +import software.amazon.awssdk.metrics.MetricLevel; import software.amazon.awssdk.metrics.SdkMetric; import software.amazon.awssdk.metrics.publishers.cloudwatch.FixedTimeMetricCollection; import software.amazon.awssdk.services.cloudwatch.model.PutMetricDataRequest; @@ -43,6 +43,7 @@ public class MetricCollectionAggregatorTest { private static final String DEFAULT_NAMESPACE = "namespace"; private static final Set> DEFAULT_DIMENSIONS = Stream.of(CoreMetric.SERVICE_ID, CoreMetric.OPERATION_NAME) .collect(Collectors.toSet()); + private static final MetricLevel DEFAULT_METRIC_LEVEL = MetricLevel.INFO; private static final Set DEFAULT_CATEGORIES = Collections.singleton(MetricCategory.HTTP_CLIENT); private static final Set> DEFAULT_DETAILED_METRICS = Collections.emptySet(); @@ -340,7 +341,7 @@ public void metricsFromOtherCategoriesAreIgnored() { MetricCollectionAggregator aggregator = defaultAggregator(); MetricCollector collector = collector(); collector.reportMetric(CoreMetric.SERVICE_ID, "ServiceId"); - collector.reportMetric(CoreMetric.HTTP_STATUS_CODE, 404); + collector.reportMetric(HttpMetric.HTTP_STATUS_CODE, 404); aggregator.addCollection(collectToFixedTime(collector)); assertThat(aggregator.getRequests()).isEmpty(); @@ -374,6 +375,13 @@ public void durationsAreTransformedCorrectly() { assertThat(transformMetricValueUsingAggregator(metric, Duration.ofSeconds(10))).isEqualTo(10_000); } + @Test + public void booleansAreTransformedCorrectly() { + SdkMetric metric = someMetric(Boolean.class); + assertThat(transformMetricValueUsingAggregator(metric, false)).isEqualTo(0.0); + assertThat(transformMetricValueUsingAggregator(metric, true)).isEqualTo(1.0); + } + private Double transformMetricValueUsingAggregator(SdkMetric metric, T input) { MetricCollectionAggregator aggregator = aggregatorWithCustomDetailedMetrics(metric); MetricCollector collector = collector(); @@ -407,6 +415,7 @@ private MetricCollectionAggregator defaultAggregator() { return new MetricCollectionAggregator(DEFAULT_NAMESPACE, DEFAULT_DIMENSIONS, DEFAULT_CATEGORIES, + DEFAULT_METRIC_LEVEL, DEFAULT_DETAILED_METRICS); } @@ -414,6 +423,7 @@ private MetricCollectionAggregator aggregatorWithCustomDetailedMetrics(SdkMetric return new MetricCollectionAggregator(DEFAULT_NAMESPACE, DEFAULT_DIMENSIONS, DEFAULT_CATEGORIES, + DEFAULT_METRIC_LEVEL, Stream.of(detailedMetrics).collect(Collectors.toSet())); } @@ -427,7 +437,9 @@ private SdkMetric someMetric() { private SdkMetric someMetric(Class clazz) { return SdkMetric.create(getClass().getSimpleName() + UUID.randomUUID().toString(), - clazz, MetricCategory.HTTP_CLIENT); + clazz, + MetricLevel.INFO, + MetricCategory.HTTP_CLIENT); } private MetricCollection collectToFixedTime(MetricCollector collector) { diff --git a/services/transcribestreaming/src/it/java/software/amazon/awssdk/services/transcribestreaming/TranscribeStreamingIntegrationTest.java b/services/transcribestreaming/src/it/java/software/amazon/awssdk/services/transcribestreaming/TranscribeStreamingIntegrationTest.java index 4ba696d8f35a..54fb9a67d4af 100644 --- a/services/transcribestreaming/src/it/java/software/amazon/awssdk/services/transcribestreaming/TranscribeStreamingIntegrationTest.java +++ b/services/transcribestreaming/src/it/java/software/amazon/awssdk/services/transcribestreaming/TranscribeStreamingIntegrationTest.java @@ -40,6 +40,7 @@ import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; import software.amazon.awssdk.core.internal.util.Mimetype; import software.amazon.awssdk.core.metrics.CoreMetric; +import software.amazon.awssdk.http.HttpMetric; import software.amazon.awssdk.metrics.MetricCollection; import software.amazon.awssdk.metrics.MetricPublisher; import software.amazon.awssdk.regions.Region; @@ -160,13 +161,13 @@ private void verifyMetrics() throws InterruptedException { MetricCollection attemptCollection = capturedCollection.children().get(0); assertThat(attemptCollection.name()).isEqualTo("ApiCallAttempt"); - assertThat(attemptCollection.metricValues(CoreMetric.HTTP_STATUS_CODE)) + assertThat(attemptCollection.metricValues(HttpMetric.HTTP_STATUS_CODE)) .containsExactly(200); assertThat(attemptCollection.metricValues(CoreMetric.SIGNING_DURATION).get(0)) .isGreaterThanOrEqualTo(Duration.ZERO); assertThat(attemptCollection.metricValues(CoreMetric.AWS_REQUEST_ID).get(0)).isNotEmpty(); - assertThat(attemptCollection.metricValues(CoreMetric.HTTP_REQUEST_ROUND_TRIP_TIME).get(0)) + assertThat(attemptCollection.metricValues(CoreMetric.SERVICE_CALL_DURATION).get(0)) .isGreaterThanOrEqualTo(Duration.ofMillis(100)); } diff --git a/test/codegen-generated-classes-test/src/test/java/software/amazon/awssdk/services/metrics/CoreMetricsTest.java b/test/codegen-generated-classes-test/src/test/java/software/amazon/awssdk/services/metrics/CoreMetricsTest.java index a24c81fb01cd..388551ed4cdd 100644 --- a/test/codegen-generated-classes-test/src/test/java/software/amazon/awssdk/services/metrics/CoreMetricsTest.java +++ b/test/codegen-generated-classes-test/src/test/java/software/amazon/awssdk/services/metrics/CoreMetricsTest.java @@ -16,12 +16,12 @@ package software.amazon.awssdk.services.metrics; import static org.assertj.core.api.Assertions.assertThat; -import static org.hamcrest.Matchers.is; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; + import java.io.ByteArrayInputStream; import java.io.IOException; import java.time.Duration; @@ -41,6 +41,7 @@ import software.amazon.awssdk.http.ExecutableHttpRequest; import software.amazon.awssdk.http.HttpExecuteRequest; import software.amazon.awssdk.http.HttpExecuteResponse; +import software.amazon.awssdk.http.HttpMetric; import software.amazon.awssdk.http.SdkHttpClient; import software.amazon.awssdk.http.SdkHttpFullResponse; import software.amazon.awssdk.metrics.MetricCollection; @@ -150,28 +151,33 @@ public void testApiCall_operationSuccessful_addsMetrics() { .containsExactly(SERVICE_ID); assertThat(capturedCollection.metricValues(CoreMetric.OPERATION_NAME)) .containsExactly("AllTypes"); + assertThat(capturedCollection.metricValues(CoreMetric.API_CALL_SUCCESSFUL)).containsExactly(true); + assertThat(capturedCollection.metricValues(CoreMetric.API_CALL_DURATION).get(0)) + .isGreaterThan(Duration.ZERO); assertThat(capturedCollection.metricValues(CoreMetric.CREDENTIALS_FETCH_DURATION).get(0)) - .isGreaterThanOrEqualTo(Duration.ZERO); + .isGreaterThanOrEqualTo(Duration.ZERO); assertThat(capturedCollection.metricValues(CoreMetric.MARSHALLING_DURATION).get(0)) - .isGreaterThanOrEqualTo(Duration.ZERO); - assertThat(capturedCollection.metricValues(CoreMetric.API_CALL_DURATION).get(0)) - .isGreaterThan(Duration.ZERO); + .isGreaterThanOrEqualTo(Duration.ZERO); + assertThat(capturedCollection.metricValues(CoreMetric.RETRY_COUNT)).containsExactly(0); assertThat(capturedCollection.children()).hasSize(1); MetricCollection attemptCollection = capturedCollection.children().get(0); assertThat(attemptCollection.name()).isEqualTo("ApiCallAttempt"); - assertThat(attemptCollection.metricValues(CoreMetric.HTTP_STATUS_CODE)) - .containsExactly(200); + assertThat(attemptCollection.metricValues(CoreMetric.BACKOFF_DELAY_DURATION)) + .containsExactly(Duration.ZERO); + assertThat(attemptCollection.metricValues(HttpMetric.HTTP_STATUS_CODE)) + .containsExactly(200); assertThat(attemptCollection.metricValues(CoreMetric.SIGNING_DURATION).get(0)) .isGreaterThanOrEqualTo(Duration.ZERO); assertThat(attemptCollection.metricValues(CoreMetric.AWS_REQUEST_ID)) - .containsExactly(REQUEST_ID); + .containsExactly(REQUEST_ID); assertThat(attemptCollection.metricValues(CoreMetric.AWS_EXTENDED_REQUEST_ID)) - .containsExactly(EXTENDED_REQUEST_ID); - - assertThat(attemptCollection.metricValues(CoreMetric.HTTP_REQUEST_ROUND_TRIP_TIME).get(0)) - .isGreaterThanOrEqualTo(Duration.ofMillis(100)); + .containsExactly(EXTENDED_REQUEST_ID); + assertThat(attemptCollection.metricValues(CoreMetric.SERVICE_CALL_DURATION).get(0)) + .isGreaterThanOrEqualTo(Duration.ofMillis(100)); + assertThat(attemptCollection.metricValues(CoreMetric.UNMARSHALLING_DURATION).get(0)) + .isGreaterThanOrEqualTo(Duration.ZERO); } @Test @@ -204,96 +210,28 @@ public void testApiCall_serviceReturnsError_errorInfoIncludedInMetrics() throws MetricCollection capturedCollection = collectionCaptor.getValue(); assertThat(capturedCollection.children()).hasSize(MAX_RETRIES + 1); + assertThat(capturedCollection.metricValues(CoreMetric.RETRY_COUNT)).containsExactly(MAX_RETRIES); + assertThat(capturedCollection.metricValues(CoreMetric.API_CALL_SUCCESSFUL)).containsExactly(false); for (MetricCollection requestMetrics : capturedCollection.children()) { - assertThat(requestMetrics.metricValues(CoreMetric.EXCEPTION).get(0)) - // Note: for some reason we don't throw the same exact - // instance as the one unmarshalled by the response - // handler (seems we make a copy upstream) so an - // isSameAs assertion fails here - .isInstanceOf(EmptyModeledException.class); - // A service exception is still a successful HTTP execution so // we should still have HTTP metrics as well. - assertThat(requestMetrics.metricValues(CoreMetric.HTTP_STATUS_CODE)) - .containsExactly(500); + assertThat(requestMetrics.metricValues(HttpMetric.HTTP_STATUS_CODE)) + .containsExactly(500); assertThat(requestMetrics.metricValues(CoreMetric.AWS_REQUEST_ID)) - .containsExactly(REQUEST_ID); + .containsExactly(REQUEST_ID); assertThat(requestMetrics.metricValues(CoreMetric.AWS_EXTENDED_REQUEST_ID)) - .containsExactly(EXTENDED_REQUEST_ID); - assertThat(requestMetrics.metricValues(CoreMetric.HTTP_REQUEST_ROUND_TRIP_TIME).get(0)) - .isGreaterThanOrEqualTo(Duration.ZERO); + .containsExactly(EXTENDED_REQUEST_ID); + assertThat(requestMetrics.metricValues(CoreMetric.SERVICE_CALL_DURATION)).hasOnlyOneElementSatisfying(d -> { + assertThat(d).isGreaterThanOrEqualTo(Duration.ZERO); + }); + assertThat(requestMetrics.metricValues(CoreMetric.UNMARSHALLING_DURATION)).hasOnlyOneElementSatisfying(d -> { + assertThat(d).isGreaterThanOrEqualTo(Duration.ZERO); + }); } } } - @Test - public void testApiCall_clientSideExceptionThrown_includedInMetrics() { - when(mockHttpClient.prepareRequest(any(HttpExecuteRequest.class))).thenThrow(new RuntimeException("oops")); - - thrown.expect(RuntimeException.class); - try { - client.allTypes(); - } finally { - - ArgumentCaptor collectionCaptor = ArgumentCaptor.forClass(MetricCollection.class); - verify(mockPublisher).publish(collectionCaptor.capture()); - - MetricCollection capturedCollection = collectionCaptor.getValue(); - - MetricCollection requestMetrics = capturedCollection.children().get(0); - - assertThat(requestMetrics.metricValues(CoreMetric.EXCEPTION).get(0)) - .isExactlyInstanceOf(RuntimeException.class); - } - } - - @Test - public void testApiCall_requestExecutionThrowsClientSideException_includedInMetrics() throws IOException { - IOException ioe = new IOException("oops"); - ExecutableHttpRequest mockExecutableRequest = mock(ExecutableHttpRequest.class); - when(mockExecutableRequest.call()).thenThrow(ioe); - - when(mockHttpClient.prepareRequest(any(HttpExecuteRequest.class))).thenReturn(mockExecutableRequest); - - thrown.expectCause(is(ioe)); - try { - client.allTypes(); - } finally { - ArgumentCaptor collectionCaptor = ArgumentCaptor.forClass(MetricCollection.class); - verify(mockPublisher).publish(collectionCaptor.capture()); - - MetricCollection callMetrics = collectionCaptor.getValue(); - MetricCollection attemptMetrics = callMetrics.children().get(0); - - assertThat(attemptMetrics.metricValues(CoreMetric.EXCEPTION)).containsExactly(ioe); - } - } - - @Test - public void testApiCall_streamingOutput_transformerThrows_includedInMetrics() throws IOException { - IOException ioe = new IOException("oops"); - ExecutableHttpRequest mockExecutableRequest = mock(ExecutableHttpRequest.class); - when(mockExecutableRequest.call()).thenThrow(ioe); - - when(mockHttpClient.prepareRequest(any(HttpExecuteRequest.class))).thenReturn(mockExecutableRequest); - - thrown.expectCause(is(ioe)); - try { - client.streamingOutputOperation(req -> {}, (response, is) -> { - throw ioe; - }); - } finally { - ArgumentCaptor collectionCaptor = ArgumentCaptor.forClass(MetricCollection.class); - verify(mockPublisher).publish(collectionCaptor.capture()); - - MetricCollection callMetrics = collectionCaptor.getValue(); - MetricCollection attemptMetrics = callMetrics.children().get(0); - - assertThat(attemptMetrics.metricValues(CoreMetric.EXCEPTION)).containsExactly(ioe); - } - } - private static HttpExecuteResponse mockExecuteResponse(SdkHttpFullResponse httpResponse) { HttpExecuteResponse mockResponse = mock(HttpExecuteResponse.class); when(mockResponse.httpResponse()).thenReturn(httpResponse); diff --git a/test/codegen-generated-classes-test/src/test/java/software/amazon/awssdk/services/metrics/async/BaseAsyncCoreMetricsTest.java b/test/codegen-generated-classes-test/src/test/java/software/amazon/awssdk/services/metrics/async/BaseAsyncCoreMetricsTest.java index 216477f33784..386e3cc4860d 100644 --- a/test/codegen-generated-classes-test/src/test/java/software/amazon/awssdk/services/metrics/async/BaseAsyncCoreMetricsTest.java +++ b/test/codegen-generated-classes-test/src/test/java/software/amazon/awssdk/services/metrics/async/BaseAsyncCoreMetricsTest.java @@ -25,7 +25,6 @@ import com.github.tomakehurst.wiremock.http.Fault; import com.github.tomakehurst.wiremock.stubbing.Scenario; -import java.io.IOException; import java.time.Duration; import java.util.concurrent.CompletableFuture; import java.util.function.Supplier; @@ -35,6 +34,7 @@ import org.mockito.runners.MockitoJUnitRunner; import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.core.metrics.CoreMetric; +import software.amazon.awssdk.http.HttpMetric; import software.amazon.awssdk.metrics.MetricCollection; import software.amazon.awssdk.metrics.MetricPublisher; import software.amazon.awssdk.services.protocolrestjson.model.EmptyModeledException; @@ -57,7 +57,7 @@ public void apiCall_operationSuccessful_addsMetrics() { verify(publisher()).publish(collectionCaptor.capture()); MetricCollection capturedCollection = collectionCaptor.getValue(); - verifyApiCallCollection(capturedCollection); + verifySuccessfulApiCallCollection(capturedCollection); assertThat(capturedCollection.children()).hasSize(1); MetricCollection attemptCollection = capturedCollection.children().get(0); @@ -65,7 +65,7 @@ public void apiCall_operationSuccessful_addsMetrics() { assertThat(attemptCollection.name()).isEqualTo("ApiCallAttempt"); verifySuccessfulApiCallAttemptCollection(attemptCollection); - assertThat(attemptCollection.metricValues(CoreMetric.HTTP_REQUEST_ROUND_TRIP_TIME).get(0)) + assertThat(attemptCollection.metricValues(CoreMetric.SERVICE_CALL_DURATION).get(0)) .isGreaterThanOrEqualTo(FIXED_DELAY); } @@ -79,7 +79,7 @@ public void apiCall_allRetryAttemptsFailedOf500() { verify(publisher()).publish(collectionCaptor.capture()); MetricCollection capturedCollection = collectionCaptor.getValue(); - verifyApiCallCollection(capturedCollection); + verifyFailedApiCallCollection(capturedCollection); assertThat(capturedCollection.children()).hasSize(MAX_RETRIES + 1); capturedCollection.children().forEach(this::verifyFailedApiCallAttemptCollection); @@ -95,19 +95,17 @@ public void apiCall_allRetryAttemptsFailedOfNetworkError() { verify(publisher()).publish(collectionCaptor.capture()); MetricCollection capturedCollection = collectionCaptor.getValue(); - verifyApiCallCollection(capturedCollection); + verifyFailedApiCallCollection(capturedCollection); assertThat(capturedCollection.children()).hasSize(MAX_RETRIES + 1); capturedCollection.children().forEach(requestMetrics -> { - assertThat(requestMetrics.metricValues(CoreMetric.EXCEPTION).get(0)) - .isInstanceOf(IOException.class); - assertThat(requestMetrics.metricValues(CoreMetric.HTTP_STATUS_CODE)) + assertThat(requestMetrics.metricValues(HttpMetric.HTTP_STATUS_CODE)) .isEmpty(); assertThat(requestMetrics.metricValues(CoreMetric.AWS_REQUEST_ID)) .isEmpty(); assertThat(requestMetrics.metricValues(CoreMetric.AWS_EXTENDED_REQUEST_ID)) .isEmpty(); - assertThat(requestMetrics.metricValues(CoreMetric.HTTP_REQUEST_ROUND_TRIP_TIME).get(0)) + assertThat(requestMetrics.metricValues(CoreMetric.SERVICE_CALL_DURATION).get(0)) .isGreaterThanOrEqualTo(FIXED_DELAY); }); } @@ -123,6 +121,9 @@ public void apiCall_firstAttemptFailedRetrySucceeded() { MetricCollection capturedCollection = collectionCaptor.getValue(); verifyApiCallCollection(capturedCollection); + assertThat(capturedCollection.metricValues(CoreMetric.RETRY_COUNT)).containsExactly(1); + assertThat(capturedCollection.metricValues(CoreMetric.API_CALL_SUCCESSFUL)).containsExactly(true); + assertThat(capturedCollection.children()).hasSize(2); MetricCollection failedAttempt = capturedCollection.children().get(0); @@ -150,27 +151,41 @@ void addDelayIfNeeded() { abstract MetricPublisher publisher(); private void verifyFailedApiCallAttemptCollection(MetricCollection requestMetrics) { - assertThat(requestMetrics.metricValues(CoreMetric.EXCEPTION).get(0)) - .isInstanceOf(EmptyModeledException.class); - assertThat(requestMetrics.metricValues(CoreMetric.HTTP_STATUS_CODE)) + assertThat(requestMetrics.metricValues(HttpMetric.HTTP_STATUS_CODE)) .containsExactly(500); assertThat(requestMetrics.metricValues(CoreMetric.AWS_REQUEST_ID)) .containsExactly(REQUEST_ID); assertThat(requestMetrics.metricValues(CoreMetric.AWS_EXTENDED_REQUEST_ID)) .containsExactly(EXTENDED_REQUEST_ID); - assertThat(requestMetrics.metricValues(CoreMetric.HTTP_REQUEST_ROUND_TRIP_TIME).get(0)) + assertThat(requestMetrics.metricValues(CoreMetric.BACKOFF_DELAY_DURATION).get(0)) + .isGreaterThanOrEqualTo(Duration.ZERO); + assertThat(requestMetrics.metricValues(CoreMetric.SERVICE_CALL_DURATION).get(0)) .isGreaterThanOrEqualTo(Duration.ZERO); } private void verifySuccessfulApiCallAttemptCollection(MetricCollection attemptCollection) { - assertThat(attemptCollection.metricValues(CoreMetric.HTTP_STATUS_CODE)) + assertThat(attemptCollection.metricValues(HttpMetric.HTTP_STATUS_CODE)) .containsExactly(200); assertThat(attemptCollection.metricValues(CoreMetric.AWS_REQUEST_ID)) .containsExactly(REQUEST_ID); - assertThat(attemptCollection.metricValues(CoreMetric.SIGNING_DURATION).get(0)) - .isGreaterThanOrEqualTo(Duration.ZERO); assertThat(attemptCollection.metricValues(CoreMetric.AWS_EXTENDED_REQUEST_ID)) .containsExactly(EXTENDED_REQUEST_ID); + assertThat(attemptCollection.metricValues(CoreMetric.BACKOFF_DELAY_DURATION).get(0)) + .isGreaterThanOrEqualTo(Duration.ZERO); + assertThat(attemptCollection.metricValues(CoreMetric.SIGNING_DURATION).get(0)) + .isGreaterThanOrEqualTo(Duration.ZERO); + } + + private void verifyFailedApiCallCollection(MetricCollection capturedCollection) { + verifyApiCallCollection(capturedCollection); + assertThat(capturedCollection.metricValues(CoreMetric.RETRY_COUNT)).containsExactly(MAX_RETRIES); + assertThat(capturedCollection.metricValues(CoreMetric.API_CALL_SUCCESSFUL)).containsExactly(false); + } + + private void verifySuccessfulApiCallCollection(MetricCollection capturedCollection) { + verifyApiCallCollection(capturedCollection); + assertThat(capturedCollection.metricValues(CoreMetric.RETRY_COUNT)).containsExactly(0); + assertThat(capturedCollection.metricValues(CoreMetric.API_CALL_SUCCESSFUL)).containsExactly(true); } private void verifyApiCallCollection(MetricCollection capturedCollection) { diff --git a/utils/src/main/java/software/amazon/awssdk/utils/http/SdkHttpUtils.java b/utils/src/main/java/software/amazon/awssdk/utils/http/SdkHttpUtils.java index 10d8d5222cff..b1d2fa7773c9 100644 --- a/utils/src/main/java/software/amazon/awssdk/utils/http/SdkHttpUtils.java +++ b/utils/src/main/java/software/amazon/awssdk/utils/http/SdkHttpUtils.java @@ -21,6 +21,7 @@ import java.net.URI; import java.net.URLDecoder; import java.net.URLEncoder; +import java.util.Collection; import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; @@ -276,6 +277,21 @@ public static Stream allMatchingHeaders(Map> header .flatMap(e -> e.getValue() != null ? e.getValue().stream() : Stream.empty()); } + /** + * Perform a case-insensitive search for a particular header in the provided map of headers. + * + * @param headersToSearch The headers to search. + * @param headersToFind The headers to search for (case insensitively). + * @return A stream providing the values for the headers that matched the requested header. + */ + public static Stream allMatchingHeadersFromCollection(Map> headersToSearch, + Collection headersToFind) { + return headersToSearch.entrySet().stream() + .filter(e -> headersToFind.stream() + .anyMatch(headerToFind -> e.getKey().equalsIgnoreCase(headerToFind))) + .flatMap(e -> e.getValue() != null ? e.getValue().stream() : Stream.empty()); + } + /** * Perform a case-insensitive search for a particular header in the provided map of headers, returning the first matching * header, if one is found. @@ -290,6 +306,19 @@ public static Optional firstMatchingHeader(Map> hea return allMatchingHeaders(headers, header).findFirst(); } + /** + * Perform a case-insensitive search for a set of headers in the provided map of headers, returning the first matching + * header, if one is found. + * + * @param headersToSearch The headers to search. + * @param headersToFind The header to search for (case insensitively). + * @return The first header that matched a requested one, or empty if one was not found. + */ + public static Optional firstMatchingHeaderFromCollection(Map> headersToSearch, + Collection headersToFind) { + return allMatchingHeadersFromCollection(headersToSearch, headersToFind).findFirst(); + } + public static boolean isSingleHeader(String h) { return SINGLE_HEADERS.contains(StringUtils.lowerCase(h)); } diff --git a/utils/src/test/java/software/amazon/awssdk/utils/SdkHttpUtilsTest.java b/utils/src/test/java/software/amazon/awssdk/utils/SdkHttpUtilsTest.java index 99accd9ac2b4..0d885deb410e 100644 --- a/utils/src/test/java/software/amazon/awssdk/utils/SdkHttpUtilsTest.java +++ b/utils/src/test/java/software/amazon/awssdk/utils/SdkHttpUtilsTest.java @@ -148,4 +148,29 @@ public void headerRetrievalWorksCorrectly() { assertThat(SdkHttpUtils.firstMatchingHeader(headers, null)).isNotPresent(); assertThat(SdkHttpUtils.firstMatchingHeader(headers, "nothing")).isNotPresent(); } + + @Test + public void headersFromCollectionWorksCorrectly() { + Map> headers = new HashMap<>(); + headers.put("FOO", asList("bar", "baz")); + headers.put("foo", singletonList(null)); + headers.put("other", singletonList("foo")); + headers.put("Foo", singletonList("baz2")); + + assertThat(SdkHttpUtils.allMatchingHeadersFromCollection(headers, asList("nothing"))).isEmpty(); + assertThat(SdkHttpUtils.allMatchingHeadersFromCollection(headers, asList("foo"))) + .containsExactlyInAnyOrder("bar", "baz", null, "baz2"); + assertThat(SdkHttpUtils.allMatchingHeadersFromCollection(headers, asList("nothing", "foo"))) + .containsExactlyInAnyOrder("bar", "baz", null, "baz2"); + assertThat(SdkHttpUtils.allMatchingHeadersFromCollection(headers, asList("foo", "nothing"))) + .containsExactlyInAnyOrder("bar", "baz", null, "baz2"); + assertThat(SdkHttpUtils.allMatchingHeadersFromCollection(headers, asList("foo", "other"))) + .containsExactlyInAnyOrder("bar", "baz", null, "foo", "baz2"); + + assertThat(SdkHttpUtils.firstMatchingHeaderFromCollection(headers, asList("nothing"))).isEmpty(); + assertThat(SdkHttpUtils.firstMatchingHeaderFromCollection(headers, asList("foo"))).hasValue("bar"); + assertThat(SdkHttpUtils.firstMatchingHeaderFromCollection(headers, asList("nothing", "foo"))).hasValue("bar"); + assertThat(SdkHttpUtils.firstMatchingHeaderFromCollection(headers, asList("foo", "nothing"))).hasValue("bar"); + assertThat(SdkHttpUtils.firstMatchingHeaderFromCollection(headers, asList("foo", "other"))).hasValue("foo"); + } }