diff --git a/pom.xml b/pom.xml index 462a71778..8f2701054 100644 --- a/pom.xml +++ b/pom.xml @@ -37,7 +37,8 @@ powertools-validation powertools-test-suite powertools-cloudformation - powertools-idempotency + powertools-idempotency + powertools-e2e-tests examples diff --git a/powertools-e2e-tests/README.md b/powertools-e2e-tests/README.md new file mode 100644 index 000000000..42fe9191b --- /dev/null +++ b/powertools-e2e-tests/README.md @@ -0,0 +1,16 @@ +## End-to-end tests +This module is internal and meant to be used for end-to-end (E2E) testing of Lambda Powertools for Java. + +__Prerequisites__: +- An AWS account is needed as well as a local environment able to reach this account +([credentials](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/credentials.html)). +- [Java 11+](https://docs.aws.amazon.com/corretto/latest/corretto-11-ug/downloads-list.html) +- [Docker](https://docs.docker.com/engine/install/) + +To execute the E2E tests, use the following command: `export JAVA_VERSION=11 && mvn clean verify -Pe2e` + +### Under the hood +This module leverages the following components: +- AWS CDK to define the infrastructure and synthesize a CloudFormation template and the assets (lambda function packages) +- The AWS S3 SDK to push the assets on S3 +- The AWS CloudFormation SDK to deploy the template \ No newline at end of file diff --git a/powertools-e2e-tests/handlers/idempotency/pom.xml b/powertools-e2e-tests/handlers/idempotency/pom.xml new file mode 100644 index 000000000..aa7870389 --- /dev/null +++ b/powertools-e2e-tests/handlers/idempotency/pom.xml @@ -0,0 +1,57 @@ + + 4.0.0 + + + software.amazon.lambda + e2e-test-handlers-parent + 1.0.0 + + + e2e-test-handler-idempotency + jar + A Lambda function using powertools idempotency + + + + software.amazon.lambda + powertools-idempotency + + + com.amazonaws + aws-lambda-java-events + + + + + + + + org.codehaus.mojo + aspectj-maven-plugin + + ${maven.compiler.source} + ${maven.compiler.target} + ${maven.compiler.target} + + + software.amazon.lambda + powertools-idempotency + + + + + + + compile + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + diff --git a/powertools-e2e-tests/handlers/idempotency/src/main/java/software/amazon/lambda/powertools/e2e/Function.java b/powertools-e2e-tests/handlers/idempotency/src/main/java/software/amazon/lambda/powertools/e2e/Function.java new file mode 100644 index 000000000..cc6eec4fa --- /dev/null +++ b/powertools-e2e-tests/handlers/idempotency/src/main/java/software/amazon/lambda/powertools/e2e/Function.java @@ -0,0 +1,48 @@ +package software.amazon.lambda.powertools.e2e; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.lambda.powertools.idempotency.Idempotency; +import software.amazon.lambda.powertools.idempotency.IdempotencyConfig; +import software.amazon.lambda.powertools.idempotency.Idempotent; +import software.amazon.lambda.powertools.idempotency.persistence.DynamoDBPersistenceStore; + +import java.time.Duration; +import java.time.Instant; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; +import java.util.TimeZone; + + +public class Function implements RequestHandler { + + public Function() { + this(DynamoDbClient + .builder() + .httpClient(UrlConnectionHttpClient.builder().build()) + .region(Region.of(System.getenv("AWS_REGION"))) + .build()); + } + + public Function(DynamoDbClient client) { + Idempotency.config().withConfig( + IdempotencyConfig.builder() + .withExpiration(Duration.of(10, ChronoUnit.SECONDS)) + .build()) + .withPersistenceStore( + DynamoDBPersistenceStore.builder() + .withDynamoDbClient(client) + .withTableName(System.getenv("IDEMPOTENCY_TABLE")) + .build() + ).configure(); + } + + @Idempotent + public String handleRequest(Input input, Context context) { + DateTimeFormatter dtf = DateTimeFormatter.ISO_DATE_TIME.withZone(TimeZone.getTimeZone("UTC").toZoneId()); + return dtf.format(Instant.now()); + } +} \ No newline at end of file diff --git a/powertools-e2e-tests/handlers/idempotency/src/main/java/software/amazon/lambda/powertools/e2e/Input.java b/powertools-e2e-tests/handlers/idempotency/src/main/java/software/amazon/lambda/powertools/e2e/Input.java new file mode 100644 index 000000000..c5c2a121e --- /dev/null +++ b/powertools-e2e-tests/handlers/idempotency/src/main/java/software/amazon/lambda/powertools/e2e/Input.java @@ -0,0 +1,20 @@ +package software.amazon.lambda.powertools.e2e; + +public class Input { + private String message; + + public Input(String message) { + this.message = message; + } + + public Input() { + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } +} diff --git a/powertools-e2e-tests/handlers/logging/pom.xml b/powertools-e2e-tests/handlers/logging/pom.xml new file mode 100644 index 000000000..a46702ca4 --- /dev/null +++ b/powertools-e2e-tests/handlers/logging/pom.xml @@ -0,0 +1,57 @@ + + 4.0.0 + + + software.amazon.lambda + e2e-test-handlers-parent + 1.0.0 + + + e2e-test-handler-logging + jar + A Lambda function using powertools logging + + + + software.amazon.lambda + powertools-logging + + + com.amazonaws + aws-lambda-java-events + + + + + + + + org.codehaus.mojo + aspectj-maven-plugin + + ${maven.compiler.source} + ${maven.compiler.target} + ${maven.compiler.target} + + + software.amazon.lambda + powertools-logging + + + + + + + compile + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + diff --git a/powertools-e2e-tests/handlers/logging/src/main/java/software/amazon/lambda/powertools/e2e/Function.java b/powertools-e2e-tests/handlers/logging/src/main/java/software/amazon/lambda/powertools/e2e/Function.java new file mode 100644 index 000000000..5a9a87109 --- /dev/null +++ b/powertools-e2e-tests/handlers/logging/src/main/java/software/amazon/lambda/powertools/e2e/Function.java @@ -0,0 +1,22 @@ +package software.amazon.lambda.powertools.e2e; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import software.amazon.lambda.powertools.logging.Logging; +import software.amazon.lambda.powertools.logging.LoggingUtils; + +public class Function implements RequestHandler { + + private static final Logger LOG = LogManager.getLogger(Function.class); + + @Logging + public String handleRequest(Input input, Context context) { + + LoggingUtils.appendKeys(input.getKeys()); + LOG.info(input.getMessage()); + + return "OK"; + } +} \ No newline at end of file diff --git a/powertools-e2e-tests/handlers/logging/src/main/java/software/amazon/lambda/powertools/e2e/Input.java b/powertools-e2e-tests/handlers/logging/src/main/java/software/amazon/lambda/powertools/e2e/Input.java new file mode 100644 index 000000000..83afbbd5a --- /dev/null +++ b/powertools-e2e-tests/handlers/logging/src/main/java/software/amazon/lambda/powertools/e2e/Input.java @@ -0,0 +1,27 @@ +package software.amazon.lambda.powertools.e2e; + +import java.util.Map; + +public class Input { + private String message; + private Map keys; + + public Input() { + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public Map getKeys() { + return keys; + } + + public void setKeys(Map keys) { + this.keys = keys; + } +} diff --git a/powertools-e2e-tests/handlers/logging/src/main/resources/log4j2.xml b/powertools-e2e-tests/handlers/logging/src/main/resources/log4j2.xml new file mode 100644 index 000000000..8925f70b9 --- /dev/null +++ b/powertools-e2e-tests/handlers/logging/src/main/resources/log4j2.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/powertools-e2e-tests/handlers/metrics/pom.xml b/powertools-e2e-tests/handlers/metrics/pom.xml new file mode 100644 index 000000000..e591f4966 --- /dev/null +++ b/powertools-e2e-tests/handlers/metrics/pom.xml @@ -0,0 +1,57 @@ + + 4.0.0 + + + software.amazon.lambda + e2e-test-handlers-parent + 1.0.0 + + + e2e-test-handler-metrics + jar + A Lambda function using powertools metrics + + + + software.amazon.lambda + powertools-metrics + + + com.amazonaws + aws-lambda-java-events + + + + + + + + org.codehaus.mojo + aspectj-maven-plugin + + ${maven.compiler.source} + ${maven.compiler.target} + ${maven.compiler.target} + + + software.amazon.lambda + powertools-metrics + + + + + + + compile + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + diff --git a/powertools-e2e-tests/handlers/metrics/src/main/java/software/amazon/lambda/powertools/e2e/Function.java b/powertools-e2e-tests/handlers/metrics/src/main/java/software/amazon/lambda/powertools/e2e/Function.java new file mode 100644 index 000000000..e643de9d5 --- /dev/null +++ b/powertools-e2e-tests/handlers/metrics/src/main/java/software/amazon/lambda/powertools/e2e/Function.java @@ -0,0 +1,26 @@ +package software.amazon.lambda.powertools.e2e; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import software.amazon.cloudwatchlogs.emf.logger.MetricsLogger; +import software.amazon.cloudwatchlogs.emf.model.DimensionSet; +import software.amazon.cloudwatchlogs.emf.model.Unit; +import software.amazon.lambda.powertools.metrics.Metrics; +import software.amazon.lambda.powertools.metrics.MetricsUtils; + +public class Function implements RequestHandler { + + MetricsLogger metricsLogger = MetricsUtils.metricsLogger(); + + @Metrics(captureColdStart = true) + public String handleRequest(Input input, Context context) { + + DimensionSet dimensionSet = new DimensionSet(); + input.getDimensions().forEach((key, value) -> dimensionSet.addDimension(key, value)); + metricsLogger.putDimensions(dimensionSet); + + input.getMetrics().forEach((key, value) -> metricsLogger.putMetric(key, value, Unit.COUNT)); + + return "OK"; + } +} \ No newline at end of file diff --git a/powertools-e2e-tests/handlers/metrics/src/main/java/software/amazon/lambda/powertools/e2e/Input.java b/powertools-e2e-tests/handlers/metrics/src/main/java/software/amazon/lambda/powertools/e2e/Input.java new file mode 100644 index 000000000..5ff8a7125 --- /dev/null +++ b/powertools-e2e-tests/handlers/metrics/src/main/java/software/amazon/lambda/powertools/e2e/Input.java @@ -0,0 +1,29 @@ +package software.amazon.lambda.powertools.e2e; + +import java.util.Map; + +public class Input { + private Map metrics; + + private Map dimensions; + + public Map getMetrics() { + return metrics; + } + + public void setMetrics(Map metrics) { + this.metrics = metrics; + } + + public Input() { + } + + + public Map getDimensions() { + return dimensions; + } + + public void setDimensions(Map dimensions) { + this.dimensions = dimensions; + } +} diff --git a/powertools-e2e-tests/handlers/pom.xml b/powertools-e2e-tests/handlers/pom.xml new file mode 100644 index 000000000..b82a9a0c9 --- /dev/null +++ b/powertools-e2e-tests/handlers/pom.xml @@ -0,0 +1,119 @@ + + 4.0.0 + + software.amazon.lambda + e2e-test-handlers-parent + 1.0.0 + pom + Handlers for End-to-End tests + Fake handlers that use Lambda Powertools for Java. + + + 1.14.0 + UTF-8 + 11 + 11 + + 1.2.2 + 3.11.0 + 3.2.4 + 1.14.0 + 3.10.1 + + + + logging + tracing + metrics + idempotency + + + + + + software.amazon.lambda + powertools-logging + ${lambda.powertools.version} + + + software.amazon.lambda + powertools-tracing + ${lambda.powertools.version} + + + software.amazon.lambda + powertools-metrics + ${lambda.powertools.version} + + + software.amazon.lambda + powertools-idempotency + ${lambda.powertools.version} + + + com.amazonaws + aws-lambda-java-core + ${lambda.java.core} + + + com.amazonaws + aws-lambda-java-events + ${lambda.java.events} + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + ${maven.shade.version} + + false + function + + + + package + + shade + + + + + + + + + + + + io.github.edwgiz + log4j-maven-shade-plugin-extensions + 2.17.2 + + + + + org.codehaus.mojo + aspectj-maven-plugin + ${aspectj.version} + + + org.apache.maven.plugins + maven-compiler-plugin + ${maven.compiler.version} + + ${maven.compiler.source} + ${maven.compiler.target} + + + + + + + diff --git a/powertools-e2e-tests/handlers/tracing/pom.xml b/powertools-e2e-tests/handlers/tracing/pom.xml new file mode 100644 index 000000000..831669a3d --- /dev/null +++ b/powertools-e2e-tests/handlers/tracing/pom.xml @@ -0,0 +1,57 @@ + + 4.0.0 + + + software.amazon.lambda + e2e-test-handlers-parent + 1.0.0 + + + e2e-test-handler-tracing + jar + A Lambda function using powertools tracing + + + + software.amazon.lambda + powertools-tracing + + + com.amazonaws + aws-lambda-java-events + + + + + + + + org.codehaus.mojo + aspectj-maven-plugin + + ${maven.compiler.source} + ${maven.compiler.target} + ${maven.compiler.target} + + + software.amazon.lambda + powertools-tracing + + + + + + + compile + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + diff --git a/powertools-e2e-tests/handlers/tracing/src/main/java/software/amazon/lambda/powertools/e2e/Function.java b/powertools-e2e-tests/handlers/tracing/src/main/java/software/amazon/lambda/powertools/e2e/Function.java new file mode 100644 index 000000000..f7b2c5e5d --- /dev/null +++ b/powertools-e2e-tests/handlers/tracing/src/main/java/software/amazon/lambda/powertools/e2e/Function.java @@ -0,0 +1,40 @@ +package software.amazon.lambda.powertools.e2e; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import software.amazon.lambda.powertools.tracing.Tracing; +import software.amazon.lambda.powertools.tracing.TracingUtils; + +public class Function implements RequestHandler { + + @Tracing + public String handleRequest(Input input, Context context) { + try { + Thread.sleep(100); // simulate stuff + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + String message = buildMessage(input.getMessage(), context.getFunctionName()); + + TracingUtils.withSubsegment("internal_stuff", subsegment -> { + try { + Thread.sleep(100); // simulate stuff + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + + return message; + } + + @Tracing + private String buildMessage(String message, String funcName) { + TracingUtils.putAnnotation("message", message); + try { + Thread.sleep(150); // simulate other stuff + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return String.format("%s (%s)", message, funcName); + } +} \ No newline at end of file diff --git a/powertools-e2e-tests/handlers/tracing/src/main/java/software/amazon/lambda/powertools/e2e/Input.java b/powertools-e2e-tests/handlers/tracing/src/main/java/software/amazon/lambda/powertools/e2e/Input.java new file mode 100644 index 000000000..29cf618ba --- /dev/null +++ b/powertools-e2e-tests/handlers/tracing/src/main/java/software/amazon/lambda/powertools/e2e/Input.java @@ -0,0 +1,17 @@ +package software.amazon.lambda.powertools.e2e; + +public class Input { + private String message; + + public Input() { + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + +} diff --git a/powertools-e2e-tests/handlers/tracing/src/main/resources/log4j2.xml b/powertools-e2e-tests/handlers/tracing/src/main/resources/log4j2.xml new file mode 100644 index 000000000..8925f70b9 --- /dev/null +++ b/powertools-e2e-tests/handlers/tracing/src/main/resources/log4j2.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/powertools-e2e-tests/pom.xml b/powertools-e2e-tests/pom.xml new file mode 100644 index 000000000..77e9fa458 --- /dev/null +++ b/powertools-e2e-tests/pom.xml @@ -0,0 +1,175 @@ + + + 4.0.0 + + powertools-parent + software.amazon.lambda + 1.14.0 + + + powertools-e2e-tests + AWS Lambda Powertools for Java library End-to-end tests + AWS Lambda Powertools for Java End-To-End Tests + + + + 8 + 8 + 10.1.138 + 2.47.0 + + + + + ch.qos.logback + logback-classic + 1.4.4 + + + + software.amazon.awssdk + lambda + ${aws.sdk.version} + test + + + + software.amazon.awssdk + cloudwatch + ${aws.sdk.version} + test + + + + software.amazon.awssdk + xray + ${aws.sdk.version} + test + + + + software.amazon.awssdk + url-connection-client + test + + + + org.junit.jupiter + junit-jupiter-api + test + + + + org.junit.jupiter + junit-jupiter-engine + test + + + + org.assertj + assertj-core + test + + + + com.evanlennick + retry4j + 0.15.0 + test + + + + software.amazon.awscdk + aws-cdk-lib + ${cdk.version} + test + + + + software.constructs + constructs + ${constructs.version} + test + + + software.amazon.awssdk + s3 + ${aws.sdk.version} + test + + + software.amazon.awssdk + cloudformation + ${aws.sdk.version} + test + + + software.amazon.awssdk + sts + ${aws.sdk.version} + test + + + org.yaml + snakeyaml + 1.33 + test + + + org.aspectj + aspectjrt + compile + + + software.amazon.lambda + powertools-serialization + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.10.1 + + ${maven.compiler.source} + ${maven.compiler.target} + UTF-8 + + + + + + + + e2e + + + + org.apache.maven.plugins + maven-failsafe-plugin + 2.22.2 + + + + integration-test + verify + + + + + + **/*E2ET.java + + + + + + + + + \ No newline at end of file diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/IdempotencyE2ET.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/IdempotencyE2ET.java new file mode 100644 index 000000000..4133f986f --- /dev/null +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/IdempotencyE2ET.java @@ -0,0 +1,61 @@ +package software.amazon.lambda.powertools; + +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import software.amazon.lambda.powertools.testutils.Infrastructure; +import software.amazon.lambda.powertools.testutils.lambda.InvocationResult; + +import java.time.Year; +import java.util.Collections; +import java.util.concurrent.TimeUnit; + +import static software.amazon.lambda.powertools.testutils.lambda.LambdaInvoker.invokeFunction; + +public class IdempotencyE2ET { + private static Infrastructure infrastructure; + private static String functionName; + + @BeforeAll + @Timeout(value = 5, unit = TimeUnit.MINUTES) + public static void setup() { + infrastructure = Infrastructure.builder() + .testName(IdempotencyE2ET.class.getSimpleName()) + .pathToFunction("idempotency") + .idempotencyTable("idempo") + .environmentVariables(Collections.singletonMap("IDEMPOTENCY_TABLE", "idempo")) + .build(); + functionName = infrastructure.deploy(); + } + + @AfterAll + public static void tearDown() { + if (infrastructure != null) + infrastructure.destroy(); + } + + @Test + public void test_ttlNotExpired_sameResult_ttlExpired_differentResult() throws InterruptedException { + // GIVEN + String event = "{\"message\":\"TTL 10sec\"}"; + + // WHEN + // First invocation + InvocationResult result1 = invokeFunction(functionName, event); + + // Second invocation (should get same result) + InvocationResult result2 = invokeFunction(functionName, event); + + Thread.sleep(12000); + + // Third invocation (should get different result) + InvocationResult result3 = invokeFunction(functionName, event); + + // THEN + Assertions.assertThat(result1.getResult()).contains(Year.now().toString()); + Assertions.assertThat(result2.getResult()).isEqualTo(result1.getResult()); + Assertions.assertThat(result3.getResult()).isNotEqualTo(result2.getResult()); + } +} diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/LoggingE2ET.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/LoggingE2ET.java new file mode 100644 index 000000000..15c5eb935 --- /dev/null +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/LoggingE2ET.java @@ -0,0 +1,77 @@ +package software.amazon.lambda.powertools; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import software.amazon.lambda.powertools.testutils.Infrastructure; +import software.amazon.lambda.powertools.testutils.lambda.InvocationResult; + +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static software.amazon.lambda.powertools.testutils.lambda.LambdaInvoker.invokeFunction; +import static software.amazon.lambda.powertools.testutils.logging.InvocationLogs.Level.INFO; + +public class LoggingE2ET { + + private static final ObjectMapper objectMapper = new ObjectMapper(); + + private static Infrastructure infrastructure; + private static String functionName; + + @BeforeAll + @Timeout(value = 5, unit = TimeUnit.MINUTES) + public static void setup() { + infrastructure = Infrastructure.builder() + .testName(LoggingE2ET.class.getSimpleName()) + .pathToFunction("logging") + .environmentVariables( + Stream.of(new String[][]{ + {"POWERTOOLS_LOG_LEVEL", "INFO"}, + {"POWERTOOLS_SERVICE_NAME", LoggingE2ET.class.getSimpleName()} + }) + .collect(Collectors.toMap(data -> data[0], data -> data[1]))) + .build(); + functionName = infrastructure.deploy(); + } + + @AfterAll + public static void tearDown() { + if (infrastructure != null) + infrastructure.destroy(); + } + + @Test + public void test_logInfoWithAdditionalKeys() throws JsonProcessingException { + // GIVEN + String orderId = UUID.randomUUID().toString(); + String event = "{\"message\":\"New Order\", \"keys\":{\"orderId\":\"" + orderId +"\"}}"; + + // WHEN + InvocationResult invocationResult1 = invokeFunction(functionName, event); + InvocationResult invocationResult2 = invokeFunction(functionName, event); + + // THEN + String[] functionLogs = invocationResult1.getLogs().getFunctionLogs(INFO); + assertThat(functionLogs).hasSize(1); + + JsonNode jsonNode = objectMapper.readTree(functionLogs[0]); + assertThat(jsonNode.get("message").asText()).isEqualTo("New Order"); + assertThat(jsonNode.get("orderId").asText()).isEqualTo(orderId); + assertThat(jsonNode.get("coldStart").asBoolean()).isTrue(); + assertThat(jsonNode.get("function_request_id").asText()).isEqualTo(invocationResult1.getRequestId()); + + // second call should not be cold start + functionLogs = invocationResult2.getLogs().getFunctionLogs(INFO); + assertThat(functionLogs).hasSize(1); + jsonNode = objectMapper.readTree(functionLogs[0]); + assertThat(jsonNode.get("coldStart").asBoolean()).isFalse(); + } +} diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/MetricsE2ET.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/MetricsE2ET.java new file mode 100644 index 000000000..4b8d7ea5a --- /dev/null +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/MetricsE2ET.java @@ -0,0 +1,73 @@ +package software.amazon.lambda.powertools; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import software.amazon.lambda.powertools.testutils.Infrastructure; +import software.amazon.lambda.powertools.testutils.lambda.InvocationResult; +import software.amazon.lambda.powertools.testutils.metrics.MetricsFetcher; + +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static software.amazon.lambda.powertools.testutils.lambda.LambdaInvoker.invokeFunction; + +public class MetricsE2ET { + private static final String namespace = "MetricsE2ENamespace_"+UUID.randomUUID(); + private static final String service = "MetricsE2EService_"+UUID.randomUUID(); + private static Infrastructure infrastructure; + private static String functionName; + + @BeforeAll + @Timeout(value = 5, unit = TimeUnit.MINUTES) + public static void setup() { + infrastructure = Infrastructure.builder() + .testName(MetricsE2ET.class.getSimpleName()) + .pathToFunction("metrics") + .environmentVariables( + Stream.of(new String[][]{ + {"POWERTOOLS_METRICS_NAMESPACE", namespace}, + {"POWERTOOLS_SERVICE_NAME", service} + }) + .collect(Collectors.toMap(data -> data[0], data -> data[1]))) + .build(); + functionName = infrastructure.deploy(); + } + + @AfterAll + public static void tearDown() { + if (infrastructure != null) + infrastructure.destroy(); + } + + @Test + public void test_recordMetrics() { + // GIVEN + String event1 = "{ \"metrics\": {\"orders\": 1, \"products\": 4}, \"dimensions\": { \"Environment\": \"test\"} }"; + + // WHEN + InvocationResult invocationResult = invokeFunction(functionName, event1); + + // THEN + MetricsFetcher metricsFetcher = new MetricsFetcher(); + List coldStart = metricsFetcher.fetchMetrics(invocationResult.getStart(), invocationResult.getEnd(), 60, namespace, "ColdStart", Stream.of(new String[][]{ + {"FunctionName", functionName}, + {"Service", service}} + ).collect(Collectors.toMap(data -> data[0], data -> data[1]))); + assertThat(coldStart.get(0)).isEqualTo(1); + List orderMetrics = metricsFetcher.fetchMetrics(invocationResult.getStart(), invocationResult.getEnd(), 60, namespace, "orders", Collections.singletonMap("Environment", "test")); + assertThat(orderMetrics.get(0)).isEqualTo(1); + List productMetrics = metricsFetcher.fetchMetrics(invocationResult.getStart(), invocationResult.getEnd(), 60, namespace, "products", Collections.singletonMap("Environment", "test")); + assertThat(productMetrics.get(0)).isEqualTo(4); + orderMetrics = metricsFetcher.fetchMetrics(invocationResult.getStart(), invocationResult.getEnd(), 60, namespace, "orders", Collections.singletonMap("Service", service)); + assertThat(orderMetrics.get(0)).isEqualTo(1); + productMetrics = metricsFetcher.fetchMetrics(invocationResult.getStart(), invocationResult.getEnd(), 60, namespace, "products", Collections.singletonMap("Service", service)); + assertThat(productMetrics.get(0)).isEqualTo(4); + } +} diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/TracingE2ET.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/TracingE2ET.java new file mode 100644 index 000000000..1f002fe60 --- /dev/null +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/TracingE2ET.java @@ -0,0 +1,88 @@ +package software.amazon.lambda.powertools; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import software.amazon.lambda.powertools.testutils.Infrastructure; +import software.amazon.lambda.powertools.testutils.lambda.InvocationResult; +import software.amazon.lambda.powertools.testutils.tracing.SegmentDocument.SubSegment; +import software.amazon.lambda.powertools.testutils.tracing.Trace; +import software.amazon.lambda.powertools.testutils.tracing.TraceFetcher; + +import java.util.Collections; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static software.amazon.lambda.powertools.testutils.lambda.LambdaInvoker.invokeFunction; + +public class TracingE2ET { + private static final String service = "TracingE2EService_" + UUID.randomUUID(); // "TracingE2EService_e479fb27-422b-4107-9f8c-086c62e1cd12"; + + private static Infrastructure infrastructure; + private static String functionName; + + @BeforeAll + @Timeout(value = 5, unit = TimeUnit.MINUTES) + public static void setup() { + infrastructure = Infrastructure.builder() + .testName(TracingE2ET.class.getSimpleName()) + .pathToFunction("tracing") + .tracing(true) + .environmentVariables(Collections.singletonMap("POWERTOOLS_SERVICE_NAME", service)) + .build(); + functionName = infrastructure.deploy(); + } + + @AfterAll + public static void tearDown() { + if (infrastructure != null) + infrastructure.destroy(); + } + + @Test + public void test_tracing() { + // GIVEN + String message = "Hello World"; + String event = String.format("{\"message\":\"%s\"}", message); + String result = String.format("%s (%s)", message, functionName); + + // WHEN + InvocationResult invocationResult = invokeFunction(functionName, event); + + // THEN + Trace trace = TraceFetcher.builder() + .start(invocationResult.getStart()) + .end(invocationResult.getEnd()) + .functionName(functionName) + .build() + .fetchTrace(); + + assertThat(trace.getSubsegments()).hasSize(1); + SubSegment handleRequest = trace.getSubsegments().get(0); + assertThat(handleRequest.getName()).isEqualTo("## handleRequest"); + assertThat(handleRequest.getAnnotations()).hasSize(2); + assertThat(handleRequest.getAnnotations().get("ColdStart")).isEqualTo(true); + assertThat(handleRequest.getAnnotations().get("Service")).isEqualTo(service); + assertThat(handleRequest.getMetadata()).hasSize(1); + Map metadata = (Map) handleRequest.getMetadata().get(service); + assertThat(metadata.get("handleRequest response")).isEqualTo(result); + assertThat(handleRequest.getSubsegments()).hasSize(2); + + SubSegment sub = handleRequest.getSubsegments().get(0); + assertThat(sub.getName()).isIn("## internal_stuff", "## buildMessage"); + + sub = handleRequest.getSubsegments().get(1); + assertThat(sub.getName()).isIn("## internal_stuff", "## buildMessage"); + + SubSegment buildMessage = handleRequest.getSubsegments().stream().filter(subSegment -> subSegment.getName().equals("## buildMessage")).findFirst().orElse(null); + assertThat(buildMessage).isNotNull(); + assertThat(buildMessage.getAnnotations()).hasSize(1); + assertThat(buildMessage.getAnnotations().get("message")).isEqualTo(message); + assertThat(buildMessage.getMetadata()).hasSize(1); + metadata = (Map) buildMessage.getMetadata().get(service); + assertThat(metadata.get("buildMessage response")).isEqualTo(result); + } +} diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java new file mode 100644 index 000000000..f3659e5c7 --- /dev/null +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java @@ -0,0 +1,355 @@ +package software.amazon.lambda.powertools.testutils; + +import com.fasterxml.jackson.databind.JsonNode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.yaml.snakeyaml.Yaml; +import software.amazon.awscdk.Stack; +import software.amazon.awscdk.*; +import software.amazon.awscdk.cxapi.CloudAssembly; +import software.amazon.awscdk.services.dynamodb.Attribute; +import software.amazon.awscdk.services.dynamodb.AttributeType; +import software.amazon.awscdk.services.dynamodb.BillingMode; +import software.amazon.awscdk.services.dynamodb.Table; +import software.amazon.awscdk.services.lambda.Code; +import software.amazon.awscdk.services.lambda.Function; +import software.amazon.awscdk.services.lambda.Tracing; +import software.amazon.awscdk.services.logs.LogGroup; +import software.amazon.awscdk.services.logs.RetentionDays; +import software.amazon.awscdk.services.s3.assets.AssetOptions; +import software.amazon.awssdk.core.waiters.WaiterResponse; +import software.amazon.awssdk.http.SdkHttpClient; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.cloudformation.CloudFormationClient; +import software.amazon.awssdk.services.cloudformation.model.*; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.utils.StringUtils; +import software.amazon.lambda.powertools.utilities.JsonConfig; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.*; +import java.util.stream.Collectors; + +import static java.util.Collections.singletonList; + +/** + * This class is in charge of bootstrapping the infrastructure for the tests. + *
+ * Tests are actually run on AWS, so we need to provision Lambda functions, DynamoDB table (for Idempotency), + * CloudWatch log groups, ... + *
+ * It uses the Cloud Development Kit (CDK) to define required resources. The CDK stack is then synthesized to retrieve + * the CloudFormation templates and the assets (function jars). Assets are uploaded to S3 (with the SDK `PutObjectRequest`) + * and the CloudFormation stack is created (with the SDK `createStack`) + */ +public class Infrastructure { + private static final Logger LOG = LoggerFactory.getLogger(Infrastructure.class); + + private final String stackName; + private final boolean tracing; + private final Map envVar; + private final JavaRuntime runtime; + private final App app; + private final Stack stack; + private final long timeout; + private final String pathToFunction; + private final S3Client s3; + private final CloudFormationClient cfn; + private final Region region; + private final String account; + private final String idempotencyTable; + private String functionName; + private Object cfnTemplate; + private String cfnAssetDirectory; + private final SdkHttpClient httpClient; + + private Infrastructure(Builder builder) { + this.stackName = builder.stackName; + this.tracing = builder.tracing; + this.envVar = builder.environmentVariables; + this.runtime = builder.runtime; + this.timeout = builder.timeoutInSeconds; + this.pathToFunction = builder.pathToFunction; + this.idempotencyTable = builder.idemPotencyTable; + + this.app = new App(); + this.stack = createStackWithLambda(); + + this.synthesize(); + + this.httpClient = UrlConnectionHttpClient.builder().build(); + this.region = Region.of(System.getProperty("AWS_DEFAULT_REGION", "eu-west-1")); + this.account = StsClient.builder() + .httpClient(httpClient) + .region(region) + .build().getCallerIdentity().account(); + + s3 = S3Client.builder() + .httpClient(httpClient) + .region(region) + .build(); + cfn = CloudFormationClient.builder() + .httpClient(httpClient) + .region(region) + .build(); + } + + /** + * Use the CloudFormation SDK to create the stack + * @return the name of the function deployed part of the stack + */ + public String deploy() { + uploadAssets(); + LOG.info("Deploying '" + stackName + "' on account " + account); + cfn.createStack(CreateStackRequest.builder() + .stackName(stackName) + .templateBody(new Yaml().dump(cfnTemplate)) + .timeoutInMinutes(10) + .onFailure(OnFailure.ROLLBACK) + .capabilities(Capability.CAPABILITY_IAM) + .build()); + WaiterResponse waiterResponse = cfn.waiter().waitUntilStackCreateComplete(DescribeStacksRequest.builder().stackName(stackName).build()); + if (waiterResponse.matched().response().isPresent()) { + LOG.info("Stack " + waiterResponse.matched().response().get().stacks().get(0).stackName() + " successfully deployed"); + } else { + throw new RuntimeException("Failed to create stack"); + } + return functionName; + } + + /** + * Destroy the CloudFormation stack + */ + public void destroy() { + LOG.info("Deleting '" + stackName + "' on account " + account); + cfn.deleteStack(DeleteStackRequest.builder().stackName(stackName).build()); + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + public long timeoutInSeconds = 30; + public String pathToFunction; + public String testName; + private String stackName; + private boolean tracing = false; + private JavaRuntime runtime; + private Map environmentVariables = new HashMap<>(); + private String idemPotencyTable; + + private Builder() { + getJavaRuntime(); + } + + /** + * Retrieve the java runtime to use for the lambda function. + */ + private void getJavaRuntime() { + String javaVersion = System.getenv("JAVA_VERSION"); // must be set in GitHub actions + if (javaVersion == null) { + throw new IllegalArgumentException("JAVA_VERSION is not set"); + } + if (javaVersion.startsWith("8")) { + runtime = JavaRuntime.JAVA8AL2; + } else if (javaVersion.startsWith("11")) { + runtime = JavaRuntime.JAVA11; + } else { + throw new IllegalArgumentException("Unsupported Java version " + javaVersion); + } + LOG.debug("Java Version set to {}, using runtime {}", javaVersion, runtime.getRuntime()); + } + + public Infrastructure build() { + Objects.requireNonNull(testName, "testName must not be null"); + + String uuid = UUID.randomUUID().toString().replace("-", "").substring(0, 12); + stackName = testName + "-" + uuid; + + Objects.requireNonNull(pathToFunction, "pathToFunction must not be null"); + return new Infrastructure(this); + } + + public Builder testName(String testName) { + this.testName = testName; + return this; + } + + public Builder pathToFunction(String pathToFunction) { + this.pathToFunction = pathToFunction; + return this; + } + + public Builder tracing(boolean tracing) { + this.tracing = tracing; + return this; + } + + public Builder idempotencyTable(String tableName) { + this.idemPotencyTable = tableName; + return this; + } + + public Builder environmentVariables(Map environmentVariables) { + this.environmentVariables = environmentVariables; + return this; + } + + public Builder timeoutInSeconds(long timeoutInSeconds) { + this.timeoutInSeconds = timeoutInSeconds; + return this; + } + } + + /** + * Build the CDK Stack containing the required resources (Lambda function, LogGroup, DDB Table) + * @return the CDK stack + */ + private Stack createStackWithLambda() { + Stack stack = new Stack(app, stackName); + List packagingInstruction = Arrays.asList( + "/bin/sh", + "-c", + "cd " + pathToFunction + + " && timeout -s SIGKILL 5m mvn clean install -ff " + + " -Dmaven.test.skip=true " + + " -Dmaven.resources.skip=true " + + " -Dmaven.compiler.source=" + runtime.getMvnProperty() + + " -Dmaven.compiler.target=" + runtime.getMvnProperty() + + " && cp /asset-input/" + pathToFunction + "/target/function.jar /asset-output/" + ); + + BundlingOptions.Builder builderOptions = BundlingOptions.builder() + .command(packagingInstruction) + .image(runtime.getCdkRuntime().getBundlingImage()) + .volumes(singletonList( + // Mount local .m2 repo to avoid download all the dependencies again inside the container + DockerVolume.builder() + .hostPath(System.getProperty("user.home") + "/.m2/") + .containerPath("/root/.m2/") + .build() + )) + .user("root") + .outputType(BundlingOutput.ARCHIVED); + + functionName = stackName + "-function"; + + LOG.debug("Building Lambda function with command "+ packagingInstruction.stream().collect(Collectors.joining(" ", "[", "]"))); + Function function = Function.Builder + .create(stack, functionName) + .code(Code.fromAsset("handlers/", AssetOptions.builder() + .bundling(builderOptions + .command(packagingInstruction) + .build()) + .build())) + .functionName(functionName) + .handler("software.amazon.lambda.powertools.e2e.Function::handleRequest") + .memorySize(1024) + .timeout(Duration.seconds(timeout)) + .runtime(runtime.getCdkRuntime()) + .environment(envVar) + .tracing(tracing ? Tracing.ACTIVE : Tracing.DISABLED) + .build(); + + LogGroup.Builder + .create(stack, functionName + "-logs") + .logGroupName("/aws/lambda/" + functionName) + .retention(RetentionDays.ONE_DAY) + .removalPolicy(RemovalPolicy.DESTROY) + .build(); + + if (!StringUtils.isEmpty(idempotencyTable)) { + Table table = Table.Builder + .create(stack, "IdempotencyTable") + .billingMode(BillingMode.PAY_PER_REQUEST) + .removalPolicy(RemovalPolicy.DESTROY) + .partitionKey(Attribute.builder().name("id").type(AttributeType.STRING).build()) + .tableName(idempotencyTable) + .timeToLiveAttribute("expiration") + .build(); + table.grantReadWriteData(function); + } + + return stack; + } + + /** + * cdk synth to retrieve the CloudFormation template and assets directory + */ + private void synthesize() { + CloudAssembly synth = app.synth(); + cfnTemplate = synth.getStackByName(stack.getStackName()).getTemplate(); + cfnAssetDirectory = synth.getDirectory(); + } + + /** + * Upload assets (mainly lambda function jars) to S3 + */ + private void uploadAssets() { + Map assets = findAssets(); + assets.forEach((objectKey, asset) -> { + if (!asset.assetPath.endsWith(".jar")) { + return; + } + ListObjectsV2Response objects = s3.listObjectsV2(ListObjectsV2Request.builder().bucket(asset.bucketName).build()); + if (objects.contents().stream().anyMatch(o -> o.key().equals(objectKey))) { + LOG.debug("Asset already exists, skipping"); + return; + } + LOG.info("Uploading asset " + objectKey + " to " + asset.bucketName); + s3.putObject(PutObjectRequest.builder().bucket(asset.bucketName).key(objectKey).build(), Paths.get(cfnAssetDirectory, asset.assetPath)); + }); + } + + /** + * Reading the cdk assets.json file to retrieve the list of assets to push to S3 + * @return a map of assets + */ + private Map findAssets() { + Map assets = new HashMap<>(); + try { + JsonNode jsonNode = JsonConfig.get().getObjectMapper().readTree(new File(cfnAssetDirectory, stackName + ".assets.json")); + JsonNode files = jsonNode.get("files"); + files.iterator().forEachRemaining(file -> { + String assetPath = file.get("source").get("path").asText(); + String assetPackaging = file.get("source").get("packaging").asText(); + String bucketName = file.get("destinations").get("current_account-current_region").get("bucketName").asText(); + String objectKey = file.get("destinations").get("current_account-current_region").get("objectKey").asText(); + Asset asset = new Asset(assetPath, assetPackaging, bucketName.replace("${AWS::AccountId}", account).replace("${AWS::Region}", region.toString())); + assets.put(objectKey, asset); + }); + } catch (IOException e) { + throw new RuntimeException(e); + } + return assets; + } + + private static class Asset { + private final String assetPath; + private final String assetPackaging; + private final String bucketName; + + Asset(String assetPath, String assetPackaging, String bucketName) { + this.assetPath = assetPath; + this.assetPackaging = assetPackaging; + this.bucketName = bucketName; + } + + @Override + public String toString() { + return "Asset{" + + "assetPath='" + assetPath + '\'' + + ", assetPackaging='" + assetPackaging + '\'' + + ", bucketName='" + bucketName + '\'' + + '}'; + } + } +} diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/JavaRuntime.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/JavaRuntime.java new file mode 100644 index 000000000..94ec13518 --- /dev/null +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/JavaRuntime.java @@ -0,0 +1,37 @@ +package software.amazon.lambda.powertools.testutils; + +import software.amazon.awscdk.services.lambda.Runtime; + +public enum JavaRuntime { + JAVA8("java8", Runtime.JAVA_8, "1.8"), + JAVA8AL2("java8.al2", Runtime.JAVA_8_CORRETTO, "1.8"), + JAVA11("java11", Runtime.JAVA_11, "11"); + + private final String runtime; + private final Runtime cdkRuntime; + + private final String mvnProperty; + + JavaRuntime(String runtime, Runtime cdkRuntime, String mvnProperty) { + this.runtime = runtime; + this.cdkRuntime = cdkRuntime; + this.mvnProperty = mvnProperty; + } + + public Runtime getCdkRuntime() { + return cdkRuntime; + } + + public String getRuntime() { + return runtime; + } + + @Override + public String toString() { + return runtime; + } + + public String getMvnProperty() { + return mvnProperty; + } +} diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/lambda/InvocationResult.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/lambda/InvocationResult.java new file mode 100644 index 000000000..168fec71b --- /dev/null +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/lambda/InvocationResult.java @@ -0,0 +1,43 @@ +package software.amazon.lambda.powertools.testutils.lambda; + +import software.amazon.awssdk.services.lambda.model.InvokeResponse; +import software.amazon.lambda.powertools.testutils.logging.InvocationLogs; + +import java.time.Instant; + +public class InvocationResult { + + private final InvocationLogs logs; + private final String result; + + private final String requestId; + private final Instant start; + private final Instant end; + + public InvocationResult(InvokeResponse response, Instant start, Instant end) { + requestId = response.responseMetadata().requestId(); + logs = new InvocationLogs(response.logResult(), requestId); + result = response.payload().asUtf8String(); + this.start = start; + this.end = end; + } + public InvocationLogs getLogs() { + return logs; + } + + public String getResult() { + return result; + } + + public String getRequestId() { + return requestId; + } + + public Instant getStart() { + return start; + } + + public Instant getEnd() { + return end; + } +} diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/lambda/LambdaInvoker.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/lambda/LambdaInvoker.java new file mode 100644 index 000000000..ecde1042e --- /dev/null +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/lambda/LambdaInvoker.java @@ -0,0 +1,39 @@ +package software.amazon.lambda.powertools.testutils.lambda; + +import software.amazon.awssdk.core.SdkBytes; +import software.amazon.awssdk.http.SdkHttpClient; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.lambda.LambdaClient; +import software.amazon.awssdk.services.lambda.model.InvokeRequest; +import software.amazon.awssdk.services.lambda.model.InvokeResponse; +import software.amazon.awssdk.services.lambda.model.LogType; + +import java.time.Clock; +import java.time.Instant; + +import static java.time.temporal.ChronoUnit.MINUTES; + +public class LambdaInvoker { + private static final SdkHttpClient httpClient = UrlConnectionHttpClient.builder().build(); + private static final Region region = Region.of(System.getProperty("AWS_DEFAULT_REGION", "eu-west-1")); + private static final LambdaClient lambda = LambdaClient.builder() + .httpClient(httpClient) + .region(region) + .build(); + + public static InvocationResult invokeFunction(String functionName, String input) { + SdkBytes payload = SdkBytes.fromUtf8String(input); + + InvokeRequest request = InvokeRequest.builder() + .functionName(functionName) + .payload(payload) + .logType(LogType.TAIL) + .build(); + + Instant start = Instant.now(Clock.systemUTC()).truncatedTo(MINUTES); + InvokeResponse response = lambda.invoke(request); + Instant end = start.plus(1, MINUTES); + return new InvocationResult(response, start, end); + } +} diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/logging/InvocationLogs.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/logging/InvocationLogs.java new file mode 100644 index 000000000..1ae1cfad7 --- /dev/null +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/logging/InvocationLogs.java @@ -0,0 +1,71 @@ +package software.amazon.lambda.powertools.testutils.logging; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Base64; +import java.util.stream.IntStream; + +/** + * Logs for a specific Lambda invocation + */ +public class InvocationLogs { + private final String[] logs; + private final String[] functionLogs; + + public InvocationLogs(String base64Logs, String requestId) { + String rawLogs = new String(Base64.getDecoder().decode(base64Logs), StandardCharsets.UTF_8); + this.logs = rawLogs.split("\n"); + + String start = String.format("START RequestId: %s", requestId); + String end = String.format("END RequestId: %s", requestId); + int startPos = IntStream.range(0, logs.length) + .filter(i -> logs[i].startsWith(start)) + .findFirst() + .orElse(-1); + int endPos = IntStream.range(0, logs.length) + .filter(i -> logs[i].equals(end)) + .findFirst() + .orElse(-1); + this.functionLogs = Arrays.copyOfRange(this.logs, startPos + 1, endPos); + } + + public String[] getAllLogs() { + return logs; + } + + /** + * Return only logs from function, exclude START, END, and REPORT and other elements generated by Lambda service + * @return only logs generated by the function + */ + public String[] getFunctionLogs() { + return this.functionLogs; + } + + public String[] getFunctionLogs(Level level) { + String[] filtered = getFunctionLogs(); + + return Arrays.stream(filtered).filter(log -> log.contains("\"level\":\""+level.getLevel()+"\"")).toArray(String[]::new); + } + + public enum Level { + DEBUG("DEBUG"), + INFO("INFO"), + WARN("WARN"), + ERROR("ERROR"); + + private final String level; + + Level(String lvl) { + this.level = lvl; + } + + public String getLevel() { + return level; + } + + @Override + public String toString() { + return level; + } + } +} diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/metrics/MetricsFetcher.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/metrics/MetricsFetcher.java new file mode 100644 index 000000000..eb3cd63a4 --- /dev/null +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/metrics/MetricsFetcher.java @@ -0,0 +1,95 @@ +package software.amazon.lambda.powertools.testutils.metrics; + +import com.evanlennick.retry4j.CallExecutor; +import com.evanlennick.retry4j.CallExecutorBuilder; +import com.evanlennick.retry4j.Status; +import com.evanlennick.retry4j.config.RetryConfig; +import com.evanlennick.retry4j.config.RetryConfigBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.http.SdkHttpClient; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.cloudwatch.CloudWatchClient; +import software.amazon.awssdk.services.cloudwatch.model.*; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; + +import static java.time.Duration.ofSeconds; + +/** + * Class in charge of retrieving the actual metrics of a Lambda execution on CloudWatch + */ +public class MetricsFetcher { + private static final Logger LOG = LoggerFactory.getLogger(MetricsFetcher.class); + + private static final SdkHttpClient httpClient = UrlConnectionHttpClient.builder().build(); + private static final Region region = Region.of(System.getProperty("AWS_DEFAULT_REGION", "eu-west-1")); + private static final CloudWatchClient cloudwatch = CloudWatchClient.builder() + .httpClient(httpClient) + .region(region) + .build(); + + /** + * Retrieve the metric values from start to end. Different parameters are required (see {@link CloudWatchClient#getMetricData} for more info). + * Use a retry mechanism as metrics may not be available instantaneously after a function runs. + * @param start + * @param end + * @param period + * @param namespace + * @param metricName + * @param dimensions + * @return + */ + public List fetchMetrics(Instant start, Instant end, int period, String namespace, String metricName, Map dimensions) { + List dimensionsList = new ArrayList<>(); + if (dimensions != null) + dimensions.forEach((key, value) -> dimensionsList.add(Dimension.builder().name(key).value(value).build())); + + Callable> callable = () -> { + LOG.debug("Get Metrics for namespace {}, start {}, end {}, metric {}, dimensions {}", namespace, start, end, metricName, dimensionsList); + GetMetricDataResponse metricData = cloudwatch.getMetricData(GetMetricDataRequest.builder() + .startTime(start) + .endTime(end) + .metricDataQueries(MetricDataQuery.builder() + .id(metricName.toLowerCase()) + .metricStat(MetricStat.builder() + .unit(StandardUnit.COUNT) + .metric(Metric.builder() + .namespace(namespace) + .metricName(metricName) + .dimensions(dimensionsList) + .build()) + .period(period) + .stat("Sum") + .build()) + .returnData(true) + .build()) + .build()); + List values = metricData.metricDataResults().get(0).values(); + if (values == null || values.isEmpty()) { + throw new Exception("No data found for metric " + metricName); + } + return values; + }; + + RetryConfig retryConfig = new RetryConfigBuilder() + .withMaxNumberOfTries(10) + .retryOnAnyException() + .withDelayBetweenTries(ofSeconds(2)) + .withRandomExponentialBackoff() + .build(); + CallExecutor> callExecutor = new CallExecutorBuilder>() + .config(retryConfig) + .afterFailedTryListener(s -> { + LOG.warn(s.getLastExceptionThatCausedRetry().getMessage() + ", attempts: " + s.getTotalTries()); + }) + .build(); + Status> status = callExecutor.execute(callable); + return status.getResult(); + } +} diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/tracing/SegmentDocument.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/tracing/SegmentDocument.java new file mode 100644 index 000000000..08f4bf7d8 --- /dev/null +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/tracing/SegmentDocument.java @@ -0,0 +1,189 @@ +package software.amazon.lambda.powertools.testutils.tracing; + +import com.fasterxml.jackson.annotation.JsonSetter; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class SegmentDocument { + private String id; + + @JsonSetter("trace_id") + private String traceId; + + private String name; + + @JsonSetter("start_time") + private long startTime; + + @JsonSetter("end_time") + private long endTime; + + private String origin; + + private List subsegments = new ArrayList<>(); + + public SegmentDocument() { + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getTraceId() { + return traceId; + } + + public void setTraceId(String traceId) { + this.traceId = traceId; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public long getStartTime() { + return startTime; + } + + public void setStartTime(long startTime) { + this.startTime = startTime; + } + + public long getEndTime() { + return endTime; + } + + public void setEndTime(long endTime) { + this.endTime = endTime; + } + + public String getOrigin() { + return origin; + } + + public void setOrigin(String origin) { + this.origin = origin; + } + + public List getSubsegments() { + return subsegments; + } + + public void setSubsegments(List subsegments) { + this.subsegments = subsegments; + } + + public Duration getDuration() { + return Duration.ofMillis(endTime - startTime); + } + + public boolean hasSubsegments() { + return !subsegments.isEmpty(); + } + + public static class SubSegment{ + private String id; + + private String name; + + @JsonSetter("start_time") + private long startTime; + + @JsonSetter("end_time") + private long endTime; + + private List subsegments = new ArrayList<>(); + + private Map annotations; + + private Map metadata; + + private String namespace; + + public SubSegment() { + } + + public boolean hasSubsegments() { + return !subsegments.isEmpty(); + } + + public Duration getDuration() { + return Duration.ofMillis(endTime - startTime); + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public long getStartTime() { + return startTime; + } + + public void setStartTime(long startTime) { + this.startTime = startTime; + } + + public long getEndTime() { + return endTime; + } + + public void setEndTime(long endTime) { + this.endTime = endTime; + } + + public List getSubsegments() { + return subsegments; + } + + public void setSubsegments(List subsegments) { + this.subsegments = subsegments; + } + + public Map getAnnotations() { + return annotations; + } + + public void setAnnotations(Map annotations) { + this.annotations = annotations; + } + + public Map getMetadata() { + return metadata; + } + + public void setMetadata(Map metadata) { + this.metadata = metadata; + } + + public String getNamespace() { + return namespace; + } + + public void setNamespace(String namespace) { + this.namespace = namespace; + } + } +} diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/tracing/Trace.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/tracing/Trace.java new file mode 100644 index 000000000..15026a9d1 --- /dev/null +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/tracing/Trace.java @@ -0,0 +1,21 @@ +package software.amazon.lambda.powertools.testutils.tracing; + +import software.amazon.lambda.powertools.testutils.tracing.SegmentDocument.SubSegment; + +import java.util.ArrayList; +import java.util.List; + +public class Trace { + private final List subsegments = new ArrayList<>(); + + public Trace() { + } + + public List getSubsegments() { + return subsegments; + } + + public void addSubSegment(SubSegment subSegment) { + subsegments.add(subSegment); + } +} diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/tracing/TraceFetcher.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/tracing/TraceFetcher.java new file mode 100644 index 000000000..e7cd13640 --- /dev/null +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/tracing/TraceFetcher.java @@ -0,0 +1,210 @@ +package software.amazon.lambda.powertools.testutils.tracing; + +import com.evanlennick.retry4j.CallExecutor; +import com.evanlennick.retry4j.CallExecutorBuilder; +import com.evanlennick.retry4j.Status; +import com.evanlennick.retry4j.config.RetryConfig; +import com.evanlennick.retry4j.config.RetryConfigBuilder; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.http.SdkHttpClient; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.xray.XRayClient; +import software.amazon.awssdk.services.xray.model.*; +import software.amazon.lambda.powertools.testutils.tracing.SegmentDocument.SubSegment; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.stream.Collectors; + +import static java.time.Duration.ofSeconds; + +/** + * Class in charge of retrieving the actual traces of a Lambda execution on X-Ray + */ +public class TraceFetcher { + + private static final ObjectMapper MAPPER = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + private static final Logger LOG = LoggerFactory.getLogger(TraceFetcher.class); + + private final Instant start; + private final Instant end; + private final String filterExpression; + private final List excludedSegments; + + /** + * @param start beginning of the time slot to search in + * @param end end of the time slot to search in + * @param filterExpression eventual filter for the search + * @param excludedSegments list of segment to exclude from the search + */ + public TraceFetcher(Instant start, Instant end, String filterExpression, List excludedSegments) { + this.start = start; + this.end = end; + this.filterExpression = filterExpression; + this.excludedSegments = excludedSegments; + } + + public static Builder builder() { + return new Builder(); + } + + /** + * Retrieve the traces corresponding to a specific function during a specific time slot. + * Use a retry mechanism as traces may not be available instantaneously after a function runs. + * + * @return traces + */ + public Trace fetchTrace() { + Callable callable = () -> { + List traceIds = getTraceIds(); + return getTrace(traceIds); + }; + + RetryConfig retryConfig = new RetryConfigBuilder() + .withMaxNumberOfTries(10) + .retryOnAnyException() + .withDelayBetweenTries(ofSeconds(5)) + .withRandomExponentialBackoff() + .build(); + CallExecutor callExecutor = new CallExecutorBuilder() + .config(retryConfig) + .afterFailedTryListener(s -> {LOG.warn(s.getLastExceptionThatCausedRetry().getMessage() + ", attempts: " + s.getTotalTries());}) + .build(); + Status status = callExecutor.execute(callable); + return status.getResult(); + } + + /** + * Retrieve traces from trace ids. + * @param traceIds + * @return + */ + private Trace getTrace(List traceIds) { + BatchGetTracesResponse tracesResponse = xray.batchGetTraces(BatchGetTracesRequest.builder() + .traceIds(traceIds) + .build()); + if (!tracesResponse.hasTraces()) { + throw new RuntimeException("No trace found"); + } + Trace traceRes = new Trace(); + tracesResponse.traces().forEach(trace -> { + if (trace.hasSegments()) { + trace.segments().forEach(segment -> { + try { + SegmentDocument document = MAPPER.readValue(segment.document(), SegmentDocument.class); + if (document.getOrigin().equals("AWS::Lambda::Function")) { + if (document.hasSubsegments()) { + getNestedSubSegments(document.getSubsegments(), traceRes, Collections.emptyList()); + } + } + } catch (JsonProcessingException e) { + LOG.error("Failed to parse segment document: " + e.getMessage()); + throw new RuntimeException(e); + } + }); + } + }); + return traceRes; + } + + private void getNestedSubSegments(List subsegments, Trace traceRes, List idsToIgnore) { + subsegments.forEach(subsegment -> { + List subSegmentIdsToIgnore = Collections.emptyList(); + if (!excludedSegments.contains(subsegment.getName()) && !idsToIgnore.contains(subsegment.getId())) { + traceRes.addSubSegment(subsegment); + if (subsegment.hasSubsegments()) { + subSegmentIdsToIgnore = subsegment.getSubsegments().stream().map(SubSegment::getId).collect(Collectors.toList()); + } + } + if (subsegment.hasSubsegments()) { + getNestedSubSegments(subsegment.getSubsegments(), traceRes, subSegmentIdsToIgnore); + } + }); + } + + /** + * Use the X-Ray SDK to retrieve the trace ids corresponding to a specific function during a specific time slot + * @return a list of trace ids + */ + private List getTraceIds() { + GetTraceSummariesResponse traceSummaries = xray.getTraceSummaries(GetTraceSummariesRequest.builder() + .startTime(start) + .endTime(end) + .timeRangeType(TimeRangeType.EVENT) + .sampling(false) + .filterExpression(filterExpression) + .build()); + if (!traceSummaries.hasTraceSummaries()) { + throw new RuntimeException("No trace id found"); + } + List traceIds = traceSummaries.traceSummaries().stream().map(TraceSummary::id).collect(Collectors.toList()); + if (traceIds.isEmpty()) { + throw new RuntimeException("No trace id found"); + } + return traceIds; + } + + private static final SdkHttpClient httpClient = UrlConnectionHttpClient.builder().build(); + private static final Region region = Region.of(System.getProperty("AWS_DEFAULT_REGION", "eu-west-1")); + private static final XRayClient xray = XRayClient.builder() + .httpClient(httpClient) + .region(region) + .build(); + + public static class Builder { + private Instant start; + private Instant end; + private String filterExpression; + private List excludedSegments = Arrays.asList("Initialization", "Invocation", "Overhead"); + + public TraceFetcher build() { + if (filterExpression == null) + throw new IllegalArgumentException("filterExpression or functionName is required"); + if (start == null) + throw new IllegalArgumentException("start is required"); + if (end == null) + end = start.plus(1, ChronoUnit.MINUTES); + LOG.debug("Looking for traces from {} to {} with filter {}", start, end, filterExpression); + return new TraceFetcher(start, end, filterExpression, excludedSegments); + } + + public Builder start(Instant start) { + this.start = start; + return this; + } + + public Builder end(Instant end) { + this.end = end; + return this; + } + + public Builder filterExpression(String filterExpression) { + this.filterExpression = filterExpression; + return this; + } + + /** + * "Initialization", "Invocation", "Overhead" are excluded by default + * @param excludedSegments + * @return + */ + public Builder excludeSegments(List excludedSegments) { + this.excludedSegments = excludedSegments; + return this; + } + + public Builder functionName(String functionName) { + this.filterExpression = String.format("service(id(name: \"%s\", type: \"AWS::Lambda::Function\"))", functionName); + return this; + } + } +} diff --git a/powertools-e2e-tests/src/test/resources/logback-test.xml b/powertools-e2e-tests/src/test/resources/logback-test.xml new file mode 100644 index 000000000..aac638007 --- /dev/null +++ b/powertools-e2e-tests/src/test/resources/logback-test.xml @@ -0,0 +1,20 @@ + + + + + + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + \ No newline at end of file