Added Fargate usage with Firelens example

Author: Simran Bajaj

examples/README.md

@@ -29,7 +29,7 @@ Run the example:
 ./examples/agent/bin/run.sh
 ```
 
-## FireLens on ECS
+## FireLens on ECS EC2
 
 You can deploy the example by running the following:
 
@@ -45,7 +45,7 @@ aws s3api create-bucket --bucket <bucket-name> --region <region>
 # create ECS service
 
 # deploy
-./examples/ecs-firelens/publish.sh \
+./examples/ecs-firelens/bin/publish.sh \
   <account-id> \
   <region> \
   <image-name> \
@@ -54,3 +54,48 @@ aws s3api create-bucket --bucket <bucket-name> --region <region>
   <ecs-task-family> \
   <ecs-service-name>
 ```  
+
+## FireLens on ECS Fargate
+
+For running on Fargate, s3 file option is not supported for Fluent-bit config. Hence, we need to build the fluent-bit custom config image and then use its reference in our Firelens container definition.
+
+For building the custom fluent-bit image, clone the [amazon-ecs-firelens-examples](https://github.com/aws-samples/amazon-ecs-firelens-examples) and modify the contents of [extra.conf](https://github.com/aws-samples/amazon-ecs-firelens-examples/blob/mainline/examples/fluent-bit/config-file-type-file/extra.conf) in the amazon-ecs-firelens-examples repository. Post this run the following commands to build the custom fluent-bit image:-
+
+```sh 
+# create an ECR repository for the Fluentbit-config image
+aws ecr create-repository --repository-name <config-image-name> --region <region>
+
+# Navigate to the config file directory
+cd examples/fluent-bit/config-file-type-file
+
+# Build the docker image from Dockerfile. Replace config-image-name with your image name
+docker build -t <config-image-name> .
+
+# Tag the recently built docker image . Replace the config-image-name, account-id and region with your values.
+docker tag <config-image-name>:latest <account-id>.dkr.ecr.<region>.amazonaws.com/<config-image-name>:latest
+
+# Push the docker image to ECR
+docker push <account-id>.dkr.ecr.<region>.amazonaws.com/<config-image-name>:latest
+```
+
+For executing EMF application on Fargate, you need to execute the following commands :-
+
+```sh 
+# create an ECR repository for the example image
+aws ecr create-repository --repository-name <image-name> --region <region>
+
+# create ECS cluster
+# create ECS task definition
+# create ECS service
+
+# deploy
+./examples/ecs-firelens/bin/publish-fargate.sh 
+<account-id> \
+<region> \
+<example-image> \
+<fargate-config-image> \
+<ecs-cluster> \
+<ecs-task> \
+<ecs-service>
+
+```

examples/ecs-firelens/Dockerfile

@@ -1,8 +1,8 @@
-FROM openjdk:8-jdk-slim
+FROM openjdk:11
 RUN mkdir -p /app
 
 # copy the source files over
 COPY build/libs/*.jar /app/app.jar
 
 ENV JAVA_OPTS=""
-ENTRYPOINT [ "sh", "-c", "java $JAVA_OPTS -Djava.security.egd=file:/dev/./urandom -jar /app/app.jar" ]
+ENTRYPOINT [ "sh", "-c", "java $JAVA_OPTS -Djava.security.egd=file:/dev/./urandom -jar /app/app.jar" ]

examples/ecs-firelens/container-definitions-fargate.template.json

@@ -0,0 +1,47 @@
+[
+  {
+    "name": "example",
+    "image": "<account-id>.dkr.ecr.<region>.amazonaws.com/<image-name>:latest",
+    "environment": [{
+      "name": "AWS_EMF_AGENT_ENDPOINT",
+      "value": "tcp://127.0.0.1:25888"
+    }
+    ],
+    "essential": true,
+    "logConfiguration": {
+      "logDriver": "awsfirelens",
+      "options": {
+        "Name": "cloudwatch",
+        "region": "<region>",
+        "log_key": "log",
+        "log_group_name": "aws-emf-ecs-firelens-example-metrics",
+        "auto_create_group": "true",
+        "log_stream_prefix": "emf-",
+        "retry_limit": "2",
+        "log_format": "json/emf"
+      }
+    }
+  },
+  {
+    "name": "fluent-bit",
+    "image": "<account-id>.dkr.ecr.<region>.amazonaws.com/<Firelens-custom-conf-image>:latest",
+    "essential": true,
+    "firelensConfiguration": {
+      "type": "fluentbit",
+      "options": {
+        "config-file-type": "file",
+        "config-file-value": "/extra.conf",
+        "enable-ecs-log-metadata": "false"
+      }
+    },
+    "logConfiguration": {
+      "logDriver": "awslogs",
+      "options": {
+        "awslogs-group": "firelens-container",
+        "awslogs-region": "<region>",
+        "awslogs-create-group": "true",
+        "awslogs-stream-prefix": "firelens"
+      }
+    }
+  }
+]

examples/ecs-firelens/extra.conf

@@ -0,0 +1,26 @@
+# TCP input used for EMF payloads
+[INPUT]
+    Name        tcp
+    Listen      0.0.0.0
+    Port        25888
+    Chunk_Size  32
+    Buffer_Size 64
+    Format      none
+    Tag         emf-${HOSTNAME}
+    # This tag is used by the output plugin to determine the LogStream
+    # including the HOSTNAME is a way to increase the number of LogStreams.
+    # The maximum throughput on a
+    # single LogStream is 5 MB/s (max 1 MB at max 5 TPS).
+    # In AWSVPC mode, the HOSTNAME is the ENI private IP
+    # in bridge mode, the HOSTNAME is the Docker container ID
+
+# Output for EMF over TCP -> CloudWatch
+[OUTPUT]
+    Name                cloudwatch
+    Match               emf-*
+    region              us-east-1
+    log_key             log
+    log_group_name      aws-emf-ecs-firelens-example-metrics
+    log_stream_prefix   from-fluent-bit-
+    auto_create_group   true
+    log_format          json/emf

examples/ecs-firelens/src/main/java/App.java

@@ -24,7 +24,6 @@
 import software.amazon.cloudwatchlogs.emf.logger.MetricsLogger;
 import software.amazon.cloudwatchlogs.emf.model.Unit;
 import sun.misc.Signal;
-
 import java.io.IOException;
 import java.io.OutputStream;
 import java.net.InetSocketAddress;
@@ -36,9 +35,12 @@ public class App {
 
     public static void main(String[] args) throws Exception {
         registerShutdownHook();
-
-        int portNumber = 8000;
+        MetricsLogger logger = new MetricsLogger();
+        logger.setNamespace("FargateEMF");
+        logger.putMetric("Latency", 63, Unit.MILLISECONDS);
+        logger.flush();
         HttpServer server = HttpServer.create(new InetSocketAddress(8000), 0);
+        int portNumber = 8000;
         System.out.println("Server started. Listening on " + portNumber);
         server.createContext("/", new SimpleHandler());
         server.setExecutor(null);