Merge pull request #2 from sharathkumaranbu/tests

Tests and Dockerization

Author: cwdcwd

.dockerignore

@@ -0,0 +1,4 @@
+node_modules/
+.env
+coverage/
+.nyc_output/

.gitignore

@@ -3,3 +3,5 @@ node_modules
 *.log
 .DS_Store
 .env
+coverage
+.nyc_output

README.md

@@ -7,12 +7,6 @@
 - ElasticSearch
 - Docker, Docker Compose
 
-## Notes
-
-In Elasticsearch v6+, it can support one type per index, see
-https://www.elastic.co/guide/en/elasticsearch/reference/current/removal-of-types.html
-so for all 4 types, same configured index type is used, types are distinguished by the 'resource' attribute of indexed data.
-
 ## Configuration
 
 Configuration for the notification server is at `config/default.js`.
@@ -29,11 +23,8 @@ The following parameters can be set in config files or in env variables:
 - CREATE_DATA_TOPIC: create data Kafka topic, default value is 'submission.notification.create'
 - UPDATE_DATA_TOPIC: update data Kafka topic, default value is 'submission.notification.update'
 - DELETE_DATA_TOPIC: delete data Kafka topic, default value is 'submission.notification.delete'
-- ELASTICSEARCH_CONFIG: the config to create Elasticsearch client, see
-    https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/configuration.html
-    for full details, SSL connection can be configured in ssl config field
-- ELASTICSEARCH_INDEX: the Elasticsearch index to store Kafka messages data, default value is 'submission-index'
-- ELASTICSEARCH_INDEX_TYPE: the Elasticsearch index type name, default value is 'submission'
+
+Refer to `esConfig` variable in `config/default.js` for ES related configuration.
 
 Also note that there is a `/health` endpoint that checks for the health of the app. This sets up an expressjs server and listens on the environment variable `PORT`. It's not part of the configuration file and needs to be passed as an environment variable
 
@@ -64,10 +55,6 @@ Also note that there is a `/health` endpoint that checks for the health of the a
   `bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic submission.notification.create --from-beginning`
 - writing/reading messages to/from other topics are similar
 
-## ElasticSearch setup
-
-- go to docker folder, run `docker-compose up`
-
 ## Local deployment
 
 - install dependencies `npm i`
@@ -77,6 +64,63 @@ Also note that there is a `/health` endpoint that checks for the health of the a
 - run tests `npm run test`
 - start processor app `npm start`
 
+## Local Deployment with Docker
+
+To run the Submission ES Processor using docker, follow the below steps
+
+1. Navigate to the directory `docker`
+
+2. Rename the file `sample.api.env` to `api.env`
+
+3. Set the required AWS credentials in the file `api.env`
+
+4. Once that is done, run the following command
+
+```
+docker-compose up
+```
+
+5. When you are running the application for the first time, It will take some time initially to download the image and install the dependencies
+
+## Unit tests and Integration tests
+
+Integration tests use different index `submission-test` which is not same as the usual index `submission`.
+
+Please ensure to create the index `submission-test` or the index specified in the environment variable `ES_INDEX_TEST` before running the Integration tests. You could re-use the existing scripts to create index but you would need to set the below environment variable
+
+```
+export ES_INDEX=submission-test
+```
+
+#### Running unit tests and coverage
+
+To run unit tests alone
+
+```
+npm run test
+```
+
+To run unit tests with coverage report
+
+```
+npm run cov
+```
+
+#### Running integration tests and coverage
+
+To run integration tests alone
+
+```
+npm run e2e
+```
+
+To run integration tests with coverage report
+
+```
+npm run cov-e2e
+```
+
+
 ## Verification
 
 - start kafka server, start elasticsearch, initialize Elasticsearch, start processor app

config/default.js

@@ -1,10 +1,9 @@
 /**
- * The configuration file.
+ * The default configuration file.
  */
-const AWS = require('aws-sdk')
-const httpAWSes = require('http-aws-es')
 
 module.exports = {
+  DISABLE_LOGGING: process.env.DISABLE_LOGGING || false, // If true, logging will be disabled
   LOG_LEVEL: process.env.LOG_LEVEL || 'debug',
 
   KAFKA_URL: process.env.KAFKA_URL || 'localhost:9092',
@@ -17,18 +16,11 @@ module.exports = {
   UPDATE_DATA_TOPIC: process.env.UPDATE_DATA_TOPIC || 'submission.notification.update',
   DELETE_DATA_TOPIC: process.env.DELETE_DATA_TOPIC || 'submission.notification.delete',
 
-  // see https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/configuration.html
-  // for full config details, for SSL connection, see the ssl field details in above page
-  ELASTICSEARCH_CONFIG: {
-    apiVersion: process.env.ES_API_VERSION || '6.3',
-    host: process.env.ELASTICSEARCH_HOST || 'localhost:9200',
-    connectionClass: httpAWSes,
-    amazonES: {
-      region: process.env.AWS_REGION || 'us-east-1',
-      credentials: new AWS.EnvironmentCredentials('AWS')
-    }
-  },
-
-  ELASTICSEARCH_INDEX: process.env.ELASTICSEARCH_INDEX || 'submission-index',
-  ELASTICSEARCH_INDEX_TYPE: process.env.ELASTICSEARCH_INDEX_TYPE || 'submission'
+  esConfig: {
+    HOST: process.env.ES_HOST,
+    AWS_REGION: process.env.AWS_REGION || 'us-east-1', // AWS Region to be used if we use AWS ES
+    API_VERSION: process.env.ES_API_VERSION || '6.3',
+    ES_INDEX: process.env.ES_INDEX || 'submission-test',
+    ES_TYPE: process.env.ES_TYPE || '_doc' // ES 6.x accepts only 1 Type per index and it's mandatory to define it
+  }
 }

config/production.js

@@ -0,0 +1,7 @@
+/**
+ * Production configuration file
+ */
+
+module.exports = {
+  LOG_LEVEL: process.env.LOG_LEVEL || 'info'
+}

config/test.js

@@ -0,0 +1,13 @@
+/**
+ * Configuration file to be used while running tests
+ */
+
+module.exports = {
+  DISABLE_LOGGING: false, // If true, logging will be disabled
+  LOG_LEVEL: 'debug',
+  esConfig: {
+    ES_HOST: process.env.ES_HOST || 'https://test.es.com',
+    ES_INDEX: process.env.ES_INDEX_TEST || 'submission-test',
+    ES_TYPE: process.env.ES_TYPE_TEST || '_doc' // ES 6.x accepts only 1 Type per index and it's mandatory to define it
+  }
+}

docker/Dockerfile

@@ -0,0 +1,11 @@
+# Use the base image with Node.js 8.11.3
+FROM node:8.11.3
+
+# Copy the current directory into the Docker image
+COPY . /submission-processor-es
+
+# Set working directory for future use
+WORKDIR /submission-processor-es
+
+# Install the dependencies from package.json
+RUN npm install

docker/docker-compose.yml

@@ -1,6 +1,10 @@
-version: "2"
+version: '3'
 services:
-  esearch:
-    image: "docker.elastic.co/elasticsearch/elasticsearch:6.3.1"
-    ports:
-      - "9200:9200"
+  submission-processor-es:
+    build:
+      context: ../
+      dockerfile: docker/Dockerfile
+    env_file:
+      - api.env
+    command: npm start
+    network_mode: "host"

docker/sample.api.env

@@ -0,0 +1,3 @@
+AWS_ACCESS_KEY_ID=<AWS Access Key ID>
+AWS_SECRET_ACCESS_KEY=<AWS Secret Access Key>
+ES_HOST=<ES Host Endpoint>