Update elasticsearch host to support elastic cloud since we are using enrich feature, which is unsupported by AWS ES

Author: callmekatootie

README.md

@@ -35,7 +35,7 @@ Configuration for the application is at config/default.js and config/production.
 - UBAHN_UPDATE_TOPIC: Kafka topic for update message
 - UBAHN_DELETE_TOPIC: Kafka topic for delete message
 - UBAHN_AGGREGATE_TOPIC: Kafka topic that is used to combine all create, update and delete message(s)
-- ES.HOST: Elasticsearch host
+- ES_HOST: Elasticsearch host
 - ES.DOCUMENTS: Elasticsearch index, type and id mapping for resources.
 - ATTRIBUTE_GROUP_PIPELINE_ID: The pipeline id for enrichment with attribute group. Default is `attributegroup-pipeline`
 - SKILL_PROVIDER_PIPELINE_ID: The pipeline id for enrichment with skill provider. Default is `skillprovider-pipeline`
@@ -46,6 +46,9 @@ Configuration for the application is at config/default.js and config/production.
 - ACHIEVEMENT_PROVIDER_ENRICH_POLICYNAME: The enrich policy for achievement provider. Default is `achievementprovider-policy`
 - SKILL_ENRICH_POLICYNAME: The enrich policy for skill. Default is `skill-policy`
 - ATTRIBUTE_ENRICH_POLICYNAME: The enrich policy for skill. Default is `attribute-policy`
+- ELASTICCLOUD_ID: The elastic cloud id, if your elasticsearch instance is hosted on elastic cloud. DO NOT provide a value for ES_HOST if you are using this
+- ELASTICCLOUD_USERNAME: The elastic cloud username for basic authentication. Provide this only if your elasticsearch instance is hosted on elastic cloud
+- ELASTICCLOUD_PASSWORD: The elastic cloud password for basic authentication. Provide this only if your elasticsearch instance is hosted on elastic cloud
 
 For `ES.DOCUMENTS` configuration, you will find multiple other configurations below it. Each has default values that you can override using the environment variables
 

config/default.js

@@ -51,6 +51,13 @@ module.exports = {
   // ElasticSearch
   ES: {
     HOST: process.env.ES_HOST || 'http://localhost:9200',
+
+    ELASTICCLOUD: {
+      id: process.env.ELASTICCLOUD_ID,
+      username: process.env.ELASTICCLOUD_USERNAME,
+      password: process.env.ELASTICCLOUD_PASSWORD
+    },
+
     // es mapping: _index, _type, _id
     DOCUMENTS: {
       achievementprovider: {

package-lock.json

@@ -19,7 +19,6 @@
     "@hapi/joi": "^16.1.8",
     "@hapi/joi-date": "^2.0.1",
     "amazon-qldb-driver-nodejs": "^0.1.1-preview.2",
-    "aws-elasticsearch-connector": "^9.0.0",
     "aws-sdk": "^2.627.0",
     "axios": "^0.19.2",
     "body-parser": "^1.19.0",

package.json

@@ -1,7 +1,6 @@
 const config = require('config')
 const AWS = require('aws-sdk')
 const elasticsearch = require('@elastic/elasticsearch')
-const createAwsElasticsearchConnector = require('aws-elasticsearch-connector')
 
 AWS.config.region = config.AWS_REGION
 
@@ -17,12 +16,18 @@ function getESClient () {
     return esClient
   }
   const host = config.ES.HOST
+  const cloudId = config.ES.ELASTICCLOUD.id
   if (!esClient) {
-    // AWS ES configuration is different from other providers
-    if (/.*amazonaws.*/.test(host)) {
+    if (cloudId) {
+      // Elastic Cloud configuration
       esClient = new elasticsearch.Client({
-        ...createAwsElasticsearchConnector(AWS.config),
-        node: host
+        cloud: {
+          id: cloudId
+        },
+        auth: {
+          username: config.ES.ELASTICCLOUD.username,
+          password: config.ES.ELASTICCLOUD.password
+        }
       })
     } else {
       esClient = new elasticsearch.Client({