1. Set the enrich policy name as a config

2. Fix issue where migrate data from db to es would not save boolean fields
3. Fix issue where updating skill or attribute would result in referenced fields to also be passed in bus api

Author: callmekatootie

README.md

@@ -40,6 +40,12 @@ Configuration for the application is at config/default.js and config/production.
 - ATTRIBUTE_GROUP_PIPELINE_ID: The pipeline id for enrichment with attribute group. Default is `attributegroup-pipeline`
 - SKILL_PROVIDER_PIPELINE_ID: The pipeline id for enrichment with skill provider. Default is `skillprovider-pipeline`
 - USER_PIPELINE_ID: The pipeline id for enrichment of user details. Default is `user-pipeline`
+- ATTRIBUTE_GROUP_ENRICH_POLICYNAME: The enrich policy for attribute group. Default is `attributegroup-policy`
+- SKILL_PROVIDER_ENRICH_POLICYNAME: The enrich policy for skill provider. Default is `skillprovider-policy`
+- ROLE_ENRICH_POLICYNAME: The enrich policy for role. Default is `role-policy`
+- ACHIEVEMENT_PROVIDER_ENRICH_POLICYNAME: The enrich policy for achievement provider. Default is `achievementprovider-policy`
+- SKILL_ENRICH_POLICYNAME: The enrich policy for skill. Default is `skill-policy`
+- ATTRIBUTE_ENRICH_POLICYNAME: The enrich policy for skill. Default is `attribute-policy`
 
 For `ES.DOCUMENTS` configuration, you will find multiple other configurations below it. Each has default values that you can override using the environment variables
 

config/default.js

@@ -55,33 +55,40 @@ module.exports = {
     DOCUMENTS: {
       achievementprovider: {
         index: process.env.ACHIEVEMENT_PROVIDER_INDEX || 'achievement_provider',
-        type: '_doc'
+        type: '_doc',
+        enrichPolicyName: process.env.ACHIEVEMENT_PROVIDER_ENRICH_POLICYNAME || 'achievementprovider-policy'
       },
       attribute: {
         index: process.env.ATTRIBUTE_INDEX || 'attribute',
-        type: '_doc'
+        type: '_doc',
+        enrichPolicyName: process.env.ATTRIBUTE_ENRICH_POLICYNAME || 'attribute-policy'
       },
       attributegroup: {
         index: process.env.ATTRIBUTE_GROUP_INDEX || 'attribute_group',
         type: '_doc',
-        pipelineId: process.env.ATTRIBUTE_GROUP_PIPELINE_ID || 'attributegroup-pipeline'
+        pipelineId: process.env.ATTRIBUTE_GROUP_PIPELINE_ID || 'attributegroup-pipeline',
+        enrichPolicyName: process.env.ATTRIBUTE_GROUP_ENRICH_POLICYNAME || 'attributegroup-policy'
       },
       organization: {
         index: process.env.ORGANIZATION_INDEX || 'organization',
-        type: '_doc'
+        type: '_doc',
+        enrichPolicyName: process.env.ORGANIZATION_ENRICH_POLICYNAME || 'organization-policy'
       },
       role: {
         index: process.env.ROLE_INDEX || 'role',
-        type: '_doc'
+        type: '_doc',
+        enrichPolicyName: process.env.ROLE_ENRICH_POLICYNAME || 'role-policy'
       },
       skill: {
         index: process.env.SKILL_INDEX || 'skill',
-        type: '_doc'
+        type: '_doc',
+        enrichPolicyName: process.env.SKILL_ENRICH_POLICYNAME || 'skill-policy'
       },
       skillprovider: {
         index: process.env.SKILL_PROVIDER_INDEX || 'skill_provider',
         type: '_doc',
-        pipelineId: process.env.SKILL_PROVIDER_PIPELINE_ID || 'skillprovider-pipeline'
+        pipelineId: process.env.SKILL_PROVIDER_PIPELINE_ID || 'skillprovider-pipeline',
+        enrichPolicyName: process.env.SKILL_PROVIDER_ENRICH_POLICYNAME || 'skillprovider-policy'
       },
       user: {
         index: process.env.USER_INDEX || 'user',

scripts/constants.js

@@ -10,7 +10,7 @@ const topResources = {
     index: config.get('ES.DOCUMENTS.skillprovider.index'),
     type: config.get('ES.DOCUMENTS.skillprovider.type'),
     enrich: {
-      policyName: 'skillprovider-policy',
+      policyName: config.get('ES.DOCUMENTS.skillprovider.enrichPolicyName'),
       matchField: 'id',
       enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy']
     },
@@ -26,7 +26,7 @@ const topResources = {
     index: config.get('ES.DOCUMENTS.role.index'),
     type: config.get('ES.DOCUMENTS.role.type'),
     enrich: {
-      policyName: 'role-policy',
+      policyName: config.get('ES.DOCUMENTS.role.enrichPolicyName'),
       matchField: 'id',
       enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy']
     }
@@ -36,7 +36,7 @@ const topResources = {
     index: config.get('ES.DOCUMENTS.achievementprovider.index'),
     type: config.get('ES.DOCUMENTS.achievementprovider.type'),
     enrich: {
-      policyName: 'achievementprovider-policy',
+      policyName: config.get('ES.DOCUMENTS.achievementprovider.enrichPolicyName'),
       matchField: 'id',
       enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy']
     }
@@ -46,7 +46,7 @@ const topResources = {
     index: config.get('ES.DOCUMENTS.attributegroup.index'),
     type: config.get('ES.DOCUMENTS.attributegroup.type'),
     enrich: {
-      policyName: 'attributegroup-policy',
+      policyName: config.get('ES.DOCUMENTS.attributegroup.enrichPolicyName'),
       matchField: 'id',
       enrichFields: ['id', 'name', 'organizationId', 'created', 'updated', 'createdBy', 'updatedBy']
     },
@@ -62,7 +62,7 @@ const topResources = {
     index: config.get('ES.DOCUMENTS.skill.index'),
     type: config.get('ES.DOCUMENTS.skill.type'),
     enrich: {
-      policyName: 'skill-policy',
+      policyName: config.get('ES.DOCUMENTS.skill.enrichPolicyName'),
       matchField: 'id',
       enrichFields: ['id', 'skillProviderId', 'name', 'externalId', 'uri', 'created', 'updated', 'createdBy', 'updatedBy', 'skillprovider']
     },
@@ -77,7 +77,7 @@ const topResources = {
     index: config.get('ES.DOCUMENTS.attribute.index'),
     type: config.get('ES.DOCUMENTS.attribute.type'),
     enrich: {
-      policyName: 'attribute-policy',
+      policyName: config.get('ES.DOCUMENTS.attribute.enrichPolicyName'),
       matchField: 'id',
       enrichFields: ['id', 'name', 'attributeGroupId', 'created', 'updated', 'createdBy', 'updatedBy', 'attributegroup']
     },
@@ -168,7 +168,7 @@ const organizationResources = {
     propertyName: config.get('ES.DOCUMENTS.organizationskillprovider.orgField'),
     relateKey: 'skillProviderId',
     enrich: {
-      policyName: 'organization-policy',
+      policyName: config.get('ES.DOCUMENTS.organization.enrichPolicyName'),
       matchField: 'id',
       enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy', 'skillProviders']
     }

scripts/db/dumpDbToEs.js

@@ -29,40 +29,88 @@ const RESOURCES_IN_ORDER = [
 
 const client = getESClient()
 
+const RESOURCE_NOT_FOUND = 'resource_not_found_exception'
+const INDEX_NOT_FOUND = 'index_not_found_exception'
+
 /**
  * Cleans up the data in elasticsearch
  * @param {Array} keys Array of models
  */
 async function cleanupES (keys) {
   const client = getESClient()
-  await client.ingest.deletePipeline({
-    id: topResources.user.pipeline.id
-  })
-  await client.ingest.deletePipeline({
-    id: topResources.skillprovider.pipeline.id
-  })
-  await client.ingest.deletePipeline({
-    id: topResources.attributegroup.pipeline.id
-  })
-  for (let i = 0; i < keys.length; i++) {
-    const key = keys[i]
-    if (models[key].tableName) {
-      const esResourceName = modelToESIndexMapping[key]
-      if (_.includes(_.keys(topResources), esResourceName)) {
-        if (topResources[esResourceName].enrich) {
-          await client.enrich.deletePolicy({
-            name: topResources[esResourceName].enrich.policyName
-          })
+  try {
+    await client.ingest.deletePipeline({
+      id: topResources.user.pipeline.id
+    })
+  } catch (e) {
+    if (e.meta && e.meta.body.error.type !== RESOURCE_NOT_FOUND) {
+      throw e
+    }
+  }
+
+  try {
+    await client.ingest.deletePipeline({
+      id: topResources.skillprovider.pipeline.id
+    })
+  } catch (e) {
+    if (e.meta && e.meta.body.error.type !== RESOURCE_NOT_FOUND) {
+      throw e
+    }
+  }
+
+  try {
+    await client.ingest.deletePipeline({
+      id: topResources.attributegroup.pipeline.id
+    })
+  } catch (e) {
+    if (e.meta && e.meta.body.error.type !== RESOURCE_NOT_FOUND) {
+      throw e
+    }
+  }
+
+  try {
+    for (let i = 0; i < keys.length; i++) {
+      const key = keys[i]
+      if (models[key].tableName) {
+        const esResourceName = modelToESIndexMapping[key]
+        if (_.includes(_.keys(topResources), esResourceName)) {
+          if (topResources[esResourceName].enrich) {
+            try {
+              await client.enrich.deletePolicy({
+                name: topResources[esResourceName].enrich.policyName
+              })
+            } catch (e) {
+              if (e.meta && e.meta.body.error.type !== RESOURCE_NOT_FOUND) {
+                throw e
+              }
+            }
+          }
+
+          try {
+            await client.indices.delete({
+              index: topResources[esResourceName].index
+            })
+          } catch (e) {
+            if (e.meta && e.meta.body.error.type !== INDEX_NOT_FOUND) {
+              throw e
+            }
+          }
+        } else if (_.includes(_.keys(organizationResources), esResourceName)) {
+          try {
+            await client.enrich.deletePolicy({
+              name: organizationResources[esResourceName].enrich.policyName
+            })
+          } catch (e) {
+            if (e.meta && e.meta.body.error.type !== RESOURCE_NOT_FOUND) {
+              throw e
+            }
+          }
         }
-        await client.indices.delete({
-          index: topResources[esResourceName].index
-        })
-      } else if (_.includes(_.keys(organizationResources), esResourceName)) {
-        await client.enrich.deletePolicy({
-          name: organizationResources[esResourceName].enrich.policyName
-        })
       }
     }
+  } catch (e) {
+    console.log(JSON.stringify(e))
+    throw e
   }
   console.log('Existing data in elasticsearch has been deleted!')
 }

src/common/helper.js

@@ -109,6 +109,9 @@ function readerToJson (reader) {
           toRealValue(r, setValue(name, []))
           r.stepOut()
           break
+        case IonTypes.BOOL:
+          setValue(name, r.booleanValue())
+          break
       }
       nextT = reader.next()
     }

src/common/service-helper.js

@@ -190,7 +190,7 @@ function getServiceMethods (Model, createSchema, patchSchema, searchSchema, buil
   async function patch (id, entity, auth, params) {
     await makeSureRefExist(entity)
 
-    const dbEntity = await get(id, auth, params)
+    const dbEntity = await get(id, auth, params, {}, true)
     const newEntity = new Model()
     _.extend(newEntity, dbEntity, entity)
     newEntity.updated = new Date()
@@ -218,23 +218,26 @@ function getServiceMethods (Model, createSchema, patchSchema, searchSchema, buil
    * @param auth the auth obj
    * @param params the path parameters
    * @param query the query parameters
+   * @param fromDb Should we bypass Elasticsearch for the record and fetch from db instead?
    * @return {Promise} the db device
    */
-  async function get (id, auth, params, query = {}) {
+  async function get (id, auth, params, query = {}, fromDb = false) {
     let recordObj
     // Merge path and query params
     const trueParams = _.assign(params, query)
-    try {
-      const result = await esHelper.getFromElasticSearch(resource, id, auth, trueParams)
-      // check permission
-      permissionCheck(auth, result)
-      return result
-    } catch (err) {
-      // return error if enrich fails or permission fails
-      if ((resource === 'user' && trueParams.enrich) || (err.status && err.status === 403)) {
-        throw errors.elasticSearchEnrichError(err.message)
+    if (!fromDb) {
+      try {
+        const result = await esHelper.getFromElasticSearch(resource, id, auth, trueParams)
+        // check permission
+        permissionCheck(auth, result)
+        return result
+      } catch (err) {
+        // return error if enrich fails or permission fails
+        if ((resource === 'user' && trueParams.enrich) || (err.status && err.status === 403)) {
+          throw errors.elasticSearchEnrichError(err.message)
+        }
+        logger.logFullError(err)
       }
-      logger.logFullError(err)
     }
     if (_.isNil(trueParams) || _.isEmpty(trueParams)) {
       recordObj = await models.DBHelper.get(Model, id)
@@ -302,7 +305,7 @@ function getServiceMethods (Model, createSchema, patchSchema, searchSchema, buil
    */
   async function remove (id, auth, params) {
     let payload
-    await get(id, auth, params) // check exist
+    await get(id, auth, params, {}, true) // check exist
     await models.DBHelper.delete(Model, id, buildQueryByParams(params))
     if (SUB_USER_DOCUMENTS[resource] || SUB_ORG_DOCUMENTS[resource]) {
       payload = _.assign({}, params)