Merge pull request #25 from topcoder-platform/develop

Sync master with develop

Author: callmekatootie

README.md

@@ -64,6 +64,13 @@ The following parameters can be set in config files or in env variables:
 - ELASTICCLOUD_ID: The elastic cloud id, if your elasticsearch instance is hosted on elastic cloud. DO NOT provide a value for ES_HOST if you are using this
 - ELASTICCLOUD_USERNAME: The elastic cloud username for basic authentication. Provide this only if your elasticsearch instance is hosted on elastic cloud
 - ELASTICCLOUD_PASSWORD: The elastic cloud password for basic authentication. Provide this only if your elasticsearch instance is hosted on elastic cloud
+- AUTH0_URL: The auth0 url, Default is 'https://topcoder-dev.auth0.com/oauth/token'
+- AUTH0_AUDIENCE: The auth0 audience for accessing ubahn api(s), Default is 'https://m2m.topcoder-dev.com/'
+- AUTH0_CLIENT_ID: The auth0 client id
+- AUTH0_CLIENT_SECRET: The auth0 client secret
+- AUTH0_PROXY_SERVER_URL: The auth0 proxy server url
+- TOKEN_CACHE_TIME: The token cache time
+- TOPCODER_GROUP_API: The topcoder groups api, Default is 'https://api.topcoder-dev.com/v5/groups'
 
 There is a `/health` endpoint that checks for the health of the app. This sets up an expressjs server and listens on the environment variable `PORT`. It's not part of the configuration file and needs to be passed as an environment variable
 
@@ -89,7 +96,7 @@ Configuration for the tests is at `config/test.js`, only add such new configurat
     docker-compose up -d
     ```
 
-3. initialize Elasticsearch, create configured Elasticsearch index: `npm run init-es force`
+3. initialize Elasticsearch. Execute the `insert-data` script in the [API repository](https://github.com/topcoder-platform/u-bahn-api) to set it up and then clear only the data
 
 ## Local deployment
 

config/default.js

@@ -14,6 +14,15 @@ module.exports = {
   // Kafka group id
   KAFKA_GROUP_ID: process.env.KAFKA_GROUP_ID || 'ubahn-processor-es',
 
+  TOPCODER_GROUP_API: process.env.TOPCODER_GROUP_API || 'https://api.topcoder-dev.com/v5/groups',
+
+  AUTH0_URL: process.env.AUTH0_URL || 'https://topcoder-dev.auth0.com/oauth/token', // Auth0 credentials
+  AUTH0_AUDIENCE: process.env.AUTH0_AUDIENCE || 'https://m2m.topcoder-dev.com/',
+  TOKEN_CACHE_TIME: process.env.TOKEN_CACHE_TIME,
+  AUTH0_CLIENT_ID: process.env.AUTH0_CLIENT_ID,
+  AUTH0_CLIENT_SECRET: process.env.AUTH0_CLIENT_SECRET,
+  AUTH0_PROXY_SERVER_URL: process.env.AUTH0_PROXY_SERVER_URL,
+
   UBAHN_CREATE_TOPIC: process.env.UBAHN_CREATE_TOPIC || 'u-bahn.action.create',
   UBAHN_UPDATE_TOPIC: process.env.UBAHN_UPDATE_TOPIC || 'u-bahn.action.update',
   UBAHN_DELETE_TOPIC: process.env.UBAHN_DELETE_TOPIC || 'u-bahn.action.delete',
@@ -84,6 +93,26 @@ module.exports = {
       },
       organization: {
         enrichPolicyName: process.env.ORGANIZATION_ENRICH_POLICYNAME || 'organization-policy'
+      },
+      // sub resources under user
+      achievement: {
+        userField: process.env.USER_ACHIEVEMENT_PROPERTY_NAME || 'achievements'
+      },
+      externalprofile: {
+        userField: process.env.USER_EXTERNALPROFILE_PROPERTY_NAME || 'externalProfiles'
+      },
+      userattribute: {
+        userField: process.env.USER_ATTRIBUTE_PROPERTY_NAME || 'attributes'
+      },
+      userrole: {
+        userField: process.env.USER_ROLE_PROPERTY_NAME || 'roles'
+      },
+      userskill: {
+        userField: process.env.USER_SKILL_PROPERTY_NAME || 'skills'
+      },
+      // sub resources under organization
+      organizationskillprovider: {
+        orgField: process.env.ORGANIZATION_SKILLPROVIDER_PROPERTY_NAME || 'skillProviders'
       }
     }
   }

package.json

@@ -7,7 +7,6 @@
     "start": "node src/app.js",
     "lint": "standard",
     "lint:fix": "standard --fix",
-    "init-es": "node test/common/init-es.js",
     "view-data": "node test/common/view-data.js",
     "test": "mocha test/unit/test.js --require test/unit/prepare.js --timeout 20000 --exit",
     "test:cov": "nyc --reporter=html --reporter=text mocha test/unit/test.js --require test/unit/prepare.js --timeout 20000 --exit",
@@ -35,6 +34,8 @@
     "get-parameter-names": "^0.3.0",
     "lodash": "^4.17.19",
     "no-kafka": "^3.4.3",
+    "axios": "^0.19.2",
+    "tc-core-library-js": "appirio-tech/tc-core-library-js.git#v2.6",
     "topcoder-healthcheck-dropin": "^1.0.3",
     "winston": "^3.2.1"
   },

src/common/constants.js

@@ -10,14 +10,18 @@ const topResources = {
     index: config.get('ES.ACHIEVEMENT_PROVIDER_INDEX'),
     type: config.get('ES.ACHIEVEMENT_PROVIDER_TYPE'),
     enrich: {
-      policyName: config.get('ES.ENRICHMENT.achievementprovider.enrichPolicyName')
+      policyName: config.get('ES.ENRICHMENT.achievementprovider.enrichPolicyName'),
+      matchField: 'id',
+      enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy']
     }
   },
   attribute: {
     index: config.get('ES.ATTRIBUTE_INDEX'),
     type: config.get('ES.ATTRIBUTE_TYPE'),
     enrich: {
-      policyName: config.get('ES.ENRICHMENT.attribute.enrichPolicyName')
+      policyName: config.get('ES.ENRICHMENT.attribute.enrichPolicyName'),
+      matchField: 'id',
+      enrichFields: ['id', 'name', 'attributeGroupId', 'created', 'updated', 'createdBy', 'updatedBy', 'attributegroup']
     },
     ingest: {
       pipeline: {
@@ -29,7 +33,15 @@ const topResources = {
     index: config.get('ES.ATTRIBUTE_GROUP_INDEX'),
     type: config.get('ES.ATTRIBUTE_GROUP_TYPE'),
     enrich: {
-      policyName: config.get('ES.ENRICHMENT.attributegroup.enrichPolicyName')
+      policyName: config.get('ES.ENRICHMENT.attributegroup.enrichPolicyName'),
+      matchField: 'id',
+      enrichFields: ['id', 'name', 'organizationId', 'created', 'updated', 'createdBy', 'updatedBy']
+    },
+    pipeline: {
+      id: config.get('ES.ENRICHMENT.attributegroup.pipelineId'),
+      field: 'attributeGroupId',
+      targetField: 'attributegroup',
+      maxMatches: '1'
     }
   },
   organization: {
@@ -40,14 +52,18 @@ const topResources = {
     index: config.get('ES.ROLE_INDEX'),
     type: config.get('ES.ROLE_TYPE'),
     enrich: {
-      policyName: config.get('ES.ENRICHMENT.role.enrichPolicyName')
+      policyName: config.get('ES.ENRICHMENT.role.enrichPolicyName'),
+      matchField: 'id',
+      enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy']
     }
   },
   skill: {
     index: config.get('ES.SKILL_INDEX'),
     type: config.get('ES.SKILL_TYPE'),
     enrich: {
-      policyName: config.get('ES.ENRICHMENT.skill.enrichPolicyName')
+      policyName: config.get('ES.ENRICHMENT.skill.enrichPolicyName'),
+      matchField: 'id',
+      enrichFields: ['id', 'skillProviderId', 'name', 'externalId', 'uri', 'created', 'updated', 'createdBy', 'updatedBy', 'skillprovider']
     },
     ingest: {
       pipeline: {
@@ -59,7 +75,15 @@ const topResources = {
     index: config.get('ES.SKILL_PROVIDER_INDEX'),
     type: config.get('ES.SKILL_PROVIDER_TYPE'),
     enrich: {
-      policyName: config.get('ES.ENRICHMENT.skillprovider.enrichPolicyName')
+      policyName: config.get('ES.ENRICHMENT.skillprovider.enrichPolicyName'),
+      matchField: 'id',
+      enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy']
+    },
+    pipeline: {
+      id: config.get('ES.ENRICHMENT.skillprovider.pipelineId'),
+      field: 'skillProviderId',
+      targetField: 'skillprovider',
+      maxMatches: '1'
     }
   },
   user: {
@@ -69,6 +93,39 @@ const topResources = {
       pipeline: {
         id: config.get('ES.ENRICHMENT.user.pipelineId')
       }
+    },
+    pipeline: {
+      id: config.get('ES.ENRICHMENT.user.pipelineId'),
+      processors: [
+        {
+          referenceField: config.get('ES.ENRICHMENT.achievement.userField'),
+          enrichPolicyName: config.get('ES.ENRICHMENT.achievementprovider.enrichPolicyName'),
+          field: '_ingest._value.achievementsProviderId',
+          targetField: '_ingest._value.achievementprovider',
+          maxMatches: '1'
+        },
+        {
+          referenceField: config.get('ES.ENRICHMENT.userattribute.userField'),
+          enrichPolicyName: config.get('ES.ENRICHMENT.attribute.enrichPolicyName'),
+          field: '_ingest._value.attributeId',
+          targetField: '_ingest._value.attribute',
+          maxMatches: '1'
+        },
+        {
+          referenceField: config.get('ES.ENRICHMENT.userrole.userField'),
+          enrichPolicyName: config.get('ES.ENRICHMENT.role.enrichPolicyName'),
+          field: '_ingest._value.roleId',
+          targetField: '_ingest._value.role',
+          maxMatches: '1'
+        },
+        {
+          referenceField: config.get('ES.ENRICHMENT.userskill.userField'),
+          enrichPolicyName: config.get('ES.ENRICHMENT.skill.enrichPolicyName'),
+          field: '_ingest._value.skillId',
+          targetField: '_ingest._value.skill',
+          maxMatches: '1'
+        }
+      ]
     }
   }
 }
@@ -106,7 +163,12 @@ const organizationResources = {
   organizationskillprovider: {
     propertyName: config.get('ES.ORGANIZATION_SKILLPROVIDER_PROPERTY_NAME'),
     relateKey: 'skillProviderId',
-    validate: payload => validProperties(payload, ['organizationId', 'skillProviderId'])
+    validate: payload => validProperties(payload, ['organizationId', 'skillProviderId']),
+    enrich: {
+      policyName: config.get('ES.ENRICHMENT.organization.enrichPolicyName'),
+      matchField: 'id',
+      enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy', 'skillProviders']
+    }
   }
 }
 

src/common/helper.js

@@ -8,6 +8,10 @@ const elasticsearch = require('@elastic/elasticsearch')
 const _ = require('lodash')
 const Joi = require('@hapi/joi')
 const { Mutex } = require('async-mutex')
+const axios = require('axios')
+const logger = require('./logger')
+const m2mAuth = require('tc-core-library-js').auth.m2m
+const topcoderM2M = m2mAuth(_.pick(config, ['AUTH0_URL', 'AUTH0_AUDIENCE', 'TOKEN_CACHE_TIME', 'AUTH0_PROXY_SERVER_URL']))
 
 AWS.config.region = config.ES.AWS_REGION
 
@@ -18,6 +22,34 @@ let transactionId
 const esClientMutex = new Mutex()
 const mutexReleaseMap = {}
 
+/* Function to get M2M token
+ * (Topcoder APIs only)
+ * @returns {Promise}
+ */
+async function getTopcoderM2Mtoken () {
+  return topcoderM2M.getMachineToken(config.AUTH0_CLIENT_ID, config.AUTH0_CLIENT_SECRET)
+}
+
+/**
+ * Returns the user in Topcoder identified by the email
+ * @param {String} email The user email
+ */
+async function getUserGroup (memberId) {
+  const url = config.TOPCODER_GROUP_API
+  const token = await getTopcoderM2Mtoken()
+  const params = { memberId, membershipType: 'user', page: 1 }
+
+  logger.debug(`request GET ${url} with params: ${JSON.stringify(params)}`)
+  let groups = []
+  let groupRes = await axios.get(url, { headers: { Authorization: `Bearer ${token}` }, params })
+  while (groupRes.data.length > 0) {
+    groups = _.concat(groups, _.map(groupRes.data, g => _.pick(g, 'id', 'name')))
+    params.page = params.page + 1
+    groupRes = await axios.get(url, { headers: { Authorization: `Bearer ${token}` }, params })
+  }
+  return groups
+}
+
 /**
  * Get Kafka options
  * @return {Object} the Kafka options
@@ -201,6 +233,7 @@ module.exports = {
   getESClient,
   validProperties,
   getUser,
+  getUserGroup,
   updateUser,
   getOrg,
   updateOrg,

src/services/ProcessorService.js

@@ -49,6 +49,12 @@ async function processCreate (message, transactionId) {
       user[userResource.propertyName] = []
     }
 
+    // import groups for a new user
+    if (resource === 'externalprofile' && message.payload.externalId) {
+      const userGroups = await helper.getUserGroup(message.payload.externalId)
+      user[config.get('ES.USER_GROUP_PROPERTY_NAME')] = _.unionBy(user[config.get('ES.USER_GROUP_PROPERTY_NAME')], userGroups, 'id')
+    }
+
     // check the resource does not exist
     if (_.some(user[userResource.propertyName], [userResource.relateKey, relateId])) {
       logger.error(`Can't create existed ${resource} with the ${userResource.relateKey}: ${relateId}, userId: ${message.payload.userId}`)