Populate user details of the project members before indexing them in the Elastic Search.

README.md

@@ -25,6 +25,13 @@ The following parameters can be set in config files or in env variables:
 - UPDATE_DATA_TOPIC: update data Kafka topic, default value is 'project.action.update'
 - DELETE_DATA_TOPIC: delete data Kafka topic, default value is 'project.action.delete'
 - KAFKA_MESSAGE_ORIGINATOR: Kafka topic originator, default value is 'project-api'
+- MEMBER_SERVICE_ENDPOINT: used to get member details
+- AUTH0_URL: AUTH0 URL, used to get M2M token
+- AUTH0_PROXY_SERVER_URL: AUTH0 proxy server URL, used to get M2M token
+- AUTH0_AUDIENCE: AUTH0 audience, used to get M2M token
+- TOKEN_CACHE_TIME: AUTH0 token cache time, used to get M2M token
+- AUTH0_CLIENT_ID: AUTH0 client id, used to get M2M token
+- AUTH0_CLIENT_SECRET: AUTH0 client secret, used to get M2M token
 - esConfig: config object for Elasticsearch
 
 Refer to `esConfig` variable in `config/default.js` for ES related configuration.
@@ -69,12 +76,20 @@ Config for tests are at `config/test.js`, it overrides some default config.
 - In the `docker-es` folder, run `docker-compose up`
 
 ## Local deployment
-
 - Install dependencies `npm i`
 - Run code lint check `npm run lint`, running `npm run lint:fix` can fix some lint errors if any
 - Initialize Elasticsearch, create configured Elasticsearch index if not present: `npm run sync:es`
 - Start processor app `npm start`
 
+Note that you need to set AUTH0 related environment variables belows before you can start the processor.
+
+- AUTH0_URL
+- AUTH0_AUDIENCE
+- TOKEN_CACHE_TIME
+- AUTH0_CLIENT_ID
+- AUTH0_CLIENT_SECRET
+- AUTH0_PROXY_SERVER_URL
+
 ## Local Deployment with Docker
 
 To run the Challenge ES Processor using docker, follow the below steps
@@ -254,4 +269,4 @@ info: {
 - Then in the app console, you will see error messages
 
 - To test the health check API, run `export PORT=5000`, start the processor, then browse `http://localhost:5000/health` in a browser,
-  and you will see result `{"checksRun":1}`
+  and you will see result `{"checksRun":1}`

config/default.js

@@ -17,6 +17,15 @@ module.exports = {
   DELETE_DATA_TOPIC: process.env.DELETE_DATA_TOPIC || 'project.action.delete',
   KAFKA_MESSAGE_ORIGINATOR: process.env.KAFKA_MESSAGE_ORIGINATOR || 'project-api',
 
+  MEMBER_SERVICE_ENDPOINT: process.env.MEMBER_SERVICE_ENDPOINT || 'https://api.topcoder-dev.com/v3/members',
+
+  AUTH0_URL: process.env.AUTH0_URL,
+  AUTH0_PROXY_SERVER_URL: process.env.AUTH0_PROXY_SERVER_URL,
+  AUTH0_AUDIENCE: process.env.AUTH0_AUDIENCE,
+  TOKEN_CACHE_TIME: process.env.TOKEN_CACHE_TIME,
+  AUTH0_CLIENT_ID: process.env.AUTH0_CLIENT_ID,
+  AUTH0_CLIENT_SECRET: process.env.AUTH0_CLIENT_SECRET,
+
   esConfig: {
     HOST: process.env.ES_HOST || 'localhost:9200',
     AWS_REGION: process.env.AWS_REGION || 'us-east-1', // AWS Region to be used if we use AWS ES

package.json

@@ -30,7 +30,9 @@
     "joi": "^14.3.1",
     "lodash": "^4.17.11",
     "no-kafka": "^3.4.3",
+    "tc-core-library-js": "github:appirio-tech/tc-core-library-js#v2.6",
     "topcoder-healthcheck-dropin": "^1.0.3",
+    "urlencode": "^1.1.0",
     "winston": "^3.2.1"
   },
   "standard": {

src/common/helper.js

@@ -5,13 +5,31 @@
 const AWS = require('aws-sdk')
 const config = require('config')
 const elasticsearch = require('elasticsearch')
+const _ = require('lodash')
+const tcCoreLibAuth = require('tc-core-library-js')
+const urlencode = require('urlencode')
 
 const logger = require('./logger')
 
 AWS.config.region = config.get('esConfig.AWS_REGION')
 // ES Client mapping
 const esClients = {}
 
+const m2mAuth = tcCoreLibAuth.auth.m2m
+const util = tcCoreLibAuth.util(config)
+let m2m = null
+
+/**
+ * Get machine to machine token.
+ * @returns {Promise} promise which resolves to the m2m token
+ */
+async function getM2MToken () {
+  if (_.isNull(m2m)) {
+    m2m = m2mAuth(_.pick(config, ['AUTH0_URL', 'AUTH0_AUDIENCE', 'TOKEN_CACHE_TIME', 'AUTH0_PROXY_SERVER_URL']))
+  }
+  return m2m.getMachineToken(config.AUTH0_CLIENT_ID, config.AUTH0_CLIENT_SECRET)
+}
+
 /**
  * Get ES Client
  * @return {Object} Elastic Host Client Instance
@@ -126,9 +144,35 @@ async function updateMetadadaESPromise (updateDocHandler) {
   })
 }
 
+/**
+ * Retrieve member details from userIds
+ *
+ * @param {Array} userIds the list of userIds
+ * @returns {Promise} the member details
+ */
+async function getMemberDetailsByUserIds (userIds) {
+  try {
+    const token = await getM2MToken()
+    const httpClient = util.getHttpClient({ id: `projectMemberService_${userIds.join('_')}`, log: logger })
+    return httpClient.get(`${config.MEMBER_SERVICE_ENDPOINT}/_search`, {
+      params: {
+        query: `${_.map(userIds, id => `userId:${id}`).join(urlencode(' OR ', 'utf8'))}`,
+        fields: 'userId,handle,firstName,lastName,email'
+      },
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: `Bearer ${token}`
+      }
+    }).then(res => _.get(res, 'data.result.content', null))
+  } catch (err) {
+    return Promise.reject(err)
+  }
+}
+
 module.exports = {
   getESClient,
   updateProjectESPromise,
   updateTimelineESPromise,
-  updateMetadadaESPromise
+  updateMetadadaESPromise,
+  getMemberDetailsByUserIds
 }

src/services/ProcessorServiceProjectMember.js

@@ -53,8 +53,15 @@ async function create (message) {
   async function updateDocPromise (doc) {
     const members = _.isArray(doc._source.members) ? doc._source.members : []
     const existingMemberIndex = _.findIndex(members, p => p.id === message.id)// if member does not exists already
+    console.log(message)
     if (existingMemberIndex === -1) {
-      members.push(message)
+      if (!message.userId) {
+        members.push(message)
+        return
+      }
+      const memberDetails = await helper.getMemberDetailsByUserIds([message.userId])
+      const messageWithDetails = _.merge(message, _.pick(memberDetails[0], 'handle', 'firstName', 'lastName', 'email'))
+      members.push(messageWithDetails)
     } else { // if member already exists, ideally we should never land here, but code handles the buggy indexing
       // replaces the old inconsistent index where previously member was not removed from the index but deleted
       // from the database

test/e2e/processor.project.index.test.js

@@ -11,6 +11,7 @@ const Joi = require('joi')
 const ProcessorService = require('../../src/services/ProcessorService')
 const testHelper = require('../common/testHelper')
 const logger = require('../../src/common/logger')
+const helper = require('../../src/common/helper')
 
 const {
   projectId,
@@ -799,13 +800,21 @@ xdescribe('TC Phase Product Topic Tests', () => {
 })
 
 describe('TC Project Member Topic Tests', () => {
+  let getMemberDetailsByUserIds
+
   before(async () => {
     // runs before all tests in this block
     await ProcessorService.create(projectCreatedMessage)
+    getMemberDetailsByUserIds = helper.getMemberDetailsByUserIds
+    helper.getMemberDetailsByUserIds = async (userIds) => {
+      return [] // return empty details
+    }
   })
   after(async () => {
     // runs after all tests in this block
     await ProcessorService.deleteMessage(projectDeletedMessage)
+    // restore the method
+    helper.getMemberDetailsByUserIds = getMemberDetailsByUserIds
   })
 
   it('create project member message', async () => {