diff --git a/README.md b/README.md index 386af17..15c2086 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,13 @@ The following parameters can be set in config files or in env variables: - UPDATE_DATA_TOPIC: update data Kafka topic, default value is 'project.action.update' - DELETE_DATA_TOPIC: delete data Kafka topic, default value is 'project.action.delete' - KAFKA_MESSAGE_ORIGINATOR: Kafka topic originator, default value is 'project-api' +- MEMBER_SERVICE_ENDPOINT: used to get member details +- AUTH0_URL: AUTH0 URL, used to get M2M token +- AUTH0_PROXY_SERVER_URL: AUTH0 proxy server URL, used to get M2M token +- AUTH0_AUDIENCE: AUTH0 audience, used to get M2M token +- TOKEN_CACHE_TIME: AUTH0 token cache time, used to get M2M token +- AUTH0_CLIENT_ID: AUTH0 client id, used to get M2M token +- AUTH0_CLIENT_SECRET: AUTH0 client secret, used to get M2M token - esConfig: config object for Elasticsearch Refer to `esConfig` variable in `config/default.js` for ES related configuration. @@ -69,12 +76,20 @@ Config for tests are at `config/test.js`, it overrides some default config. - In the `docker-es` folder, run `docker-compose up` ## Local deployment - - Install dependencies `npm i` - Run code lint check `npm run lint`, running `npm run lint:fix` can fix some lint errors if any - Initialize Elasticsearch, create configured Elasticsearch index if not present: `npm run sync:es` - Start processor app `npm start` +Note that you need to set AUTH0 related environment variables belows before you can start the processor. + +- AUTH0_URL +- AUTH0_AUDIENCE +- TOKEN_CACHE_TIME +- AUTH0_CLIENT_ID +- AUTH0_CLIENT_SECRET +- AUTH0_PROXY_SERVER_URL + ## Local Deployment with Docker To run the Challenge ES Processor using docker, follow the below steps @@ -254,4 +269,4 @@ info: { - Then in the app console, you will see error messages - To test the health check API, run `export PORT=5000`, start the processor, then browse `http://localhost:5000/health` in a browser, - and you will see result `{"checksRun":1}` \ No newline at end of file + and you will see result `{"checksRun":1}` diff --git a/config/default.js b/config/default.js index e8f2d8b..46d8860 100644 --- a/config/default.js +++ b/config/default.js @@ -17,6 +17,15 @@ module.exports = { DELETE_DATA_TOPIC: process.env.DELETE_DATA_TOPIC || 'project.action.delete', KAFKA_MESSAGE_ORIGINATOR: process.env.KAFKA_MESSAGE_ORIGINATOR || 'project-api', + MEMBER_SERVICE_ENDPOINT: process.env.MEMBER_SERVICE_ENDPOINT || 'https://api.topcoder-dev.com/v3/members', + + AUTH0_URL: process.env.AUTH0_URL, + AUTH0_PROXY_SERVER_URL: process.env.AUTH0_PROXY_SERVER_URL, + AUTH0_AUDIENCE: process.env.AUTH0_AUDIENCE, + TOKEN_CACHE_TIME: process.env.TOKEN_CACHE_TIME, + AUTH0_CLIENT_ID: process.env.AUTH0_CLIENT_ID, + AUTH0_CLIENT_SECRET: process.env.AUTH0_CLIENT_SECRET, + esConfig: { HOST: process.env.ES_HOST || 'localhost:9200', AWS_REGION: process.env.AWS_REGION || 'us-east-1', // AWS Region to be used if we use AWS ES diff --git a/package.json b/package.json index 7dcca64..6745ef8 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,9 @@ "joi": "^14.3.1", "lodash": "^4.17.11", "no-kafka": "^3.4.3", + "tc-core-library-js": "github:appirio-tech/tc-core-library-js#v2.6", "topcoder-healthcheck-dropin": "^1.0.3", + "urlencode": "^1.1.0", "winston": "^3.2.1" }, "standard": { diff --git a/src/common/helper.js b/src/common/helper.js index 7fb0d03..c67d105 100644 --- a/src/common/helper.js +++ b/src/common/helper.js @@ -5,6 +5,9 @@ const AWS = require('aws-sdk') const config = require('config') const elasticsearch = require('elasticsearch') +const _ = require('lodash') +const tcCoreLibAuth = require('tc-core-library-js') +const urlencode = require('urlencode') const logger = require('./logger') @@ -12,6 +15,21 @@ AWS.config.region = config.get('esConfig.AWS_REGION') // ES Client mapping const esClients = {} +const m2mAuth = tcCoreLibAuth.auth.m2m +const util = tcCoreLibAuth.util(config) +let m2m = null + +/** + * Get machine to machine token. + * @returns {Promise} promise which resolves to the m2m token + */ +async function getM2MToken () { + if (_.isNull(m2m)) { + m2m = m2mAuth(_.pick(config, ['AUTH0_URL', 'AUTH0_AUDIENCE', 'TOKEN_CACHE_TIME', 'AUTH0_PROXY_SERVER_URL'])) + } + return m2m.getMachineToken(config.AUTH0_CLIENT_ID, config.AUTH0_CLIENT_SECRET) +} + /** * Get ES Client * @return {Object} Elastic Host Client Instance @@ -126,9 +144,35 @@ async function updateMetadadaESPromise (updateDocHandler) { }) } +/** + * Retrieve member details from userIds + * + * @param {Array} userIds the list of userIds + * @returns {Promise} the member details + */ +async function getMemberDetailsByUserIds (userIds) { + try { + const token = await getM2MToken() + const httpClient = util.getHttpClient({ id: `projectMemberService_${userIds.join('_')}`, log: logger }) + return httpClient.get(`${config.MEMBER_SERVICE_ENDPOINT}/_search`, { + params: { + query: `${_.map(userIds, id => `userId:${id}`).join(urlencode(' OR ', 'utf8'))}`, + fields: 'userId,handle,firstName,lastName,email' + }, + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${token}` + } + }).then(res => _.get(res, 'data.result.content', null)) + } catch (err) { + return Promise.reject(err) + } +} + module.exports = { getESClient, updateProjectESPromise, updateTimelineESPromise, - updateMetadadaESPromise + updateMetadadaESPromise, + getMemberDetailsByUserIds } diff --git a/src/services/ProcessorServiceProjectMember.js b/src/services/ProcessorServiceProjectMember.js index 4fddeea..4314c83 100644 --- a/src/services/ProcessorServiceProjectMember.js +++ b/src/services/ProcessorServiceProjectMember.js @@ -53,8 +53,15 @@ async function create (message) { async function updateDocPromise (doc) { const members = _.isArray(doc._source.members) ? doc._source.members : [] const existingMemberIndex = _.findIndex(members, p => p.id === message.id)// if member does not exists already + console.log(message) if (existingMemberIndex === -1) { - members.push(message) + if (!message.userId) { + members.push(message) + return + } + const memberDetails = await helper.getMemberDetailsByUserIds([message.userId]) + const messageWithDetails = _.merge(message, _.pick(memberDetails[0], 'handle', 'firstName', 'lastName', 'email')) + members.push(messageWithDetails) } else { // if member already exists, ideally we should never land here, but code handles the buggy indexing // replaces the old inconsistent index where previously member was not removed from the index but deleted // from the database diff --git a/test/e2e/processor.project.index.test.js b/test/e2e/processor.project.index.test.js index bff092e..6dbfec8 100644 --- a/test/e2e/processor.project.index.test.js +++ b/test/e2e/processor.project.index.test.js @@ -11,6 +11,7 @@ const Joi = require('joi') const ProcessorService = require('../../src/services/ProcessorService') const testHelper = require('../common/testHelper') const logger = require('../../src/common/logger') +const helper = require('../../src/common/helper') const { projectId, @@ -799,13 +800,21 @@ xdescribe('TC Phase Product Topic Tests', () => { }) describe('TC Project Member Topic Tests', () => { + let getMemberDetailsByUserIds + before(async () => { // runs before all tests in this block await ProcessorService.create(projectCreatedMessage) + getMemberDetailsByUserIds = helper.getMemberDetailsByUserIds + helper.getMemberDetailsByUserIds = async (userIds) => { + return [] // return empty details + } }) after(async () => { // runs after all tests in this block await ProcessorService.deleteMessage(projectDeletedMessage) + // restore the method + helper.getMemberDetailsByUserIds = getMemberDetailsByUserIds }) it('create project member message', async () => {