From b75a4355ec83f1be86ec46ea2953ca41b39c33e4 Mon Sep 17 00:00:00 2001 From: Caizheng Peng Date: Fri, 4 Dec 2020 21:37:18 +0800 Subject: [PATCH 1/5] create job after project was created (#595) --- config/default.json | 3 +- config/development.json | 3 +- src/events/projects/index.js | 57 ++++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 2 deletions(-) diff --git a/config/default.json b/config/default.json index 87fcaa52..b302a52f 100644 --- a/config/default.json +++ b/config/default.json @@ -75,5 +75,6 @@ "EMBED_REPORTS_MAPPING": "{\"mock\": \"/embed/looks/2\"}", "ALLOWED_USERS": "[]" }, - "DEFAULT_M2M_USERID": -101 + "DEFAULT_M2M_USERID": -101, + "taasJobApiUrl": "https://api.topcoder.com/v5/jobs" } diff --git a/config/development.json b/config/development.json index 3f3e909b..d20cddd9 100644 --- a/config/development.json +++ b/config/development.json @@ -6,5 +6,6 @@ "fileServiceEndpoint": "https://api.topcoder-dev.com/v3/files/", "connectProjectsUrl": "https://connect.topcoder-dev.com/projects/", "memberServiceEndpoint": "https://api.topcoder-dev.com/v3/members", - "identityServiceEndpoint": "https://api.topcoder-dev.com/v3/" + "identityServiceEndpoint": "https://api.topcoder-dev.com/v3/", + "taasJobApiUrl": "https://api.topcoder-dev.com/v5/jobs" } diff --git a/src/events/projects/index.js b/src/events/projects/index.js index 0f8dce34..b211cd85 100644 --- a/src/events/projects/index.js +++ b/src/events/projects/index.js @@ -5,6 +5,8 @@ import _ from 'lodash'; import Joi from 'joi'; import Promise from 'bluebird'; import config from 'config'; +import axios from 'axios'; +import moment from 'moment'; import util from '../../util'; import models from '../../models'; import { createPhaseTopic } from '../projectPhases'; @@ -14,6 +16,27 @@ const ES_PROJECT_INDEX = config.get('elasticsearchConfig.indexName'); const ES_PROJECT_TYPE = config.get('elasticsearchConfig.docType'); const eClient = util.getElasticSearchClient(); +/** + * creates taas job + * @param {Object} data the job data + * @return {Object} the job created + */ +const createTaasJob = async (data) => { + const token = await util.getM2MToken(); + const headers = { + 'Content-Type': 'application/json', + Authorization: `Bearer ${token}`, + }; + const res = await axios + .post(config.taasJobApiUrl, data, { headers }) + .catch((err) => { + const error = new Error(); + error.message = _.get(err, 'response.data.message', error.message); + throw error; + }); + return res.data; +}; + /** * Payload for deprecated BUS events like `connect.notification.project.updated`. */ @@ -164,6 +187,40 @@ async function projectCreatedKafkaHandler(app, topic, payload) { await Promise.all(topicPromises); app.logger.debug('Topics for phases are successfully created.'); } + if (project.type === 'talent-as-a-service') { + const specialists = _.get(project, 'details.taasDefinition.specialists'); + if (!specialists || !specialists.length) { + app.logger.debug(`no specialists found in the project ${project.id}`); + return; + } + const targetSpecialists = _.filter(specialists, specialist => Number(specialist.people) > 0); // must be at least one people + await Promise.all( + _.map( + targetSpecialists, + (specialist) => { + const startDate = new Date(); + const endDate = moment(startDate).add(Number(specialist.duration), 'M'); // the unit of duration is month + const skills = specialist.skills.filter(skill => skill.id).map(skill => skill.id); + return createTaasJob({ + projectId: project.id, + externalId: _.get(project, 'external.id') || String(project.id), + description: specialist.roleTitle, + startDate, + endDate, + skills, + numPositions: Number(specialist.people), + resourceType: specialist.role, + rateType: 'hourly', + workload: specialist.workLoad.title.toLowerCase(), + }).then((job) => { + app.logger.debug(`jobId: ${job.id} job created for roleTitle ${specialist.roleTitle}`); + }).catch((err) => { + app.logger.error(`Unable to create job for ${specialist.roleTitle}: ${err.message}`); + }); + }, + ), + ); + } } module.exports = { From 4116ef169407630e0aed22671c73e7dca034851b Mon Sep 17 00:00:00 2001 From: maxceem Date: Fri, 4 Dec 2020 16:35:39 +0200 Subject: [PATCH 2/5] fix: Kafka topics handlers config - As most of the topics have the same name, we cannot simply use them as objects keys. So we have to use a register method which would merge the config smartly. --- src/events/kafkaHandlers.js | 94 +++++++++++++++++++++++++++---------- 1 file changed, 70 insertions(+), 24 deletions(-) diff --git a/src/events/kafkaHandlers.js b/src/events/kafkaHandlers.js index b6bf90cf..c1c76611 100644 --- a/src/events/kafkaHandlers.js +++ b/src/events/kafkaHandlers.js @@ -1,18 +1,22 @@ /** * BUS Event Handlers */ -import { CONNECT_NOTIFICATION_EVENT, BUS_API_EVENT, RESOURCES } from '../constants'; import { - projectCreatedKafkaHandler, - projectUpdatedKafkaHandler } from './projects'; -import { projectPhaseAddedKafkaHandler, projectPhaseRemovedKafkaHandler, - projectPhaseUpdatedKafkaHandler } from './projectPhases'; + CONNECT_NOTIFICATION_EVENT, + BUS_API_EVENT, + RESOURCES, +} from '../constants'; import { - timelineAdjustedKafkaHandler, -} from './timelines'; + projectCreatedKafkaHandler, + projectUpdatedKafkaHandler, +} from './projects'; import { - milestoneUpdatedKafkaHandler, -} from './milestones'; + projectPhaseAddedKafkaHandler, + projectPhaseRemovedKafkaHandler, + projectPhaseUpdatedKafkaHandler, +} from './projectPhases'; +import { timelineAdjustedKafkaHandler } from './timelines'; +import { milestoneUpdatedKafkaHandler } from './milestones'; const kafkaHandlers = { /** @@ -33,22 +37,64 @@ const kafkaHandlers = { // Events coming from timeline/milestones (considering it as a separate module/service in future) [CONNECT_NOTIFICATION_EVENT.MILESTONE_TRANSITION_COMPLETED]: milestoneUpdatedKafkaHandler, [CONNECT_NOTIFICATION_EVENT.TIMELINE_ADJUSTED]: timelineAdjustedKafkaHandler, +}; - /** - * New Unified Bus Events - */ - [BUS_API_EVENT.PROJECT_CREATED]: { - [RESOURCES.PROJECT]: projectCreatedKafkaHandler, - }, - [BUS_API_EVENT.PROJECT_PHASE_CREATED]: { - [RESOURCES.PHASE]: projectPhaseAddedKafkaHandler, - }, - [BUS_API_EVENT.PROJECT_PHASE_UPDATED]: { - [RESOURCES.PHASE]: projectPhaseUpdatedKafkaHandler, - }, - [BUS_API_EVENT.PROJECT_PHASE_DELETED]: { - [RESOURCES.PHASE]: projectPhaseRemovedKafkaHandler, - }, +/** + * Register New Unified Bus Event Handlers + * + * We need this special method so it would properly merge topics with the same names + * but different resources. + * + * @param {String} topic Kafka topic name + * @param {String} resource resource name + * @param {Function} handler handler method + * + * @returns {void} + */ +const registerKafkaHandler = (topic, resource, handler) => { + let topicConfig = kafkaHandlers[topic]; + + // if config for topic is not yet initialized, create it + if (!topicConfig) { + topicConfig = {}; + kafkaHandlers[topic] = topicConfig; + } + + if (typeof topicConfig !== 'object') { + throw new Error( + `Topic "${topic}" should be defined as object with resource names as keys.`, + ); + } + + if (topicConfig[resource]) { + throw new Error( + `Handler for topic "${topic}" with resource ${resource} has been already registered.`, + ); + } + + topicConfig[resource] = handler; }; +registerKafkaHandler( + BUS_API_EVENT.PROJECT_CREATED, + RESOURCES.PROJECT, + projectCreatedKafkaHandler, +); +registerKafkaHandler( + BUS_API_EVENT.PROJECT_PHASE_CREATED, + RESOURCES.PHASE, + projectPhaseAddedKafkaHandler, +); +registerKafkaHandler( + BUS_API_EVENT.PROJECT_PHASE_UPDATED, + RESOURCES.PHASE, + projectPhaseUpdatedKafkaHandler, +); +registerKafkaHandler( + BUS_API_EVENT.PROJECT_PHASE_DELETED, + RESOURCES.PHASE, + projectPhaseRemovedKafkaHandler, +); + + export default kafkaHandlers; From f8e3b8caa906c8259241bbf8696d22e6b745c337 Mon Sep 17 00:00:00 2001 From: maxceem Date: Fri, 4 Dec 2020 16:46:26 +0200 Subject: [PATCH 3/5] fix: improvements to job creation - use token from ENV variable instead of M2M until TaaS API supports M2M --- src/events/projects/index.js | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/events/projects/index.js b/src/events/projects/index.js index b211cd85..1643b960 100644 --- a/src/events/projects/index.js +++ b/src/events/projects/index.js @@ -22,7 +22,10 @@ const eClient = util.getElasticSearchClient(); * @return {Object} the job created */ const createTaasJob = async (data) => { - const token = await util.getM2MToken(); + // TODO uncomment when TaaS API supports M2M tokens + // see https://github.com/topcoder-platform/taas-apis/issues/40 + // const token = await util.getM2MToken(); + const token = process.env.TAAS_API_TOKEN; const headers = { 'Content-Type': 'application/json', Authorization: `Bearer ${token}`, @@ -200,18 +203,21 @@ async function projectCreatedKafkaHandler(app, topic, payload) { (specialist) => { const startDate = new Date(); const endDate = moment(startDate).add(Number(specialist.duration), 'M'); // the unit of duration is month - const skills = specialist.skills.filter(skill => skill.id).map(skill => skill.id); + // use both, required and additional skills for jobs + const skills = specialist.skills.concat(specialist.additionalSkills) + // only include skills with `id` and ignore custom skills in jobs + .filter(skill => skill.id).map(skill => skill.id); return createTaasJob({ projectId: project.id, - externalId: _.get(project, 'external.id') || String(project.id), + externalId: '0', // hardcode for now description: specialist.roleTitle, startDate, endDate, skills, numPositions: Number(specialist.people), resourceType: specialist.role, - rateType: 'hourly', - workload: specialist.workLoad.title.toLowerCase(), + rateType: 'hourly', // hardcode for now + workload: _.get(specialist, 'workLoad.title', '').toLowerCase(), }).then((job) => { app.logger.debug(`jobId: ${job.id} job created for roleTitle ${specialist.roleTitle}`); }).catch((err) => { From cccb677d626697b4650753ad7d5deec08965a227 Mon Sep 17 00:00:00 2001 From: maxceem Date: Fri, 4 Dec 2020 17:04:58 +0200 Subject: [PATCH 4/5] chore: deploy branch "feature/create-taas-jobs" --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d9851d0d..8a7b9cac 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -114,7 +114,7 @@ workflows: - test filters: branches: - only: ['develop'] + only: ['develop', 'feature/create-taas-jobs'] - deployProd: context : org-global requires: From 4baf15567f471902b6d58b56ee76825031edcc0d Mon Sep 17 00:00:00 2001 From: maxceem Date: Mon, 7 Dec 2020 14:44:28 +0200 Subject: [PATCH 5/5] feat: use "skilliD" for job creation --- src/events/projects/index.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/events/projects/index.js b/src/events/projects/index.js index 1643b960..3b77a32f 100644 --- a/src/events/projects/index.js +++ b/src/events/projects/index.js @@ -203,10 +203,13 @@ async function projectCreatedKafkaHandler(app, topic, payload) { (specialist) => { const startDate = new Date(); const endDate = moment(startDate).add(Number(specialist.duration), 'M'); // the unit of duration is month - // use both, required and additional skills for jobs - const skills = specialist.skills.concat(specialist.additionalSkills) - // only include skills with `id` and ignore custom skills in jobs - .filter(skill => skill.id).map(skill => skill.id); + // make sure that skills would be unique in the list + const skills = _.uniq( + // use both, required and additional skills for jobs + specialist.skills.concat(specialist.additionalSkills) + // only include skills with `skillId` and ignore custom skills in jobs + .filter(skill => skill.skillId).map(skill => skill.skillId), + ); return createTaasJob({ projectId: project.id, externalId: '0', // hardcode for now