From 9a425979347fbc017462688eb7856b05d433c749 Mon Sep 17 00:00:00 2001 From: narekcat Date: Sat, 10 Apr 2021 01:32:15 +0400 Subject: [PATCH 01/55] Add work period payments. --- README.md | 2 + VERIFICATION.md | 8 +- config/default.js | 5 +- local/docker-compose.yml | 2 +- src/app.js | 6 +- src/bootstrap.js | 1 + src/scripts/createIndex.js | 13 ++ .../WorkPeriodPaymentProcessorService.js | 134 ++++++++++++++++++ .../taas.workperiodpayment.create.event.json | 1 + .../taas.workperiodpayment.update.event.json | 1 + 10 files changed, 168 insertions(+), 5 deletions(-) create mode 100644 src/services/WorkPeriodPaymentProcessorService.js create mode 100644 test/messages/taas.workperiodpayment.create.event.json create mode 100644 test/messages/taas.workperiodpayment.update.event.json diff --git a/README.md b/README.md index d8409ab..045684f 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,8 @@ The following parameters can be set in config files or in env variables: - `topics.TAAS_WORK_PERIOD_CREATE_TOPIC`: the create work period entity Kafka message topic - `topics.TAAS_WORK_PERIOD_UPDATE_TOPIC`: the update work period entity Kafka message topic - `topics.TAAS_WORK_PERIOD_DELETE_TOPIC`: the delete work period entity Kafka message topic +- `topics.TAAS_WORK_PERIOD_PAYMENT_CREATE_TOPIC`: the create work period payment entity Kafka message topic +- `topics.TAAS_WORK_PERIOD_PAYMENT_UPDATE_TOPIC`: the update work period payment entity Kafka message topic - `esConfig.HOST`: Elasticsearch host - `esConfig.AWS_REGION`: The Amazon region to use when using AWS Elasticsearch service - `esConfig.ELASTICCLOUD.id`: The elastic cloud id, if your elasticsearch instance is hosted on elastic cloud. DO NOT provide a value for ES_HOST if you are using this diff --git a/VERIFICATION.md b/VERIFICATION.md index c6930b9..5410bcf 100644 --- a/VERIFICATION.md +++ b/VERIFICATION.md @@ -2,7 +2,7 @@ ## Create documents in ES -- Run the following commands to create `Job`, `JobCandidate`, `ResourceBooking`, `WorkPeriod` documents in ES. +- Run the following commands to create `Job`, `JobCandidate`, `ResourceBooking`, `WorkPeriod`, `WorkPeriodPayment` documents in ES. ``` bash # for Job @@ -13,12 +13,14 @@ docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.resourcebooking.create < test/messages/taas.resourcebooking.create.event.json # for WorkPeriod docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.workperiod.create < test/messages/taas.workperiod.create.event.json + # for WorkPeriodPayment + docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.workperiodpayment.create < test/messages/taas.workperiodpayment.create.event.json ``` - Run `npm run view-data ` to see if documents were created. ## Update documents in ES -- Run the following commands to update `Job`, `JobCandidate`, `ResourceBooking`, `WorkPeriod` documents in ES. +- Run the following commands to update `Job`, `JobCandidate`, `ResourceBooking`, `WorkPeriod`, `WorkPeriodPayment` documents in ES. ``` bash # for Job @@ -29,6 +31,8 @@ docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.resourcebooking.update < test/messages/taas.resourcebooking.update.event.json # for WorkPeriod docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.workperiod.update < test/messages/taas.workperiod.update.event.json + # for WorkPeriodPayment + docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.workperiodpayment.update < test/messages/taas.workperiodpayment.update.event.json ``` - Run `npm run view-data ` to see if documents were updated. diff --git a/config/default.js b/config/default.js index 446ab4e..7a8dbf8 100644 --- a/config/default.js +++ b/config/default.js @@ -31,7 +31,10 @@ module.exports = { // topics for work period service TAAS_WORK_PERIOD_CREATE_TOPIC: process.env.TAAS_WORK_PERIOD_CREATE_TOPIC || 'taas.workperiod.create', TAAS_WORK_PERIOD_UPDATE_TOPIC: process.env.TAAS_WORK_PERIOD_UPDATE_TOPIC || 'taas.workperiod.update', - TAAS_WORK_PERIOD_DELETE_TOPIC: process.env.TAAS_WORK_PERIOD_DELETE_TOPIC || 'taas.workperiod.delete' + TAAS_WORK_PERIOD_DELETE_TOPIC: process.env.TAAS_WORK_PERIOD_DELETE_TOPIC || 'taas.workperiod.delete', + // topics for work period payment service + TAAS_WORK_PERIOD_PAYMENT_CREATE_TOPIC: process.env.TAAS_WORK_PERIOD_PAYMENT_CREATE_TOPIC || 'taas.workperiodpayment.create', + TAAS_WORK_PERIOD_PAYMENT_UPDATE_TOPIC: process.env.TAAS_WORK_PERIOD_PAYMENT_UPDATE_TOPIC || 'taas.workperiodpayment.update' }, esConfig: { diff --git a/local/docker-compose.yml b/local/docker-compose.yml index 5d2d803..35e9486 100644 --- a/local/docker-compose.yml +++ b/local/docker-compose.yml @@ -12,7 +12,7 @@ services: - "9092:9092" environment: KAFKA_ADVERTISED_HOST_NAME: localhost - KAFKA_CREATE_TOPICS: "taas.job.create:1:1,taas.jobcandidate.create:1:1,taas.resourcebooking.create:1:1,taas.job.update:1:1,taas.jobcandidate.update:1:1,taas.resourcebooking.update:1:1,taas.job.delete:1:1,taas.jobcandidate.delete:1:1,taas.resourcebooking.delete:1:1" + KAFKA_CREATE_TOPICS: "taas.job.create:1:1,taas.jobcandidate.create:1:1,taas.resourcebooking.create:1:1,taas.workperiod.create:1:1,taas.workperiodpayment.create:1:1,taas.job.update:1:1,taas.jobcandidate.update:1:1,taas.resourcebooking.update:1:1,taas.workperiod.update:1:1,taas.workperiodpayment.update:1:1,taas.job.delete:1:1,taas.jobcandidate.delete:1:1,taas.resourcebooking.delete:1:1,taas.workperiod.delete:1:1" KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 esearch: image: elasticsearch:7.7.1 diff --git a/src/app.js b/src/app.js index 1d3ae47..c38c03d 100644 --- a/src/app.js +++ b/src/app.js @@ -13,6 +13,7 @@ const JobProcessorService = require('./services/JobProcessorService') const JobCandidateProcessorService = require('./services/JobCandidateProcessorService') const ResourceBookingProcessorService = require('./services/ResourceBookingProcessorService') const WorkPeriodProcessorService = require('./services/WorkPeriodProcessorService') +const WorkPeriodPaymentProcessorService = require('./services/WorkPeriodPaymentProcessorService') const Mutex = require('async-mutex').Mutex const events = require('events') @@ -43,7 +44,10 @@ const topicServiceMapping = { // work period [config.topics.TAAS_WORK_PERIOD_CREATE_TOPIC]: WorkPeriodProcessorService.processCreate, [config.topics.TAAS_WORK_PERIOD_UPDATE_TOPIC]: WorkPeriodProcessorService.processUpdate, - [config.topics.TAAS_WORK_PERIOD_DELETE_TOPIC]: WorkPeriodProcessorService.processDelete + [config.topics.TAAS_WORK_PERIOD_DELETE_TOPIC]: WorkPeriodProcessorService.processDelete, + // work period payment + [config.topics.TAAS_WORK_PERIOD_PAYMENT_CREATE_TOPIC]: WorkPeriodPaymentProcessorService.processCreate, + [config.topics.TAAS_WORK_PERIOD_PAYMENT_UPDATE_TOPIC]: WorkPeriodPaymentProcessorService.processUpdate } // Start kafka consumer diff --git a/src/bootstrap.js b/src/bootstrap.js index 0f58277..6c9c001 100644 --- a/src/bootstrap.js +++ b/src/bootstrap.js @@ -10,6 +10,7 @@ Joi.jobCandidateStatus = () => Joi.string().valid('open', 'selected', 'shortlist Joi.workload = () => Joi.string().valid('full-time', 'fractional') Joi.title = () => Joi.string().max(128) Joi.paymentStatus = () => Joi.string().valid('pending', 'partially-completed', 'completed', 'cancelled') +Joi.workPeriodPaymentStatus = () => Joi.string().valid('completed', 'cancelled') // Empty string is not allowed by Joi by default and must be enabled with allow(''). // See https://joi.dev/api/?v=17.3.0#string fro details why it's like this. // In many cases we would like to allow empty string to make it easier to create UI for editing data. diff --git a/src/scripts/createIndex.js b/src/scripts/createIndex.js index 095633e..fad96d3 100644 --- a/src/scripts/createIndex.js +++ b/src/scripts/createIndex.js @@ -91,6 +91,19 @@ async function createIndex () { memberRate: { type: 'float' }, customerRate: { type: 'float' }, paymentStatus: { type: 'keyword' }, + payments: { + type: 'nested', + properties: { + workPeriodId: { type: 'keyword' }, + challengeId: { type: 'keyword' }, + amount: { type: 'float' }, + status: { type: 'keyword' }, + createdAt: { type: 'date' }, + createdBy: { type: 'keyword' }, + updatedAt: { type: 'date' }, + updatedBy: { type: 'keyword' } + } + }, createdAt: { type: 'date' }, createdBy: { type: 'keyword' }, updatedAt: { type: 'date' }, diff --git a/src/services/WorkPeriodPaymentProcessorService.js b/src/services/WorkPeriodPaymentProcessorService.js new file mode 100644 index 0000000..45fd6c5 --- /dev/null +++ b/src/services/WorkPeriodPaymentProcessorService.js @@ -0,0 +1,134 @@ +/** + * WorkPeriodPayment Processor Service + */ + +const Joi = require('@hapi/joi') +const config = require('config') +const _ = require('lodash') +const logger = require('../common/logger') +const helper = require('../common/helper') +const constants = require('../common/constants') + +const esClient = helper.getESClient() + +/** + * Process create entity message + * @param {Object} message the kafka message + * @param {String} transactionId + */ +async function processCreate (message, transactionId) { + const data = message.payload + const workPeriod = await esClient.get({ + index: config.get('esConfig.ES_INDEX_WORK_PERIOD'), + id: data.workPeriodId + }) + const payments = _.isArray(workPeriod.body._source.payments) ? workPeriod.body._source.payments : [] + payments.push(data) + + return esClient.update({ + index: config.get('esConfig.ES_INDEX_WORK_PERIOD'), + id: data.workPeriodId, + transactionId, + body: { + doc: _.assign(workPeriod.body._source, { payments }) + }, + refresh: constants.esRefreshOption + }) +} + +processCreate.schema = { + message: Joi.object().keys({ + topic: Joi.string().required(), + originator: Joi.string().required(), + timestamp: Joi.date().required(), + 'mime-type': Joi.string().required(), + payload: Joi.object().keys({ + id: Joi.string().uuid().required(), + workPeriodId: Joi.string().uuid().required(), + challengeId: Joi.string().uuid().required(), + amount: Joi.number().greater(0).allow(null), + status: Joi.workPeriodPaymentStatus().required(), + createdAt: Joi.date().required(), + createdBy: Joi.string().uuid().required(), + updatedAt: Joi.date().allow(null), + updatedBy: Joi.string().uuid().allow(null) + }).required() + }).required(), + transactionId: Joi.string().required() +} + +/** + * Process update entity message + * @param {Object} message the kafka message + * @param {String} transactionId + */ +async function processUpdate (message, transactionId) { + const data = message.payload + let workPeriod = await esClient.search({ + index: config.get('esConfig.ES_INDEX_WORK_PERIOD'), + body: { + query: { + nested: { + path: 'payments', + query: { + match: { 'payments.id': data.id } + } + } + } + } + }) + let payments + // if WorkPeriodPayment's workPeriodId changed then it must be deleted from the old WorkPeriod + // and added to the new WorkPeriod + if (workPeriod.body.hits.hits[0]._source.id !== data.workPeriodId) { + payments = _.filter(workPeriod.body.hits.hits[0]._source.payments, (payment) => payment.id !== data.id) + await esClient.update({ + index: config.get('esConfig.ES_INDEX_WORK_PERIOD'), + id: workPeriod.body.hits.hits[0]._source.id, + transactionId, + body: { + doc: _.assign(workPeriod.body.hits.hits[0]._source, { payments }) + } + }) + workPeriod = await esClient.get({ + index: config.get('esConfig.ES_INDEX_WORK_PERIOD'), + id: data.workPeriodId + }) + payments = _.isArray(workPeriod.body._source.payments) ? workPeriod.body._source.payments : [] + payments.push(data) + return esClient.update({ + index: config.get('esConfig.ES_INDEX_WORK_PERIOD'), + id: data.workPeriodId, + transactionId, + body: { + doc: _.assign(workPeriod.body._source, { payments }) + } + }) + } + + payments = _.map(workPeriod.body.hits.hits[0]._source.payments, (payment) => { + if (payment.id === data.id) { + return _.assign(payment, data) + } + return payment + }) + + return esClient.update({ + index: config.get('esConfig.ES_INDEX_WORK_PERIOD'), + id: data.workPeriodId, + transactionId, + body: { + doc: _.assign(workPeriod.body.hits.hits[0]._source, { payments }) + }, + refresh: constants.esRefreshOption + }) +} + +processUpdate.schema = processCreate.schema + +module.exports = { + processCreate, + processUpdate +} + +logger.buildService(module.exports, 'WorkPeriodPaymentProcessorService') diff --git a/test/messages/taas.workperiodpayment.create.event.json b/test/messages/taas.workperiodpayment.create.event.json new file mode 100644 index 0000000..25ab9cc --- /dev/null +++ b/test/messages/taas.workperiodpayment.create.event.json @@ -0,0 +1 @@ +{"topic":"taas.workperiodpayment.create","originator":"taas-api","timestamp":"2021-04-09T20:10:33.770Z","mime-type":"application/json","payload":{"challengeId":"00000000-0000-0000-0000-000000000000","workPeriodId":"140b7407-540d-40c3-ad23-905d932aa9c8","amount":600,"status":"completed","id":"09c80ee6-21be-45a4-9c3c-7ec4c75ece79","createdBy":"57646ff9-1cd3-4d3c-88ba-eb09a395366c","updatedAt":"2021-04-09T20:10:33.755Z","createdAt":"2021-04-09T20:10:33.755Z","updatedBy":null}} \ No newline at end of file diff --git a/test/messages/taas.workperiodpayment.update.event.json b/test/messages/taas.workperiodpayment.update.event.json new file mode 100644 index 0000000..66e5bce --- /dev/null +++ b/test/messages/taas.workperiodpayment.update.event.json @@ -0,0 +1 @@ +{"topic":"taas.workperiodpayment.update","originator":"taas-api","timestamp":"2021-04-09T20:12:26.994Z","mime-type":"application/json","payload":{"id":"09c80ee6-21be-45a4-9c3c-7ec4c75ece79","workPeriodId":"140b7407-540d-40c3-ad23-905d932aa9c8","challengeId":"00000000-0000-0000-0000-000000000000","amount":1600,"status":"completed","createdBy":"57646ff9-1cd3-4d3c-88ba-eb09a395366c","updatedBy":"57646ff9-1cd3-4d3c-88ba-eb09a395366c","createdAt":"2021-04-09T20:10:33.755Z","updatedAt":"2021-04-09T20:12:26.966Z"}} \ No newline at end of file From e965836c925a62735225c322f5f4effbd52316d9 Mon Sep 17 00:00:00 2001 From: Maksym Mykhailenko Date: Sat, 17 Apr 2021 11:41:38 +0300 Subject: [PATCH 02/55] fix: unit test --- test/messages/taas.resourcebooking.create.event.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/messages/taas.resourcebooking.create.event.json b/test/messages/taas.resourcebooking.create.event.json index 1a358ba..92ec817 100644 --- a/test/messages/taas.resourcebooking.create.event.json +++ b/test/messages/taas.resourcebooking.create.event.json @@ -1 +1 @@ -{"topic":"taas.resourcebooking.create","originator":"taas-api","timestamp":"2020-11-05T19:00:25.038Z","mime-type":"application/json","payload":{"projectId":21,"userId":"a55fe1bc-1754-45fa-9adc-cf3d6d7c377a","jobId":"ffbc24f7-301e-48d3-bf01-c056916056a2","startDate":"2020-09-27T04:17:23.131Z","endDate":"2020-09-27T04:17:23.131Z","memberRate":13.23,"customerRate":13,"rateType":"hourly","id":"60d97713-8621-476e-b006-7cb9589c7777","createdAt":"2020-11-05T19:00:23.036Z","createdBy":"a55fe1bc-1754-45fa-9adc-cf3d6d7c377a","status":"sourcing"}} \ No newline at end of file +{"topic":"taas.resourcebooking.create","originator":"taas-api","timestamp":"2020-11-05T19:00:25.038Z","mime-type":"application/json","payload":{"projectId":21,"userId":"a55fe1bc-1754-45fa-9adc-cf3d6d7c377a","jobId":"ffbc24f7-301e-48d3-bf01-c056916056a2","startDate":"2020-09-27T04:17:23.131Z","endDate":"2020-09-27T04:17:23.131Z","memberRate":13.23,"customerRate":13,"rateType":"hourly","id":"60d97713-8621-476e-b006-7cb9589c7777","createdAt":"2020-11-05T19:00:23.036Z","createdBy":"a55fe1bc-1754-45fa-9adc-cf3d6d7c377a","status":"assigned"}} \ No newline at end of file From c9661386fa06a8bf0074d8e78bd5caee6d4e3823 Mon Sep 17 00:00:00 2001 From: narekcat Date: Mon, 19 Apr 2021 01:05:05 +0400 Subject: [PATCH 03/55] Final fixes for adding work period payments. --- .../WorkPeriodPaymentProcessorService.js | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/services/WorkPeriodPaymentProcessorService.js b/src/services/WorkPeriodPaymentProcessorService.js index 45fd6c5..1f6cc54 100644 --- a/src/services/WorkPeriodPaymentProcessorService.js +++ b/src/services/WorkPeriodPaymentProcessorService.js @@ -18,19 +18,19 @@ const esClient = helper.getESClient() */ async function processCreate (message, transactionId) { const data = message.payload - const workPeriod = await esClient.get({ + const workPeriod = await esClient.getExtra({ index: config.get('esConfig.ES_INDEX_WORK_PERIOD'), id: data.workPeriodId }) - const payments = _.isArray(workPeriod.body._source.payments) ? workPeriod.body._source.payments : [] + const payments = _.isArray(workPeriod.body.payments) ? workPeriod.body.payments : [] payments.push(data) - return esClient.update({ + return esClient.updateExtra({ index: config.get('esConfig.ES_INDEX_WORK_PERIOD'), id: data.workPeriodId, transactionId, body: { - doc: _.assign(workPeriod.body._source, { payments }) + doc: _.assign(workPeriod.body, { payments }) }, refresh: constants.esRefreshOption }) @@ -77,12 +77,15 @@ async function processUpdate (message, transactionId) { } } }) + if (!workPeriod.body.hits.total.value) { + throw new Error(`id: ${data.id} "WorkPeriodPayments" not found`) + } let payments // if WorkPeriodPayment's workPeriodId changed then it must be deleted from the old WorkPeriod // and added to the new WorkPeriod if (workPeriod.body.hits.hits[0]._source.id !== data.workPeriodId) { payments = _.filter(workPeriod.body.hits.hits[0]._source.payments, (payment) => payment.id !== data.id) - await esClient.update({ + await esClient.updateExtra({ index: config.get('esConfig.ES_INDEX_WORK_PERIOD'), id: workPeriod.body.hits.hits[0]._source.id, transactionId, @@ -90,18 +93,18 @@ async function processUpdate (message, transactionId) { doc: _.assign(workPeriod.body.hits.hits[0]._source, { payments }) } }) - workPeriod = await esClient.get({ + workPeriod = await esClient.getExtra({ index: config.get('esConfig.ES_INDEX_WORK_PERIOD'), id: data.workPeriodId }) - payments = _.isArray(workPeriod.body._source.payments) ? workPeriod.body._source.payments : [] + payments = _.isArray(workPeriod.body.payments) ? workPeriod.body.payments : [] payments.push(data) - return esClient.update({ + return esClient.updateExtra({ index: config.get('esConfig.ES_INDEX_WORK_PERIOD'), id: data.workPeriodId, transactionId, body: { - doc: _.assign(workPeriod.body._source, { payments }) + doc: _.assign(workPeriod.body, { payments }) } }) } @@ -113,7 +116,7 @@ async function processUpdate (message, transactionId) { return payment }) - return esClient.update({ + return esClient.updateExtra({ index: config.get('esConfig.ES_INDEX_WORK_PERIOD'), id: data.workPeriodId, transactionId, From 130d37aac484fa1d0f8a67c5e0d6d12d7a795574 Mon Sep 17 00:00:00 2001 From: xxcxy Date: Wed, 21 Apr 2021 13:59:43 +0800 Subject: [PATCH 04/55] part1 --- README.md | 1 + config/default.js | 4 +- local/docker-compose.yml | 2 +- src/app.js | 5 +- src/bootstrap.js | 5 ++ src/common/constants.js | 5 ++ src/scripts/createIndex.js | 16 +++++ src/services/InterviewProcessorService.js | 79 +++++++++++++++++++++++ 8 files changed, 114 insertions(+), 3 deletions(-) create mode 100644 src/services/InterviewProcessorService.js diff --git a/README.md b/README.md index d8409ab..07e97c6 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ The following parameters can be set in config files or in env variables: - `topics.TAAS_WORK_PERIOD_CREATE_TOPIC`: the create work period entity Kafka message topic - `topics.TAAS_WORK_PERIOD_UPDATE_TOPIC`: the update work period entity Kafka message topic - `topics.TAAS_WORK_PERIOD_DELETE_TOPIC`: the delete work period entity Kafka message topic +- `topics.TAAS_INTERVIEW_REQUEST_TOPIC`: the request interview entity Kafka message topic - `esConfig.HOST`: Elasticsearch host - `esConfig.AWS_REGION`: The Amazon region to use when using AWS Elasticsearch service - `esConfig.ELASTICCLOUD.id`: The elastic cloud id, if your elasticsearch instance is hosted on elastic cloud. DO NOT provide a value for ES_HOST if you are using this diff --git a/config/default.js b/config/default.js index 446ab4e..478c719 100644 --- a/config/default.js +++ b/config/default.js @@ -31,7 +31,9 @@ module.exports = { // topics for work period service TAAS_WORK_PERIOD_CREATE_TOPIC: process.env.TAAS_WORK_PERIOD_CREATE_TOPIC || 'taas.workperiod.create', TAAS_WORK_PERIOD_UPDATE_TOPIC: process.env.TAAS_WORK_PERIOD_UPDATE_TOPIC || 'taas.workperiod.update', - TAAS_WORK_PERIOD_DELETE_TOPIC: process.env.TAAS_WORK_PERIOD_DELETE_TOPIC || 'taas.workperiod.delete' + TAAS_WORK_PERIOD_DELETE_TOPIC: process.env.TAAS_WORK_PERIOD_DELETE_TOPIC || 'taas.workperiod.delete', + // topics for interview service + TAAS_INTERVIEW_REQUEST_TOPIC: process.env.TAAS_INTERVIEW_REQUEST_TOPIC || 'taas.interview.request' }, esConfig: { diff --git a/local/docker-compose.yml b/local/docker-compose.yml index 5d2d803..5f355f3 100644 --- a/local/docker-compose.yml +++ b/local/docker-compose.yml @@ -12,7 +12,7 @@ services: - "9092:9092" environment: KAFKA_ADVERTISED_HOST_NAME: localhost - KAFKA_CREATE_TOPICS: "taas.job.create:1:1,taas.jobcandidate.create:1:1,taas.resourcebooking.create:1:1,taas.job.update:1:1,taas.jobcandidate.update:1:1,taas.resourcebooking.update:1:1,taas.job.delete:1:1,taas.jobcandidate.delete:1:1,taas.resourcebooking.delete:1:1" + KAFKA_CREATE_TOPICS: "taas.job.create:1:1,taas.jobcandidate.create:1:1,taas.resourcebooking.create:1:1,taas.interview.request:1:1,taas.job.update:1:1,taas.jobcandidate.update:1:1,taas.resourcebooking.update:1:1,taas.job.delete:1:1,taas.jobcandidate.delete:1:1,taas.resourcebooking.delete:1:1" KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 esearch: image: elasticsearch:7.7.1 diff --git a/src/app.js b/src/app.js index 1d3ae47..b4ee7bb 100644 --- a/src/app.js +++ b/src/app.js @@ -13,6 +13,7 @@ const JobProcessorService = require('./services/JobProcessorService') const JobCandidateProcessorService = require('./services/JobCandidateProcessorService') const ResourceBookingProcessorService = require('./services/ResourceBookingProcessorService') const WorkPeriodProcessorService = require('./services/WorkPeriodProcessorService') +const InterviewProcessorService = require('./services/InterviewProcessorService') const Mutex = require('async-mutex').Mutex const events = require('events') @@ -43,7 +44,9 @@ const topicServiceMapping = { // work period [config.topics.TAAS_WORK_PERIOD_CREATE_TOPIC]: WorkPeriodProcessorService.processCreate, [config.topics.TAAS_WORK_PERIOD_UPDATE_TOPIC]: WorkPeriodProcessorService.processUpdate, - [config.topics.TAAS_WORK_PERIOD_DELETE_TOPIC]: WorkPeriodProcessorService.processDelete + [config.topics.TAAS_WORK_PERIOD_DELETE_TOPIC]: WorkPeriodProcessorService.processDelete, + // interview + [config.topics.TAAS_INTERVIEW_REQUEST_TOPIC]: InterviewProcessorService.processRequestInterview } // Start kafka consumer diff --git a/src/bootstrap.js b/src/bootstrap.js index d25de1a..bb0761a 100644 --- a/src/bootstrap.js +++ b/src/bootstrap.js @@ -1,7 +1,11 @@ const Joi = require('@hapi/joi') const config = require('config') +const _ = require('lodash') +const { Interview } = require('../src/common/constants') const constants = require('./common/constants') +const allowedInterviewStatuses = _.values(Interview.Status) + global.Promise = require('bluebird') Joi.rateType = () => Joi.string().valid('hourly', 'daily', 'weekly', 'monthly') @@ -10,6 +14,7 @@ Joi.jobCandidateStatus = () => Joi.string().valid('open', 'selected', 'shortlist Joi.workload = () => Joi.string().valid('full-time', 'fractional') Joi.title = () => Joi.string().max(128) Joi.paymentStatus = () => Joi.string().valid('pending', 'partially-completed', 'completed', 'cancelled') +Joi.interviewStatus = () => Joi.string().valid(...allowedInterviewStatuses) // Empty string is not allowed by Joi by default and must be enabled with allow(''). // See https://joi.dev/api/?v=17.3.0#string fro details why it's like this. // In many cases we would like to allow empty string to make it easier to create UI for editing data. diff --git a/src/common/constants.js b/src/common/constants.js index 62d8720..0d47cb4 100644 --- a/src/common/constants.js +++ b/src/common/constants.js @@ -16,5 +16,10 @@ module.exports = { JobCandidateCreate: 'jobcandidate:create', JobCandidateUpdate: 'jobcandidate:update' } + }, + Interview: { + Status: { + Requested: 'Requested' + } } } diff --git a/src/scripts/createIndex.js b/src/scripts/createIndex.js index 095633e..6ba13e0 100644 --- a/src/scripts/createIndex.js +++ b/src/scripts/createIndex.js @@ -46,6 +46,22 @@ async function createIndex () { status: { type: 'keyword' }, externalId: { type: 'keyword' }, resume: { type: 'text' }, + interviews: { + type: 'nested', + properties: { + id: { type: 'keyword' }, + jobCandidateId: { type: 'keyword' }, + googleCalendarId: { type: 'keyword' }, + customMessage: { type: 'text' }, + xaiTemplate: { type: 'keyword' }, + round: { type: 'integer' }, + status: { type: 'keyword' }, + createdAt: { type: 'date' }, + createdBy: { type: 'keyword' }, + updatedAt: { type: 'date' }, + updatedBy: { type: 'keyword' } + } + }, createdAt: { type: 'date' }, createdBy: { type: 'keyword' }, updatedAt: { type: 'date' }, diff --git a/src/services/InterviewProcessorService.js b/src/services/InterviewProcessorService.js new file mode 100644 index 0000000..d02fe07 --- /dev/null +++ b/src/services/InterviewProcessorService.js @@ -0,0 +1,79 @@ +/** + * Interview Processor Service + */ + +const Joi = require('@hapi/joi') +const logger = require('../common/logger') +const helper = require('../common/helper') +const constants = require('../common/constants') +const config = require('config') + +const esClient = helper.getESClient() + +/** + * Updates jobCandidate via a painless script + * + * @param {String} jobCandidateId job candidate id + * @param {String} script script definition + * @param {String} transactionId transaction id + */ +async function updateJobCandidateViaScript (jobCandidateId, script, transactionId) { + await esClient.updateExtra({ + index: config.get('esConfig.ES_INDEX_JOB_CANDIDATE'), + id: jobCandidateId, + transactionId, + body: { script }, + refresh: constants.esRefreshOption + }) +} + +/** + * Process request interview entity message. + * Creates an interview record under jobCandidate. + * + * @param {Object} message the kafka message + * @param {String} transactionId + */ +async function processRequestInterview (message, transactionId) { + const interview = message.payload + // add interview in collection if there's already an existing collection + // or initiate a new one with this interview + const script = { + source: ` + ctx._source.containsKey("interviews") + ? ctx._source.interviews.add(params.interview) + : ctx._source.interviews = [params.interview] + `, + params: { interview } + } + await updateJobCandidateViaScript(interview.jobCandidateId, script, transactionId) +} + +processRequestInterview.schema = { + message: Joi.object().keys({ + topic: Joi.string().required(), + originator: Joi.string().required(), + timestamp: Joi.date().required(), + 'mime-type': Joi.string().required(), + payload: Joi.object().keys({ + id: Joi.string().uuid().required(), + jobCandidateId: Joi.string().uuid().required(), + googleCalendarId: Joi.string().allow(null), + customMessage: Joi.string().allow(null), + xaiTemplate: Joi.string().required(), + round: Joi.number().integer().positive().required(), + status: Joi.interviewStatus().required(), + createdAt: Joi.date().required(), + createdBy: Joi.string().uuid().required(), + updatedAt: Joi.date().allow(null), + updatedBy: Joi.string().uuid().allow(null) + }).required() + }).required(), + transactionId: Joi.string().required() +} + +module.exports = { + processRequestInterview +} + +logger.buildService(module.exports, 'InterviewProcessorService') From 47ae415d081b8e08c320d8ab078e67dc7aa0adab Mon Sep 17 00:00:00 2001 From: xxcxy Date: Wed, 21 Apr 2021 14:00:37 +0800 Subject: [PATCH 05/55] part2 --- README.md | 1 + config/default.js | 3 +- local/docker-compose.yml | 2 +- src/app.js | 3 +- src/common/constants.js | 7 ++- src/scripts/createIndex.js | 2 + src/services/InterviewProcessorService.js | 65 ++++++++++++++++++++++- 7 files changed, 77 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 07e97c6..519f60c 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ The following parameters can be set in config files or in env variables: - `topics.TAAS_WORK_PERIOD_UPDATE_TOPIC`: the update work period entity Kafka message topic - `topics.TAAS_WORK_PERIOD_DELETE_TOPIC`: the delete work period entity Kafka message topic - `topics.TAAS_INTERVIEW_REQUEST_TOPIC`: the request interview entity Kafka message topic +- `topics.TAAS_INTERVIEW_UPDATE_TOPIC`: the update interview entity Kafka message topic - `esConfig.HOST`: Elasticsearch host - `esConfig.AWS_REGION`: The Amazon region to use when using AWS Elasticsearch service - `esConfig.ELASTICCLOUD.id`: The elastic cloud id, if your elasticsearch instance is hosted on elastic cloud. DO NOT provide a value for ES_HOST if you are using this diff --git a/config/default.js b/config/default.js index 478c719..9e53a7e 100644 --- a/config/default.js +++ b/config/default.js @@ -33,7 +33,8 @@ module.exports = { TAAS_WORK_PERIOD_UPDATE_TOPIC: process.env.TAAS_WORK_PERIOD_UPDATE_TOPIC || 'taas.workperiod.update', TAAS_WORK_PERIOD_DELETE_TOPIC: process.env.TAAS_WORK_PERIOD_DELETE_TOPIC || 'taas.workperiod.delete', // topics for interview service - TAAS_INTERVIEW_REQUEST_TOPIC: process.env.TAAS_INTERVIEW_REQUEST_TOPIC || 'taas.interview.request' + TAAS_INTERVIEW_REQUEST_TOPIC: process.env.TAAS_INTERVIEW_REQUEST_TOPIC || 'taas.interview.requested', + TAAS_INTERVIEW_UPDATE_TOPIC: process.env.TAAS_INTERVIEW_UPDATE_TOPIC || 'taas.interview.update' }, esConfig: { diff --git a/local/docker-compose.yml b/local/docker-compose.yml index 5f355f3..d5c65ac 100644 --- a/local/docker-compose.yml +++ b/local/docker-compose.yml @@ -12,7 +12,7 @@ services: - "9092:9092" environment: KAFKA_ADVERTISED_HOST_NAME: localhost - KAFKA_CREATE_TOPICS: "taas.job.create:1:1,taas.jobcandidate.create:1:1,taas.resourcebooking.create:1:1,taas.interview.request:1:1,taas.job.update:1:1,taas.jobcandidate.update:1:1,taas.resourcebooking.update:1:1,taas.job.delete:1:1,taas.jobcandidate.delete:1:1,taas.resourcebooking.delete:1:1" + KAFKA_CREATE_TOPICS: "taas.job.create:1:1,taas.jobcandidate.create:1:1,taas.resourcebooking.create:1:1,taas.interview.requested:1:1,taas.interview.update:1:1,taas.job.update:1:1,taas.jobcandidate.update:1:1,taas.resourcebooking.update:1:1,taas.job.delete:1:1,taas.jobcandidate.delete:1:1,taas.resourcebooking.delete:1:1" KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 esearch: image: elasticsearch:7.7.1 diff --git a/src/app.js b/src/app.js index b4ee7bb..471acb5 100644 --- a/src/app.js +++ b/src/app.js @@ -46,7 +46,8 @@ const topicServiceMapping = { [config.topics.TAAS_WORK_PERIOD_UPDATE_TOPIC]: WorkPeriodProcessorService.processUpdate, [config.topics.TAAS_WORK_PERIOD_DELETE_TOPIC]: WorkPeriodProcessorService.processDelete, // interview - [config.topics.TAAS_INTERVIEW_REQUEST_TOPIC]: InterviewProcessorService.processRequestInterview + [config.topics.TAAS_INTERVIEW_REQUEST_TOPIC]: InterviewProcessorService.processRequestInterview, + [config.topics.TAAS_INTERVIEW_UPDATE_TOPIC]: InterviewProcessorService.processUpdateInterview } // Start kafka consumer diff --git a/src/common/constants.js b/src/common/constants.js index 0d47cb4..d7f16ec 100644 --- a/src/common/constants.js +++ b/src/common/constants.js @@ -19,7 +19,12 @@ module.exports = { }, Interview: { Status: { - Requested: 'Requested' + Scheduling: 'Scheduling', + Scheduled: 'Scheduled', + RequestedForReschedule: 'Requested for reschedule', + Rescheduled: 'Rescheduled', + Completed: 'Completed', + Cancelled: 'Cancelled' } } } diff --git a/src/scripts/createIndex.js b/src/scripts/createIndex.js index 6ba13e0..8303f3f 100644 --- a/src/scripts/createIndex.js +++ b/src/scripts/createIndex.js @@ -52,6 +52,8 @@ async function createIndex () { id: { type: 'keyword' }, jobCandidateId: { type: 'keyword' }, googleCalendarId: { type: 'keyword' }, + startTimestamp: { type: 'date' }, + attendeesList: { type: 'keyword' }, customMessage: { type: 'text' }, xaiTemplate: { type: 'keyword' }, round: { type: 'integer' }, diff --git a/src/services/InterviewProcessorService.js b/src/services/InterviewProcessorService.js index d02fe07..15af5ef 100644 --- a/src/services/InterviewProcessorService.js +++ b/src/services/InterviewProcessorService.js @@ -2,6 +2,7 @@ * Interview Processor Service */ +const _ = require('lodash') const Joi = require('@hapi/joi') const logger = require('../common/logger') const helper = require('../common/helper') @@ -66,14 +67,74 @@ processRequestInterview.schema = { createdAt: Joi.date().required(), createdBy: Joi.string().uuid().required(), updatedAt: Joi.date().allow(null), - updatedBy: Joi.string().uuid().allow(null) + updatedBy: Joi.string().uuid().allow(null), + attendeesList: Joi.array().items(Joi.string().email()).allow(null), + startTimestamp: Joi.date().allow(null) + }).required() + }).required(), + transactionId: Joi.string().required() +} + +/** + * Process update interview entity message. + * Update an interview record under jobCandidate. + * + * @param {Object} message the kafka message + * @param {String} transactionId + */ +async function processUpdateInterview (message, transactionId) { + const data = message.payload + const { body: jobCandidate } = await esClient.getExtra({ + index: config.get('esConfig.ES_INDEX_JOB_CANDIDATE'), + id: data.jobCandidateId + }) + const interviews = jobCandidate.interviews || [] + const index = _.findIndex(interviews, ['id', data.id]) + if (index === -1) { + interviews.push(data) + } else { + interviews.splice(index, 1, data) + } + jobCandidate.interviews = interviews + await esClient.updateExtra({ + index: config.get('esConfig.ES_INDEX_JOB_CANDIDATE'), + id: data.jobCandidateId, + transactionId, + body: { + doc: jobCandidate + }, + refresh: constants.esRefreshOption + }) +} + +processUpdateInterview.schema = { + message: Joi.object().keys({ + topic: Joi.string().required(), + originator: Joi.string().required(), + timestamp: Joi.date().required(), + 'mime-type': Joi.string().required(), + payload: Joi.object().keys({ + id: Joi.string().uuid().required(), + jobCandidateId: Joi.string().uuid().required(), + googleCalendarId: Joi.string().allow(null), + customMessage: Joi.string().allow(null), + xaiTemplate: Joi.string().required(), + round: Joi.number().integer().positive().required(), + status: Joi.interviewStatus().required(), + createdAt: Joi.date().required(), + createdBy: Joi.string().uuid().required(), + updatedAt: Joi.date().required(), + updatedBy: Joi.string().uuid().required(), + attendeesList: Joi.array().items(Joi.string().email()).allow(null), + startTimestamp: Joi.date().allow(null) }).required() }).required(), transactionId: Joi.string().required() } module.exports = { - processRequestInterview + processRequestInterview, + processUpdateInterview } logger.buildService(module.exports, 'InterviewProcessorService') From 4a6b647c04ce4012e635adfded408d72ceff4806 Mon Sep 17 00:00:00 2001 From: nkumar-topcoder <33625707+nkumar-topcoder@users.noreply.github.com> Date: Wed, 21 Apr 2021 13:58:40 +0530 Subject: [PATCH 06/55] Revert "Feature/interview scheduler" --- README.md | 2 - config/default.js | 3 - local/docker-compose.yml | 2 +- src/app.js | 4 - src/bootstrap.js | 5 - src/common/constants.js | 10 -- src/scripts/createIndex.js | 18 --- src/services/InterviewProcessorService.js | 140 ---------------------- 8 files changed, 1 insertion(+), 183 deletions(-) delete mode 100644 src/services/InterviewProcessorService.js diff --git a/README.md b/README.md index 9cf88c6..045684f 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,6 @@ The following parameters can be set in config files or in env variables: - `topics.TAAS_WORK_PERIOD_CREATE_TOPIC`: the create work period entity Kafka message topic - `topics.TAAS_WORK_PERIOD_UPDATE_TOPIC`: the update work period entity Kafka message topic - `topics.TAAS_WORK_PERIOD_DELETE_TOPIC`: the delete work period entity Kafka message topic -- `topics.TAAS_INTERVIEW_REQUEST_TOPIC`: the request interview entity Kafka message topic -- `topics.TAAS_INTERVIEW_UPDATE_TOPIC`: the update interview entity Kafka message topic - `topics.TAAS_WORK_PERIOD_PAYMENT_CREATE_TOPIC`: the create work period payment entity Kafka message topic - `topics.TAAS_WORK_PERIOD_PAYMENT_UPDATE_TOPIC`: the update work period payment entity Kafka message topic - `esConfig.HOST`: Elasticsearch host diff --git a/config/default.js b/config/default.js index d1f2fae..7a8dbf8 100644 --- a/config/default.js +++ b/config/default.js @@ -32,9 +32,6 @@ module.exports = { TAAS_WORK_PERIOD_CREATE_TOPIC: process.env.TAAS_WORK_PERIOD_CREATE_TOPIC || 'taas.workperiod.create', TAAS_WORK_PERIOD_UPDATE_TOPIC: process.env.TAAS_WORK_PERIOD_UPDATE_TOPIC || 'taas.workperiod.update', TAAS_WORK_PERIOD_DELETE_TOPIC: process.env.TAAS_WORK_PERIOD_DELETE_TOPIC || 'taas.workperiod.delete', - // topics for interview service - TAAS_INTERVIEW_REQUEST_TOPIC: process.env.TAAS_INTERVIEW_REQUEST_TOPIC || 'taas.interview.requested', - TAAS_INTERVIEW_UPDATE_TOPIC: process.env.TAAS_INTERVIEW_UPDATE_TOPIC || 'taas.interview.update', // topics for work period payment service TAAS_WORK_PERIOD_PAYMENT_CREATE_TOPIC: process.env.TAAS_WORK_PERIOD_PAYMENT_CREATE_TOPIC || 'taas.workperiodpayment.create', TAAS_WORK_PERIOD_PAYMENT_UPDATE_TOPIC: process.env.TAAS_WORK_PERIOD_PAYMENT_UPDATE_TOPIC || 'taas.workperiodpayment.update' diff --git a/local/docker-compose.yml b/local/docker-compose.yml index 3d01ede..35e9486 100644 --- a/local/docker-compose.yml +++ b/local/docker-compose.yml @@ -12,7 +12,7 @@ services: - "9092:9092" environment: KAFKA_ADVERTISED_HOST_NAME: localhost - KAFKA_CREATE_TOPICS: "taas.job.create:1:1,taas.jobcandidate.create:1:1,taas.resourcebooking.create:1:1,taas.interview.requested:1:1,taas.interview.update:1:1,taas.workperiod.create:1:1,taas.workperiodpayment.create:1:1,taas.job.update:1:1,taas.jobcandidate.update:1:1,taas.resourcebooking.update:1:1,taas.workperiod.update:1:1,taas.workperiodpayment.update:1:1,taas.job.delete:1:1,taas.jobcandidate.delete:1:1,taas.resourcebooking.delete:1:1,taas.workperiod.delete:1:1" + KAFKA_CREATE_TOPICS: "taas.job.create:1:1,taas.jobcandidate.create:1:1,taas.resourcebooking.create:1:1,taas.workperiod.create:1:1,taas.workperiodpayment.create:1:1,taas.job.update:1:1,taas.jobcandidate.update:1:1,taas.resourcebooking.update:1:1,taas.workperiod.update:1:1,taas.workperiodpayment.update:1:1,taas.job.delete:1:1,taas.jobcandidate.delete:1:1,taas.resourcebooking.delete:1:1,taas.workperiod.delete:1:1" KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 esearch: image: elasticsearch:7.7.1 diff --git a/src/app.js b/src/app.js index e0f5a22..c38c03d 100644 --- a/src/app.js +++ b/src/app.js @@ -13,7 +13,6 @@ const JobProcessorService = require('./services/JobProcessorService') const JobCandidateProcessorService = require('./services/JobCandidateProcessorService') const ResourceBookingProcessorService = require('./services/ResourceBookingProcessorService') const WorkPeriodProcessorService = require('./services/WorkPeriodProcessorService') -const InterviewProcessorService = require('./services/InterviewProcessorService') const WorkPeriodPaymentProcessorService = require('./services/WorkPeriodPaymentProcessorService') const Mutex = require('async-mutex').Mutex const events = require('events') @@ -46,9 +45,6 @@ const topicServiceMapping = { [config.topics.TAAS_WORK_PERIOD_CREATE_TOPIC]: WorkPeriodProcessorService.processCreate, [config.topics.TAAS_WORK_PERIOD_UPDATE_TOPIC]: WorkPeriodProcessorService.processUpdate, [config.topics.TAAS_WORK_PERIOD_DELETE_TOPIC]: WorkPeriodProcessorService.processDelete, - // interview - [config.topics.TAAS_INTERVIEW_REQUEST_TOPIC]: InterviewProcessorService.processRequestInterview, - [config.topics.TAAS_INTERVIEW_UPDATE_TOPIC]: InterviewProcessorService.processUpdateInterview, // work period payment [config.topics.TAAS_WORK_PERIOD_PAYMENT_CREATE_TOPIC]: WorkPeriodPaymentProcessorService.processCreate, [config.topics.TAAS_WORK_PERIOD_PAYMENT_UPDATE_TOPIC]: WorkPeriodPaymentProcessorService.processUpdate diff --git a/src/bootstrap.js b/src/bootstrap.js index 8f23fe3..794ab83 100644 --- a/src/bootstrap.js +++ b/src/bootstrap.js @@ -1,11 +1,7 @@ const Joi = require('@hapi/joi') const config = require('config') -const _ = require('lodash') -const { Interview } = require('../src/common/constants') const constants = require('./common/constants') -const allowedInterviewStatuses = _.values(Interview.Status) - global.Promise = require('bluebird') Joi.rateType = () => Joi.string().valid('hourly', 'daily', 'weekly', 'monthly') @@ -15,7 +11,6 @@ Joi.jobCandidateStatus = () => Joi.string().valid('open', 'selected', 'shortlist Joi.workload = () => Joi.string().valid('full-time', 'fractional') Joi.title = () => Joi.string().max(128) Joi.paymentStatus = () => Joi.string().valid('pending', 'partially-completed', 'completed', 'cancelled') -Joi.interviewStatus = () => Joi.string().valid(...allowedInterviewStatuses) Joi.workPeriodPaymentStatus = () => Joi.string().valid('completed', 'cancelled') // Empty string is not allowed by Joi by default and must be enabled with allow(''). // See https://joi.dev/api/?v=17.3.0#string fro details why it's like this. diff --git a/src/common/constants.js b/src/common/constants.js index d7f16ec..62d8720 100644 --- a/src/common/constants.js +++ b/src/common/constants.js @@ -16,15 +16,5 @@ module.exports = { JobCandidateCreate: 'jobcandidate:create', JobCandidateUpdate: 'jobcandidate:update' } - }, - Interview: { - Status: { - Scheduling: 'Scheduling', - Scheduled: 'Scheduled', - RequestedForReschedule: 'Requested for reschedule', - Rescheduled: 'Rescheduled', - Completed: 'Completed', - Cancelled: 'Cancelled' - } } } diff --git a/src/scripts/createIndex.js b/src/scripts/createIndex.js index 7142f03..fad96d3 100644 --- a/src/scripts/createIndex.js +++ b/src/scripts/createIndex.js @@ -46,24 +46,6 @@ async function createIndex () { status: { type: 'keyword' }, externalId: { type: 'keyword' }, resume: { type: 'text' }, - interviews: { - type: 'nested', - properties: { - id: { type: 'keyword' }, - jobCandidateId: { type: 'keyword' }, - googleCalendarId: { type: 'keyword' }, - startTimestamp: { type: 'date' }, - attendeesList: { type: 'keyword' }, - customMessage: { type: 'text' }, - xaiTemplate: { type: 'keyword' }, - round: { type: 'integer' }, - status: { type: 'keyword' }, - createdAt: { type: 'date' }, - createdBy: { type: 'keyword' }, - updatedAt: { type: 'date' }, - updatedBy: { type: 'keyword' } - } - }, createdAt: { type: 'date' }, createdBy: { type: 'keyword' }, updatedAt: { type: 'date' }, diff --git a/src/services/InterviewProcessorService.js b/src/services/InterviewProcessorService.js deleted file mode 100644 index 15af5ef..0000000 --- a/src/services/InterviewProcessorService.js +++ /dev/null @@ -1,140 +0,0 @@ -/** - * Interview Processor Service - */ - -const _ = require('lodash') -const Joi = require('@hapi/joi') -const logger = require('../common/logger') -const helper = require('../common/helper') -const constants = require('../common/constants') -const config = require('config') - -const esClient = helper.getESClient() - -/** - * Updates jobCandidate via a painless script - * - * @param {String} jobCandidateId job candidate id - * @param {String} script script definition - * @param {String} transactionId transaction id - */ -async function updateJobCandidateViaScript (jobCandidateId, script, transactionId) { - await esClient.updateExtra({ - index: config.get('esConfig.ES_INDEX_JOB_CANDIDATE'), - id: jobCandidateId, - transactionId, - body: { script }, - refresh: constants.esRefreshOption - }) -} - -/** - * Process request interview entity message. - * Creates an interview record under jobCandidate. - * - * @param {Object} message the kafka message - * @param {String} transactionId - */ -async function processRequestInterview (message, transactionId) { - const interview = message.payload - // add interview in collection if there's already an existing collection - // or initiate a new one with this interview - const script = { - source: ` - ctx._source.containsKey("interviews") - ? ctx._source.interviews.add(params.interview) - : ctx._source.interviews = [params.interview] - `, - params: { interview } - } - await updateJobCandidateViaScript(interview.jobCandidateId, script, transactionId) -} - -processRequestInterview.schema = { - message: Joi.object().keys({ - topic: Joi.string().required(), - originator: Joi.string().required(), - timestamp: Joi.date().required(), - 'mime-type': Joi.string().required(), - payload: Joi.object().keys({ - id: Joi.string().uuid().required(), - jobCandidateId: Joi.string().uuid().required(), - googleCalendarId: Joi.string().allow(null), - customMessage: Joi.string().allow(null), - xaiTemplate: Joi.string().required(), - round: Joi.number().integer().positive().required(), - status: Joi.interviewStatus().required(), - createdAt: Joi.date().required(), - createdBy: Joi.string().uuid().required(), - updatedAt: Joi.date().allow(null), - updatedBy: Joi.string().uuid().allow(null), - attendeesList: Joi.array().items(Joi.string().email()).allow(null), - startTimestamp: Joi.date().allow(null) - }).required() - }).required(), - transactionId: Joi.string().required() -} - -/** - * Process update interview entity message. - * Update an interview record under jobCandidate. - * - * @param {Object} message the kafka message - * @param {String} transactionId - */ -async function processUpdateInterview (message, transactionId) { - const data = message.payload - const { body: jobCandidate } = await esClient.getExtra({ - index: config.get('esConfig.ES_INDEX_JOB_CANDIDATE'), - id: data.jobCandidateId - }) - const interviews = jobCandidate.interviews || [] - const index = _.findIndex(interviews, ['id', data.id]) - if (index === -1) { - interviews.push(data) - } else { - interviews.splice(index, 1, data) - } - jobCandidate.interviews = interviews - await esClient.updateExtra({ - index: config.get('esConfig.ES_INDEX_JOB_CANDIDATE'), - id: data.jobCandidateId, - transactionId, - body: { - doc: jobCandidate - }, - refresh: constants.esRefreshOption - }) -} - -processUpdateInterview.schema = { - message: Joi.object().keys({ - topic: Joi.string().required(), - originator: Joi.string().required(), - timestamp: Joi.date().required(), - 'mime-type': Joi.string().required(), - payload: Joi.object().keys({ - id: Joi.string().uuid().required(), - jobCandidateId: Joi.string().uuid().required(), - googleCalendarId: Joi.string().allow(null), - customMessage: Joi.string().allow(null), - xaiTemplate: Joi.string().required(), - round: Joi.number().integer().positive().required(), - status: Joi.interviewStatus().required(), - createdAt: Joi.date().required(), - createdBy: Joi.string().uuid().required(), - updatedAt: Joi.date().required(), - updatedBy: Joi.string().uuid().required(), - attendeesList: Joi.array().items(Joi.string().email()).allow(null), - startTimestamp: Joi.date().allow(null) - }).required() - }).required(), - transactionId: Joi.string().required() -} - -module.exports = { - processRequestInterview, - processUpdateInterview -} - -logger.buildService(module.exports, 'InterviewProcessorService') From 6872051e917d9b9a3996664fb813eb3e1773afbf Mon Sep 17 00:00:00 2001 From: eisbilir Date: Thu, 22 Apr 2021 02:15:47 +0300 Subject: [PATCH 07/55] Resource Booking index to not store time, only dates --- .gitignore | 1 + src/scripts/createIndex.js | 4 ++-- .../ResourceBookingProcessorService.js | 6 +++-- .../taas.resourcebooking.create.event.json | 21 ++++++++++++++++- .../taas.resourcebooking.update.event.json | 23 ++++++++++++++++++- 5 files changed, 49 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 7d71a33..f00801a 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ coverage .DS_Store .env api.env +.eslintrc.y*ml diff --git a/src/scripts/createIndex.js b/src/scripts/createIndex.js index fad96d3..33114fb 100644 --- a/src/scripts/createIndex.js +++ b/src/scripts/createIndex.js @@ -63,8 +63,8 @@ async function createIndex () { userId: { type: 'keyword' }, jobId: { type: 'keyword' }, status: { type: 'keyword' }, - startDate: { type: 'date' }, - endDate: { type: 'date' }, + startDate: { type: 'date', format: 'yyyy-MM-dd' }, + endDate: { type: 'date', format: 'yyyy-MM-dd' }, memberRate: { type: 'float' }, customerRate: { type: 'float' }, rateType: { type: 'keyword' }, diff --git a/src/services/ResourceBookingProcessorService.js b/src/services/ResourceBookingProcessorService.js index 4972afe..1dff5b4 100644 --- a/src/services/ResourceBookingProcessorService.js +++ b/src/services/ResourceBookingProcessorService.js @@ -37,8 +37,10 @@ processCreate.schema = { projectId: Joi.number().integer().required(), userId: Joi.string().uuid().required(), jobId: Joi.string().uuid().allow(null), - startDate: Joi.date().allow(null), - endDate: Joi.date().allow(null), + // eslint-disable-next-line no-useless-escape + startDate: Joi.string().regex(/^(19|20)\d\d-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])$/).allow(null), + // eslint-disable-next-line no-useless-escape + endDate: Joi.string().regex(/^(19|20)\d\d-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])$/).allow(null), memberRate: Joi.number().allow(null), customerRate: Joi.number().allow(null), rateType: Joi.rateType().required(), diff --git a/test/messages/taas.resourcebooking.create.event.json b/test/messages/taas.resourcebooking.create.event.json index 92ec817..31d6217 100644 --- a/test/messages/taas.resourcebooking.create.event.json +++ b/test/messages/taas.resourcebooking.create.event.json @@ -1 +1,20 @@ -{"topic":"taas.resourcebooking.create","originator":"taas-api","timestamp":"2020-11-05T19:00:25.038Z","mime-type":"application/json","payload":{"projectId":21,"userId":"a55fe1bc-1754-45fa-9adc-cf3d6d7c377a","jobId":"ffbc24f7-301e-48d3-bf01-c056916056a2","startDate":"2020-09-27T04:17:23.131Z","endDate":"2020-09-27T04:17:23.131Z","memberRate":13.23,"customerRate":13,"rateType":"hourly","id":"60d97713-8621-476e-b006-7cb9589c7777","createdAt":"2020-11-05T19:00:23.036Z","createdBy":"a55fe1bc-1754-45fa-9adc-cf3d6d7c377a","status":"assigned"}} \ No newline at end of file +{ + "topic": "taas.resourcebooking.create", + "originator": "taas-api", + "timestamp": "2020-11-05T19:00:25.038Z", + "mime-type": "application/json", + "payload": { + "projectId": 21, + "userId": "a55fe1bc-1754-45fa-9adc-cf3d6d7c377a", + "jobId": "ffbc24f7-301e-48d3-bf01-c056916056a2", + "startDate": "2020-09-27", + "endDate": "2020-09-27", + "memberRate": 13.23, + "customerRate": 13, + "rateType": "hourly", + "id": "60d97713-8621-476e-b006-7cb9589c7777", + "createdAt": "2020-11-05T19:00:23.036Z", + "createdBy": "a55fe1bc-1754-45fa-9adc-cf3d6d7c377a", + "status": "assigned" + } +} \ No newline at end of file diff --git a/test/messages/taas.resourcebooking.update.event.json b/test/messages/taas.resourcebooking.update.event.json index 2c07a4e..e45fc86 100644 --- a/test/messages/taas.resourcebooking.update.event.json +++ b/test/messages/taas.resourcebooking.update.event.json @@ -1 +1,22 @@ -{"topic":"taas.resourcebooking.update","originator":"taas-api","timestamp":"2020-11-05T19:00:26.407Z","mime-type":"application/json","payload":{"id":"60d97713-8621-476e-b006-7cb9589c7777","projectId":21,"userId":"a55fe1bc-1754-45fa-9adc-cf3d6d7c377a","jobId":"ffbc24f7-301e-48d3-bf01-c056916056a2","startDate":"2020-09-27T04:17:23.131Z","endDate":"2020-09-27T04:17:23.131Z","memberRate":13.23,"customerRate":13,"rateType":"hourly","status":"assigned","updatedAt":"2020-11-05T19:00:25.062Z","updatedBy":"a55fe1bc-1754-45fa-9adc-cf3d6d7c377a","createdAt":"2020-11-05T19:00:16.268Z","createdBy":"a55fe1bc-1754-45fa-9adc-cf3d6d7c377a"}} \ No newline at end of file +{ + "topic": "taas.resourcebooking.update", + "originator": "taas-api", + "timestamp": "2020-11-05T19:00:26.407Z", + "mime-type": "application/json", + "payload": { + "id": "60d97713-8621-476e-b006-7cb9589c7777", + "projectId": 21, + "userId": "a55fe1bc-1754-45fa-9adc-cf3d6d7c377a", + "jobId": "ffbc24f7-301e-48d3-bf01-c056916056a2", + "startDate": "2020-09-27", + "endDate": "2020-09-27", + "memberRate": 13.23, + "customerRate": 13, + "rateType": "hourly", + "status": "assigned", + "updatedAt": "2020-11-05T19:00:25.062Z", + "updatedBy": "a55fe1bc-1754-45fa-9adc-cf3d6d7c377a", + "createdAt": "2020-11-05T19:00:16.268Z", + "createdBy": "a55fe1bc-1754-45fa-9adc-cf3d6d7c377a" + } +} \ No newline at end of file From e7c3c6b7b710cd3258f0f195cd588c0b0367a427 Mon Sep 17 00:00:00 2001 From: eisbilir Date: Thu, 22 Apr 2021 02:26:08 +0300 Subject: [PATCH 08/55] fix comment --- src/services/ResourceBookingProcessorService.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/services/ResourceBookingProcessorService.js b/src/services/ResourceBookingProcessorService.js index 1dff5b4..f507bde 100644 --- a/src/services/ResourceBookingProcessorService.js +++ b/src/services/ResourceBookingProcessorService.js @@ -37,9 +37,7 @@ processCreate.schema = { projectId: Joi.number().integer().required(), userId: Joi.string().uuid().required(), jobId: Joi.string().uuid().allow(null), - // eslint-disable-next-line no-useless-escape startDate: Joi.string().regex(/^(19|20)\d\d-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])$/).allow(null), - // eslint-disable-next-line no-useless-escape endDate: Joi.string().regex(/^(19|20)\d\d-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])$/).allow(null), memberRate: Joi.number().allow(null), customerRate: Joi.number().allow(null), From 012dbc22032c3496651098e75ff5f85efb5aab9c Mon Sep 17 00:00:00 2001 From: Cagdas U Date: Thu, 22 Apr 2021 11:49:01 +0300 Subject: [PATCH 09/55] feat: add interview scheduler * Add ES mapping for Interviews. * Add Kafka consumers for Interview related topics. --- README.md | 3 + VERIFICATION.md | 8 +- config/default.js | 6 +- local/docker-compose.yml | 2 +- src/app.js | 7 +- src/bootstrap.js | 7 + src/common/constants.js | 14 ++ src/scripts/createIndex.js | 18 +++ src/services/InterviewProcessorService.js | 189 ++++++++++++++++++++++ 9 files changed, 249 insertions(+), 5 deletions(-) create mode 100644 src/services/InterviewProcessorService.js diff --git a/README.md b/README.md index d8409ab..39861a5 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,9 @@ The following parameters can be set in config files or in env variables: - `topics.TAAS_WORK_PERIOD_CREATE_TOPIC`: the create work period entity Kafka message topic - `topics.TAAS_WORK_PERIOD_UPDATE_TOPIC`: the update work period entity Kafka message topic - `topics.TAAS_WORK_PERIOD_DELETE_TOPIC`: the delete work period entity Kafka message topic +- `topics.TAAS_INTERVIEW_REQUEST_TOPIC`: the request interview entity Kafka message topic +- `topics.TAAS_INTERVIEW_UPDATE_TOPIC`: the update interview entity Kafka message topic +- `topics.TAAS_INTERVIEW_BULK_UPDATE_TOPIC`: the bulk update interview entity Kafka message topic - `esConfig.HOST`: Elasticsearch host - `esConfig.AWS_REGION`: The Amazon region to use when using AWS Elasticsearch service - `esConfig.ELASTICCLOUD.id`: The elastic cloud id, if your elasticsearch instance is hosted on elastic cloud. DO NOT provide a value for ES_HOST if you are using this diff --git a/VERIFICATION.md b/VERIFICATION.md index c6930b9..c2785be 100644 --- a/VERIFICATION.md +++ b/VERIFICATION.md @@ -2,13 +2,15 @@ ## Create documents in ES -- Run the following commands to create `Job`, `JobCandidate`, `ResourceBooking`, `WorkPeriod` documents in ES. +- Run the following commands to create `Job`, `JobCandidate`, `ResourceBooking`, `WorkPeriod`, `Interview` documents in ES. ``` bash # for Job docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.job.create < test/messages/taas.job.create.event.json # for JobCandidate docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.jobcandidate.create < test/messages/taas.jobcandidate.create.event.json + # for Interview + docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.interview.requested < test/messages/taas.interview.requested.event.json # for ResourceBooking docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.resourcebooking.create < test/messages/taas.resourcebooking.create.event.json # for WorkPeriod @@ -18,13 +20,15 @@ - Run `npm run view-data ` to see if documents were created. ## Update documents in ES -- Run the following commands to update `Job`, `JobCandidate`, `ResourceBooking`, `WorkPeriod` documents in ES. +- Run the following commands to update `Job`, `JobCandidate`, `ResourceBooking`, `WorkPeriod`, `Interview` documents in ES. ``` bash # for Job docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.job.update < test/messages/taas.job.update.event.json # for JobCandidate docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.jobcandidate.update < test/messages/taas.jobcandidate.update.event.json + # for Interview + docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.interview.update < test/messages/taas.interview.update.event.json # for ResourceBooking docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.resourcebooking.update < test/messages/taas.resourcebooking.update.event.json # for WorkPeriod diff --git a/config/default.js b/config/default.js index 446ab4e..bb58a75 100644 --- a/config/default.js +++ b/config/default.js @@ -31,7 +31,11 @@ module.exports = { // topics for work period service TAAS_WORK_PERIOD_CREATE_TOPIC: process.env.TAAS_WORK_PERIOD_CREATE_TOPIC || 'taas.workperiod.create', TAAS_WORK_PERIOD_UPDATE_TOPIC: process.env.TAAS_WORK_PERIOD_UPDATE_TOPIC || 'taas.workperiod.update', - TAAS_WORK_PERIOD_DELETE_TOPIC: process.env.TAAS_WORK_PERIOD_DELETE_TOPIC || 'taas.workperiod.delete' + TAAS_WORK_PERIOD_DELETE_TOPIC: process.env.TAAS_WORK_PERIOD_DELETE_TOPIC || 'taas.workperiod.delete', + // topics for interview service + TAAS_INTERVIEW_REQUEST_TOPIC: process.env.TAAS_INTERVIEW_REQUEST_TOPIC || 'taas.interview.requested', + TAAS_INTERVIEW_UPDATE_TOPIC: process.env.TAAS_INTERVIEW_UPDATE_TOPIC || 'taas.interview.update', + TAAS_INTERVIEW_BULK_UPDATE_TOPIC: process.env.TAAS_INTERVIEW_BULK_UPDATE_TOPIC || 'taas.interview.bulkUpdate' }, esConfig: { diff --git a/local/docker-compose.yml b/local/docker-compose.yml index 5d2d803..f4f8c23 100644 --- a/local/docker-compose.yml +++ b/local/docker-compose.yml @@ -12,7 +12,7 @@ services: - "9092:9092" environment: KAFKA_ADVERTISED_HOST_NAME: localhost - KAFKA_CREATE_TOPICS: "taas.job.create:1:1,taas.jobcandidate.create:1:1,taas.resourcebooking.create:1:1,taas.job.update:1:1,taas.jobcandidate.update:1:1,taas.resourcebooking.update:1:1,taas.job.delete:1:1,taas.jobcandidate.delete:1:1,taas.resourcebooking.delete:1:1" + KAFKA_CREATE_TOPICS: "taas.job.create:1:1,taas.jobcandidate.create:1:1,taas.resourcebooking.create:1:1,taas.interview.requested:1:1,taas.interview.update:1:1,taas.interview.bulkUpdate:1:1,taas.job.update:1:1,taas.jobcandidate.update:1:1,taas.resourcebooking.update:1:1,taas.job.delete:1:1,taas.jobcandidate.delete:1:1,taas.resourcebooking.delete:1:1" KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 esearch: image: elasticsearch:7.7.1 diff --git a/src/app.js b/src/app.js index 1d3ae47..1a592bf 100644 --- a/src/app.js +++ b/src/app.js @@ -13,6 +13,7 @@ const JobProcessorService = require('./services/JobProcessorService') const JobCandidateProcessorService = require('./services/JobCandidateProcessorService') const ResourceBookingProcessorService = require('./services/ResourceBookingProcessorService') const WorkPeriodProcessorService = require('./services/WorkPeriodProcessorService') +const InterviewProcessorService = require('./services/InterviewProcessorService') const Mutex = require('async-mutex').Mutex const events = require('events') @@ -43,7 +44,11 @@ const topicServiceMapping = { // work period [config.topics.TAAS_WORK_PERIOD_CREATE_TOPIC]: WorkPeriodProcessorService.processCreate, [config.topics.TAAS_WORK_PERIOD_UPDATE_TOPIC]: WorkPeriodProcessorService.processUpdate, - [config.topics.TAAS_WORK_PERIOD_DELETE_TOPIC]: WorkPeriodProcessorService.processDelete + [config.topics.TAAS_WORK_PERIOD_DELETE_TOPIC]: WorkPeriodProcessorService.processDelete, + // interview + [config.topics.TAAS_INTERVIEW_REQUEST_TOPIC]: InterviewProcessorService.processRequestInterview, + [config.topics.TAAS_INTERVIEW_UPDATE_TOPIC]: InterviewProcessorService.processUpdateInterview, + [config.topics.TAAS_INTERVIEW_BULK_UPDATE_TOPIC]: InterviewProcessorService.processBulkUpdateInterviews } // Start kafka consumer diff --git a/src/bootstrap.js b/src/bootstrap.js index d25de1a..4e59443 100644 --- a/src/bootstrap.js +++ b/src/bootstrap.js @@ -1,7 +1,12 @@ const Joi = require('@hapi/joi') const config = require('config') +const _ = require('lodash') +const { Interview } = require('../src/common/constants') const constants = require('./common/constants') +const allowedXAITemplates = _.values(Interview.XaiTemplate) +const allowedInterviewStatuses = _.values(Interview.Status) + global.Promise = require('bluebird') Joi.rateType = () => Joi.string().valid('hourly', 'daily', 'weekly', 'monthly') @@ -10,6 +15,8 @@ Joi.jobCandidateStatus = () => Joi.string().valid('open', 'selected', 'shortlist Joi.workload = () => Joi.string().valid('full-time', 'fractional') Joi.title = () => Joi.string().max(128) Joi.paymentStatus = () => Joi.string().valid('pending', 'partially-completed', 'completed', 'cancelled') +Joi.xaiTemplate = () => Joi.string().valid(...allowedXAITemplates) +Joi.interviewStatus = () => Joi.string().valid(...allowedInterviewStatuses) // Empty string is not allowed by Joi by default and must be enabled with allow(''). // See https://joi.dev/api/?v=17.3.0#string fro details why it's like this. // In many cases we would like to allow empty string to make it easier to create UI for editing data. diff --git a/src/common/constants.js b/src/common/constants.js index 62d8720..e3b917d 100644 --- a/src/common/constants.js +++ b/src/common/constants.js @@ -16,5 +16,19 @@ module.exports = { JobCandidateCreate: 'jobcandidate:create', JobCandidateUpdate: 'jobcandidate:update' } + }, + Interview: { + Status: { + Scheduling: 'Scheduling', + Scheduled: 'Scheduled', + RequestedForReschedule: 'Requested for reschedule', + Rescheduled: 'Rescheduled', + Completed: 'Completed', + Cancelled: 'Cancelled' + }, + XaiTemplate: { + '30MinInterview': '30-min-interview', + '60MinInterview': '60-min-interview' + } } } diff --git a/src/scripts/createIndex.js b/src/scripts/createIndex.js index 095633e..b631f06 100644 --- a/src/scripts/createIndex.js +++ b/src/scripts/createIndex.js @@ -46,6 +46,24 @@ async function createIndex () { status: { type: 'keyword' }, externalId: { type: 'keyword' }, resume: { type: 'text' }, + interviews: { + type: 'nested', + properties: { + id: { type: 'keyword' }, + jobCandidateId: { type: 'keyword' }, + googleCalendarId: { type: 'keyword' }, + customMessage: { type: 'text' }, + xaiTemplate: { type: 'keyword' }, + round: { type: 'integer' }, + startTimestamp: { type: 'date' }, + attendeesList: [], + status: { type: 'keyword' }, + createdAt: { type: 'date' }, + createdBy: { type: 'keyword' }, + updatedAt: { type: 'date' }, + updatedBy: { type: 'keyword' } + } + }, createdAt: { type: 'date' }, createdBy: { type: 'keyword' }, updatedAt: { type: 'date' }, diff --git a/src/services/InterviewProcessorService.js b/src/services/InterviewProcessorService.js new file mode 100644 index 0000000..2c3d595 --- /dev/null +++ b/src/services/InterviewProcessorService.js @@ -0,0 +1,189 @@ +/** + * Interview Processor Service + */ + +const Joi = require('@hapi/joi') +const _ = require('lodash') +const logger = require('../common/logger') +const helper = require('../common/helper') +const constants = require('../common/constants') +const config = require('config') + +const esClient = helper.getESClient() + +/** + * Updates jobCandidate via a painless script + * + * @param {String} jobCandidateId job candidate id + * @param {String} script script definition + * @param {String} transactionId transaction id + */ +async function updateJobCandidateViaScript (jobCandidateId, script, transactionId) { + await esClient.updateExtra({ + index: config.get('esConfig.ES_INDEX_JOB_CANDIDATE'), + id: jobCandidateId, + transactionId, + body: { script }, + refresh: constants.esRefreshOption + }) +} + +/** + * Process request interview entity message. + * Creates an interview record under jobCandidate. + * + * @param {Object} message the kafka message + * @param {String} transactionId + */ +async function processRequestInterview (message, transactionId) { + const interview = message.payload + // add interview in collection if there's already an existing collection + // or initiate a new one with this interview + const script = { + source: ` + ctx._source.containsKey("interviews") + ? ctx._source.interviews.add(params.interview) + : ctx._source.interviews = [params.interview] + `, + params: { interview } + } + await updateJobCandidateViaScript(interview.jobCandidateId, script, transactionId) +} + +processRequestInterview.schema = { + message: Joi.object().keys({ + topic: Joi.string().required(), + originator: Joi.string().required(), + timestamp: Joi.date().required(), + 'mime-type': Joi.string().required(), + payload: Joi.object().keys({ + id: Joi.string().uuid().required(), + jobCandidateId: Joi.string().uuid().required(), + googleCalendarId: Joi.string().allow(null), + customMessage: Joi.string().allow(null), + xaiTemplate: Joi.xaiTemplate().required(), + round: Joi.number().integer().positive().required(), + startTimestamp: Joi.date().allow(null), + attendeesList: Joi.array().items(Joi.string().email()).allow(null), + status: Joi.interviewStatus().required(), + createdAt: Joi.date().required(), + createdBy: Joi.string().uuid().required(), + updatedAt: Joi.date().allow(null), + updatedBy: Joi.string().uuid().allow(null) + }).required() + }).required(), + transactionId: Joi.string().required() +} + +/** + * Process update interview entity message + * Updates the interview record under jobCandidate. + * + * @param {Object} message the kafka message + * @param {String} transactionId + */ +async function processUpdateInterview (message, transactionId) { + const interview = message.payload + // if there's an interview with this id, + // update it with the payload + const script = { + source: ` + if (ctx._source.containsKey("interviews")) { + def target = ctx._source.interviews.find(i -> i.id == params.interview.id); + if (target != null) { + for (prop in params.interview.entrySet()) { + target[prop.getKey()] = prop.getValue() + } + } + } + `, + params: { interview } + } + await updateJobCandidateViaScript(interview.jobCandidateId, script, transactionId) +} + +processUpdateInterview.schema = processRequestInterview.schema + +/** + * Process bulk (partially) update interviews entity message. + * Currently supports status, updatedAt and updatedBy fields. + * Update Joi schema to allow more fields. + * (implementation should already handle new fields - just updating Joi schema should be enough) + * + * payload format: + * { + * "jobCandidateId": { + * "interviewId": { ...fields }, + * "interviewId2": { ...fields }, + * ... + * }, + * "jobCandidateId2": { // like above... }, + * ... + * } + * + * @param {Object} message the kafka message + * @param {String} transactionId + */ +async function processBulkUpdateInterviews (message, transactionId) { + const jobCandidates = message.payload + // script to update & params + const script = { + source: ` + def completedInterviews = params.jobCandidates[ctx._id]; + for (interview in completedInterviews.entrySet()) { + def interviewId = interview.getKey(); + def affectedFields = interview.getValue(); + def target = ctx._source.interviews.find(i -> i.id == interviewId); + if (target != null) { + for (field in affectedFields.entrySet()) { + target[field.getKey()] = field.getValue(); + } + } + } + `, + params: { jobCandidates } + } + // update interviews + await esClient.updateByQuery({ + index: config.get('esConfig.ES_INDEX_JOB_CANDIDATE'), + transactionId, + body: { + script, + query: { + ids: { + values: _.keys(jobCandidates) + } + } + }, + refresh: true + }) +} + +processBulkUpdateInterviews.schema = { + message: Joi.object().keys({ + topic: Joi.string().required(), + originator: Joi.string().required(), + timestamp: Joi.date().required(), + 'mime-type': Joi.string().required(), + payload: Joi.object().pattern( + Joi.string().uuid(), // key - jobCandidateId + Joi.object().pattern( + Joi.string().uuid(), // inner key - interviewId + Joi.object().keys({ + status: Joi.interviewStatus(), + updatedAt: Joi.date(), + updatedBy: Joi.string().uuid() + }) // inner value - affected fields of interview + ) // value - object containing interviews + ).min(1) // at least one key - i.e. don't allow empty object + }).required(), + transactionId: Joi.string().required() +} + +module.exports = { + processRequestInterview, + processUpdateInterview, + processBulkUpdateInterviews +} + +logger.buildService(module.exports, 'InterviewProcessorService') From 2665ac5060f31c8e6de760a42624750b8fc53405 Mon Sep 17 00:00:00 2001 From: imcaizheng Date: Thu, 22 Apr 2021 20:36:04 +0800 Subject: [PATCH 10/55] Use billing account for payments --- src/scripts/createIndex.js | 2 ++ src/services/ResourceBookingProcessorService.js | 3 ++- src/services/WorkPeriodPaymentProcessorService.js | 1 + test/messages/taas.resourcebooking.create.event.json | 5 +++-- test/messages/taas.resourcebooking.update.event.json | 3 ++- test/messages/taas.workperiod.create.event.json | 3 ++- 6 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/scripts/createIndex.js b/src/scripts/createIndex.js index 33114fb..c4601ee 100644 --- a/src/scripts/createIndex.js +++ b/src/scripts/createIndex.js @@ -68,6 +68,7 @@ async function createIndex () { memberRate: { type: 'float' }, customerRate: { type: 'float' }, rateType: { type: 'keyword' }, + billingAccountId: { type: 'integer' }, createdAt: { type: 'date' }, createdBy: { type: 'keyword' }, updatedAt: { type: 'date' }, @@ -98,6 +99,7 @@ async function createIndex () { challengeId: { type: 'keyword' }, amount: { type: 'float' }, status: { type: 'keyword' }, + billingAccountId: { type: 'integer' }, createdAt: { type: 'date' }, createdBy: { type: 'keyword' }, updatedAt: { type: 'date' }, diff --git a/src/services/ResourceBookingProcessorService.js b/src/services/ResourceBookingProcessorService.js index f507bde..f407b2b 100644 --- a/src/services/ResourceBookingProcessorService.js +++ b/src/services/ResourceBookingProcessorService.js @@ -46,7 +46,8 @@ processCreate.schema = { createdBy: Joi.string().uuid().required(), updatedAt: Joi.date().allow(null), updatedBy: Joi.string().uuid().allow(null), - status: Joi.resourceBookingStatus().required() + status: Joi.resourceBookingStatus().required(), + billingAccountId: Joi.number().allow(null) }).required() }).required(), transactionId: Joi.string().required() diff --git a/src/services/WorkPeriodPaymentProcessorService.js b/src/services/WorkPeriodPaymentProcessorService.js index 1f6cc54..d336379 100644 --- a/src/services/WorkPeriodPaymentProcessorService.js +++ b/src/services/WorkPeriodPaymentProcessorService.js @@ -48,6 +48,7 @@ processCreate.schema = { challengeId: Joi.string().uuid().required(), amount: Joi.number().greater(0).allow(null), status: Joi.workPeriodPaymentStatus().required(), + billingAccountId: Joi.number().allow(null), createdAt: Joi.date().required(), createdBy: Joi.string().uuid().required(), updatedAt: Joi.date().allow(null), diff --git a/test/messages/taas.resourcebooking.create.event.json b/test/messages/taas.resourcebooking.create.event.json index 31d6217..2f3de0a 100644 --- a/test/messages/taas.resourcebooking.create.event.json +++ b/test/messages/taas.resourcebooking.create.event.json @@ -15,6 +15,7 @@ "id": "60d97713-8621-476e-b006-7cb9589c7777", "createdAt": "2020-11-05T19:00:23.036Z", "createdBy": "a55fe1bc-1754-45fa-9adc-cf3d6d7c377a", - "status": "assigned" + "status": "assigned", + "billingAccountId": 80000071 } -} \ No newline at end of file +} diff --git a/test/messages/taas.resourcebooking.update.event.json b/test/messages/taas.resourcebooking.update.event.json index e45fc86..2b96229 100644 --- a/test/messages/taas.resourcebooking.update.event.json +++ b/test/messages/taas.resourcebooking.update.event.json @@ -14,9 +14,10 @@ "customerRate": 13, "rateType": "hourly", "status": "assigned", + "billingAccountId": 80000071, "updatedAt": "2020-11-05T19:00:25.062Z", "updatedBy": "a55fe1bc-1754-45fa-9adc-cf3d6d7c377a", "createdAt": "2020-11-05T19:00:16.268Z", "createdBy": "a55fe1bc-1754-45fa-9adc-cf3d6d7c377a" } -} \ No newline at end of file +} diff --git a/test/messages/taas.workperiod.create.event.json b/test/messages/taas.workperiod.create.event.json index 7afea61..3c2d286 100644 --- a/test/messages/taas.workperiod.create.event.json +++ b/test/messages/taas.workperiod.create.event.json @@ -14,9 +14,10 @@ "projectId": 111, "userHandle": "pshah_manager", "id": "926040c4-1709-4de2-b2b6-52adf6e5e72d", + "billingAccountId": 80000071 "createdBy": "00000000-0000-0000-0000-000000000000", "updatedAt": "2021-03-30T20:24:17.541Z", "createdAt": "2021-03-30T20:24:17.541Z", "updatedBy": null } -} \ No newline at end of file +} From 06a2e7c34ea9f22b533197abf7affadc9fe911bd Mon Sep 17 00:00:00 2001 From: Cagdas U Date: Tue, 27 Apr 2021 23:28:03 +0300 Subject: [PATCH 11/55] feat(interview-scheduler): frontend integration 1. Update x.ai template names. 2. Update resourceBooking & jobCandidate statuses. --- src/bootstrap.js | 4 ++-- src/common/constants.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/bootstrap.js b/src/bootstrap.js index 0be6ce3..c1df698 100644 --- a/src/bootstrap.js +++ b/src/bootstrap.js @@ -11,8 +11,8 @@ global.Promise = require('bluebird') Joi.rateType = () => Joi.string().valid('hourly', 'daily', 'weekly', 'monthly') Joi.jobStatus = () => Joi.string().valid('sourcing', 'in-review', 'assigned', 'closed', 'cancelled') -Joi.resourceBookingStatus = () => Joi.string().valid('assigned', 'closed', 'cancelled') -Joi.jobCandidateStatus = () => Joi.string().valid('open', 'selected', 'shortlist', 'rejected', 'cancelled', 'interview', 'topcoder-rejected') +Joi.resourceBookingStatus = () => Joi.string().valid('placed', 'closed', 'cancelled') +Joi.jobCandidateStatus = () => Joi.string().valid('open', 'placed', 'selected', 'client rejected - screening', 'client rejected - interview', 'rejected - other', 'cancelled', 'interview', 'topcoder-rejected') Joi.workload = () => Joi.string().valid('full-time', 'fractional') Joi.title = () => Joi.string().max(128) Joi.paymentStatus = () => Joi.string().valid('pending', 'partially-completed', 'completed', 'cancelled') diff --git a/src/common/constants.js b/src/common/constants.js index e3b917d..892d4af 100644 --- a/src/common/constants.js +++ b/src/common/constants.js @@ -27,8 +27,8 @@ module.exports = { Cancelled: 'Cancelled' }, XaiTemplate: { - '30MinInterview': '30-min-interview', - '60MinInterview': '60-min-interview' + '30MinInterview': '30-minutes', + '60MinInterview': '60-minutes' } } } From 63236bc2f5e32bc328c92e32a8b0bc9d75953205 Mon Sep 17 00:00:00 2001 From: urwithat Date: Thu, 29 Apr 2021 03:41:56 +0530 Subject: [PATCH 12/55] Change the XaiTemplate values --- src/common/constants.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/constants.js b/src/common/constants.js index 892d4af..c2a154a 100644 --- a/src/common/constants.js +++ b/src/common/constants.js @@ -27,8 +27,8 @@ module.exports = { Cancelled: 'Cancelled' }, XaiTemplate: { - '30MinInterview': '30-minutes', - '60MinInterview': '60-minutes' + '30MinInterview': 'interview-30', + '60MinInterview': 'interview-60' } } } From 505dd11041fed47aefc63b6975f6f2d37ccc1729 Mon Sep 17 00:00:00 2001 From: nkumar-topcoder <33625707+nkumar-topcoder@users.noreply.github.com> Date: Thu, 29 Apr 2021 13:52:12 +0530 Subject: [PATCH 13/55] add interview n client rejected screening status --- src/services/JobCandidateProcessorService.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/services/JobCandidateProcessorService.js b/src/services/JobCandidateProcessorService.js index 3eeaaaa..5500667 100644 --- a/src/services/JobCandidateProcessorService.js +++ b/src/services/JobCandidateProcessorService.js @@ -25,7 +25,8 @@ async function updateCandidateStatus ({ type, payload, previousData }) { localLogger.debug({ context: 'updateCandidateStatus', message: `jobCandidate is already in status: ${payload.status}` }) return } - if (!['rejected', 'shortlist'].includes(payload.status)) { + //if (!['rejected', 'shortlist',].includes(payload.status)) { + if (!['client rejected - screening', 'interview',].includes(payload.status)) { localLogger.debug({ context: 'updateCandidateStatus', message: `not interested status: ${payload.status}` }) return } From 1c7564fbf72e2b1e40d6db4354f70d779dba303a Mon Sep 17 00:00:00 2001 From: nkumar-topcoder <33625707+nkumar-topcoder@users.noreply.github.com> Date: Thu, 29 Apr 2021 14:09:26 +0530 Subject: [PATCH 14/55] add - allow selected status to zap --- src/services/JobCandidateProcessorService.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/JobCandidateProcessorService.js b/src/services/JobCandidateProcessorService.js index 5500667..8e331dc 100644 --- a/src/services/JobCandidateProcessorService.js +++ b/src/services/JobCandidateProcessorService.js @@ -26,7 +26,7 @@ async function updateCandidateStatus ({ type, payload, previousData }) { return } //if (!['rejected', 'shortlist',].includes(payload.status)) { - if (!['client rejected - screening', 'interview',].includes(payload.status)) { + if (!['client rejected - screening', 'interview','selected'].includes(payload.status)) { localLogger.debug({ context: 'updateCandidateStatus', message: `not interested status: ${payload.status}` }) return } From 90c4b4f52e6a40b358fa539a407dfbfb61db8e72 Mon Sep 17 00:00:00 2001 From: nkumar-topcoder <33625707+nkumar-topcoder@users.noreply.github.com> Date: Thu, 29 Apr 2021 15:02:06 +0530 Subject: [PATCH 15/55] add client rejected - interview for zap --- src/services/JobCandidateProcessorService.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/JobCandidateProcessorService.js b/src/services/JobCandidateProcessorService.js index 8e331dc..051af9a 100644 --- a/src/services/JobCandidateProcessorService.js +++ b/src/services/JobCandidateProcessorService.js @@ -26,7 +26,7 @@ async function updateCandidateStatus ({ type, payload, previousData }) { return } //if (!['rejected', 'shortlist',].includes(payload.status)) { - if (!['client rejected - screening', 'interview','selected'].includes(payload.status)) { + if (!['client rejected - screening', 'client rejected - interview','interview','selected'].includes(payload.status)) { localLogger.debug({ context: 'updateCandidateStatus', message: `not interested status: ${payload.status}` }) return } From d85138d52583becc39665cc0d91e0c1682f293b7 Mon Sep 17 00:00:00 2001 From: Cagdas U Date: Mon, 10 May 2021 08:21:59 +0300 Subject: [PATCH 16/55] feat(interview-scheduler): update mapping & validations for the new model Update ES mappings & Joi validations for the new changes on Interview schema. * Add `xaiId` field. * Add `rescheduleUrl` field. * Add `endTimestamp` field. * Add `duration` field. * Add `templateId` field. * Add `templateType` field. * Add `title` field. * Add `locationDetails` field. * Add `hostName` field. * Add `hostEmail` field. * Add `guestNames` field. * Rename field `attendeesList` to `guestEmails` * Rename field `googleCalendarId` to `calendarEventId` * Rename field `xaiTemplate` to `templateUrl` * Remove `customMessage` field --- src/scripts/createIndex.js | 20 +++++++++++++++----- src/services/InterviewProcessorService.js | 18 ++++++++++++++---- src/services/JobCandidateProcessorService.js | 4 ++-- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/src/scripts/createIndex.js b/src/scripts/createIndex.js index dc0d213..84325e2 100644 --- a/src/scripts/createIndex.js +++ b/src/scripts/createIndex.js @@ -50,14 +50,24 @@ async function createIndex () { type: 'nested', properties: { id: { type: 'keyword' }, + xaiId: { type: 'keyword' }, jobCandidateId: { type: 'keyword' }, - googleCalendarId: { type: 'keyword' }, - customMessage: { type: 'text' }, - xaiTemplate: { type: 'keyword' }, - round: { type: 'integer' }, + calendarEventId: { type: 'keyword' }, + templateUrl: { type: 'keyword' }, + templateId: { type: 'keyword' }, + templateType: { type: 'keyword' }, + title: { type: 'keyword' }, + locationDetails: { type: 'keyword' }, + duration: { type: 'integer' }, startTimestamp: { type: 'date' }, - attendeesList: [], + endTimestamp: { type: 'date' }, + hostName: { type: 'keyword' }, + hostEmail: { type: 'keyword' }, + guestNames: { type: 'keyword' }, + guestEmails: { type: 'keyword' }, + round: { type: 'integer' }, status: { type: 'keyword' }, + rescheduleUrl: { type: 'keyword' }, createdAt: { type: 'date' }, createdBy: { type: 'keyword' }, updatedAt: { type: 'date' }, diff --git a/src/services/InterviewProcessorService.js b/src/services/InterviewProcessorService.js index 2c3d595..0879589 100644 --- a/src/services/InterviewProcessorService.js +++ b/src/services/InterviewProcessorService.js @@ -58,14 +58,24 @@ processRequestInterview.schema = { 'mime-type': Joi.string().required(), payload: Joi.object().keys({ id: Joi.string().uuid().required(), + xaiId: Joi.string().uuid().allow(null), jobCandidateId: Joi.string().uuid().required(), - googleCalendarId: Joi.string().allow(null), - customMessage: Joi.string().allow(null), - xaiTemplate: Joi.xaiTemplate().required(), + calendarEventId: Joi.string().allow(null), + templateUrl: Joi.xaiTemplate().required(), + templateId: Joi.string().uuid().allow(null), + templateType: Joi.string().allow(null), + title: Joi.string().uuid().allow(null), + locationDetails: Joi.string().uuid().allow(null), round: Joi.number().integer().positive().required(), + duration: Joi.number().integer().positive().required(), startTimestamp: Joi.date().allow(null), - attendeesList: Joi.array().items(Joi.string().email()).allow(null), + endTimestamp: Joi.date().allow(null), + hostName: Joi.string().required(), + hostEmail: Joi.string().email().required(), + guestNames: Joi.array().items(Joi.string()).allow(null), + guestEmails: Joi.array().items(Joi.string().email()).allow(null), status: Joi.interviewStatus().required(), + rescheduleUrl: Joi.string().allow(null), createdAt: Joi.date().required(), createdBy: Joi.string().uuid().required(), updatedAt: Joi.date().allow(null), diff --git a/src/services/JobCandidateProcessorService.js b/src/services/JobCandidateProcessorService.js index 051af9a..48d8660 100644 --- a/src/services/JobCandidateProcessorService.js +++ b/src/services/JobCandidateProcessorService.js @@ -25,8 +25,8 @@ async function updateCandidateStatus ({ type, payload, previousData }) { localLogger.debug({ context: 'updateCandidateStatus', message: `jobCandidate is already in status: ${payload.status}` }) return } - //if (!['rejected', 'shortlist',].includes(payload.status)) { - if (!['client rejected - screening', 'client rejected - interview','interview','selected'].includes(payload.status)) { + // if (!['rejected', 'shortlist',].includes(payload.status)) { + if (!['client rejected - screening', 'client rejected - interview', 'interview', 'selected'].includes(payload.status)) { localLogger.debug({ context: 'updateCandidateStatus', message: `not interested status: ${payload.status}` }) return } From c439d9410422d675f47e010fa1c7999a4b19e065 Mon Sep 17 00:00:00 2001 From: eisbilir Date: Mon, 10 May 2021 22:01:58 +0300 Subject: [PATCH 17/55] Get Resource Bookings together with Work Periods --- README.md | 1 - config/default.js | 3 +- package-lock.json | 806 ++++++++++++++++++ package.json | 4 +- src/scripts/createIndex.js | 59 +- src/scripts/deleteIndex.js | 3 +- src/scripts/view-data.js | 3 +- src/services/JobCandidateProcessorService.js | 4 +- .../WorkPeriodPaymentProcessorService.js | 145 +++- src/services/WorkPeriodProcessorService.js | 125 ++- test/common/testData.js | 9 + test/e2e/test.js | 224 ++++- test/messages/taas.job.create.event.json | 30 +- test/messages/taas.job.delete.event.json | 10 +- test/messages/taas.job.update.event.json | 29 +- .../taas.jobcandidate.create.event.json | 15 +- .../taas.jobcandidate.delete.event.json | 10 +- .../taas.jobcandidate.update.event.json | 17 +- .../taas.resourcebooking.create.event.json | 4 +- .../taas.resourcebooking.delete.event.json | 10 +- .../taas.resourcebooking.update.event.json | 4 +- .../taas.workperiod.create.event.json | 5 +- .../taas.workperiod.update.event.json | 17 +- .../taas.workperiodpayment.create.event.json | 19 +- .../taas.workperiodpayment.update.event.json | 19 +- test/unit/prepare.js | 27 +- test/unit/test.js | 215 ++++- 27 files changed, 1609 insertions(+), 208 deletions(-) diff --git a/README.md b/README.md index b33bd0c..9136fcc 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,6 @@ The following parameters can be set in config files or in env variables: - `esConfig.ES_INDEX_JOB`: the index name for job - `esConfig.ES_INDEX_JOB_CANDIDATE`: the index name for job candidate - `esConfig.ES_INDEX_RESOURCE_BOOKING`: the index name for resource booking -- `esConfig.ES_INDEX_WORK_PERIOD`: the index name for work period - `auth0.AUTH0_URL`: Auth0 URL, used to get TC M2M token - `auth0.AUTH0_AUDIENCE`: Auth0 audience, used to get TC M2M token diff --git a/config/default.js b/config/default.js index 5476e65..1ea7851 100644 --- a/config/default.js +++ b/config/default.js @@ -54,8 +54,7 @@ module.exports = { ES_INDEX_JOB: process.env.ES_INDEX_JOB || 'job', ES_INDEX_JOB_CANDIDATE: process.env.ES_INDEX_JOB_CANDIDATE || 'job_candidate', - ES_INDEX_RESOURCE_BOOKING: process.env.ES_INDEX_RESOURCE_BOOKING || 'resource_booking', - ES_INDEX_WORK_PERIOD: process.env.ES_INDEX_WORK_PERIOD || 'work_period' + ES_INDEX_RESOURCE_BOOKING: process.env.ES_INDEX_RESOURCE_BOOKING || 'resource_booking' }, auth0: { diff --git a/package-lock.json b/package-lock.json index 6a4fb26..8db26cb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -216,6 +216,12 @@ } } }, + "@sindresorhus/is": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.14.0.tgz", + "integrity": "sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ==", + "dev": true + }, "@sinonjs/commons": { "version": "1.8.3", "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.3.tgz", @@ -251,6 +257,15 @@ "integrity": "sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ==", "dev": true }, + "@szmarczak/http-timer": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-1.1.2.tgz", + "integrity": "sha512-XIB2XbzHTN6ieIjfIMV9hlVcfPU26s2vafYWQcZHWXHOxiaRZYEDKEwdl129Zyg50+foYV2jCgtrqSA6qNuNSA==", + "dev": true, + "requires": { + "defer-to-connect": "^1.0.1" + } + }, "@tootallnate/once": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", @@ -350,6 +365,12 @@ "@types/node": "*" } }, + "abbrev": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", + "dev": true + }, "accepts": { "version": "1.3.7", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", @@ -407,6 +428,43 @@ "integrity": "sha512-RO1ibKvd27e6FEShVFfPALuHI3WjSVNeK5FIsmme/LYRNxjKuNj+Dt7bucLa6NdSv3JcVTyMlm9kGR84z1XpaQ==", "dev": true }, + "ansi-align": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-3.0.0.tgz", + "integrity": "sha512-ZpClVKqXN3RGBmKibdfWzqCY4lnjEuoNzU5T0oEFpfd/z5qJHVarukridD4juLO2FXMiwUQxr9WqQtaYa8XRYw==", + "dev": true, + "requires": { + "string-width": "^3.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "dev": true + }, + "string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dev": true, + "requires": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + } + }, + "strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dev": true, + "requires": { + "ansi-regex": "^4.1.0" + } + } + } + }, "ansi-colors": { "version": "3.2.3", "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-3.2.3.tgz", @@ -639,6 +697,111 @@ "type-is": "~1.6.17" } }, + "boxen": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/boxen/-/boxen-4.2.0.tgz", + "integrity": "sha512-eB4uT9RGzg2odpER62bBwSLvUeGC+WbRjjyyFhGsKnc8wp/m0+hQsMUvUe3H2V0D5vw0nBdO1hCJoZo5mKeuIQ==", + "dev": true, + "requires": { + "ansi-align": "^3.0.0", + "camelcase": "^5.3.1", + "chalk": "^3.0.0", + "cli-boxes": "^2.2.0", + "string-width": "^4.1.0", + "term-size": "^2.1.0", + "type-fest": "^0.8.1", + "widest-line": "^3.1.0" + }, + "dependencies": { + "ansi-regex": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "dev": true + }, + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "chalk": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz", + "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true + }, + "string-width": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz", + "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==", + "dev": true, + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.0" + } + }, + "strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "dev": true, + "requires": { + "ansi-regex": "^5.0.0" + } + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + } + } + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -699,6 +862,38 @@ "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==" }, + "cacheable-request": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-6.1.0.tgz", + "integrity": "sha512-Oj3cAGPCqOZX7Rz64Uny2GYAZNliQSqfbePrgAQ1wKAihYmCUnraBtJtKcGR4xz7wF+LoJC+ssFZvv5BgF9Igg==", + "dev": true, + "requires": { + "clone-response": "^1.0.2", + "get-stream": "^5.1.0", + "http-cache-semantics": "^4.0.0", + "keyv": "^3.0.0", + "lowercase-keys": "^2.0.0", + "normalize-url": "^4.1.0", + "responselike": "^1.0.2" + }, + "dependencies": { + "get-stream": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", + "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "dev": true, + "requires": { + "pump": "^3.0.0" + } + }, + "lowercase-keys": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz", + "integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==", + "dev": true + } + } + }, "caching-transform": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/caching-transform/-/caching-transform-3.0.2.tgz", @@ -772,12 +967,24 @@ "readdirp": "~3.2.0" } }, + "ci-info": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz", + "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==", + "dev": true + }, "circular-json": { "version": "0.3.3", "resolved": "https://registry.npmjs.org/circular-json/-/circular-json-0.3.3.tgz", "integrity": "sha512-UZK3NBx2Mca+b5LsG7bY183pHWt5Y1xts4P3Pz7ENTwGVnJOUWbRb3ocjvX7hx9tq/yTAdclXm9sZ38gNuem4A==", "dev": true }, + "cli-boxes": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-2.2.1.tgz", + "integrity": "sha512-y4coMcylgSCdVinjiDBuR8PCC2bLjyGTwEmPb9NHR/QaNU6EUOXcTY/s6VjGMD6ENSEaeQYHCY0GNGS5jfMwPw==", + "dev": true + }, "cli-cursor": { "version": "2.1.0", "resolved": "https://registry.npm.taobao.org/cli-cursor/download/cli-cursor-2.1.0.tgz", @@ -821,6 +1028,23 @@ } } }, + "clone-response": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.2.tgz", + "integrity": "sha1-0dyXOSAxTfZ/vrlCI7TuNQI56Ws=", + "dev": true, + "requires": { + "mimic-response": "^1.0.0" + }, + "dependencies": { + "mimic-response": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", + "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==", + "dev": true + } + } + }, "code-point-at": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", @@ -925,6 +1149,49 @@ "json5": "^1.0.1" } }, + "configstore": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/configstore/-/configstore-5.0.1.tgz", + "integrity": "sha512-aMKprgk5YhBNyH25hj8wGt2+D52Sw1DRRIzqBwLp2Ya9mFmY8KPvvtvmna8SxVR9JMZ4kzMD68N22vlaRpkeFA==", + "dev": true, + "requires": { + "dot-prop": "^5.2.0", + "graceful-fs": "^4.1.2", + "make-dir": "^3.0.0", + "unique-string": "^2.0.0", + "write-file-atomic": "^3.0.0", + "xdg-basedir": "^4.0.0" + }, + "dependencies": { + "make-dir": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", + "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", + "dev": true, + "requires": { + "semver": "^6.0.0" + } + }, + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + }, + "write-file-atomic": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-3.0.3.tgz", + "integrity": "sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q==", + "dev": true, + "requires": { + "imurmurhash": "^0.1.4", + "is-typedarray": "^1.0.0", + "signal-exit": "^3.0.2", + "typedarray-to-buffer": "^3.1.5" + } + } + } + }, "connection-parse": { "version": "0.0.7", "resolved": "https://registry.npm.taobao.org/connection-parse/download/connection-parse-0.0.7.tgz", @@ -1062,6 +1329,12 @@ "which": "^1.2.9" } }, + "crypto-random-string": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-2.0.0.tgz", + "integrity": "sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==", + "dev": true + }, "dashdash": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", @@ -1105,6 +1378,12 @@ "mimic-response": "^2.0.0" } }, + "deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "dev": true + }, "deep-is": { "version": "0.1.3", "resolved": "https://registry.npm.taobao.org/deep-is/download/deep-is-0.1.3.tgz", @@ -1120,6 +1399,12 @@ "strip-bom": "^3.0.0" } }, + "defer-to-connect": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-1.1.3.tgz", + "integrity": "sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ==", + "dev": true + }, "define-properties": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", @@ -1191,6 +1476,15 @@ "esutils": "^2.0.2" } }, + "dot-prop": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.3.0.tgz", + "integrity": "sha512-QM8q3zDe58hqUqjraQOmzZ1LIH9SWQJTlEKCH4kJ2oQvLZk7RbQXvtDM2XEq3fwkV9CCvvH4LA0AV+ogFsBM2Q==", + "dev": true, + "requires": { + "is-obj": "^2.0.0" + } + }, "dtrace-provider": { "version": "0.8.8", "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.8.8.tgz", @@ -1200,6 +1494,12 @@ "nan": "^2.14.0" } }, + "duplexer3": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz", + "integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=", + "dev": true + }, "ecc-jsbn": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", @@ -1302,6 +1602,12 @@ "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==", "dev": true }, + "escape-goat": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/escape-goat/-/escape-goat-2.1.1.tgz", + "integrity": "sha512-8/uIhbG12Csjy2JEW7D9pHbreaVaS/OpN3ycnyvElTdwM5n6GY6W6e2IPemfvGZeUMqZ9A/3GqIZMgKnBhAw/Q==", + "dev": true + }, "escape-html": { "version": "1.0.3", "resolved": "https://registry.npm.taobao.org/escape-html/download/escape-html-1.0.3.tgz", @@ -2109,12 +2415,57 @@ "is-glob": "^4.0.1" } }, + "global-dirs": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/global-dirs/-/global-dirs-2.1.0.tgz", + "integrity": "sha512-MG6kdOUh/xBnyo9cJFeIKkLEc1AyFq42QTU4XiX51i2NEdxLxLWXIjEjmqKeSuKR7pAZjTqUVoT2b2huxVLgYQ==", + "dev": true, + "requires": { + "ini": "1.3.7" + } + }, "globals": { "version": "11.12.0", "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", "dev": true }, + "got": { + "version": "9.6.0", + "resolved": "https://registry.npmjs.org/got/-/got-9.6.0.tgz", + "integrity": "sha512-R7eWptXuGYxwijs0eV+v3o6+XH1IqVK8dJOEecQfTmkncw9AV4dcw/Dhxi8MdlqPthxxpZyizMzyg8RTmEsG+Q==", + "dev": true, + "requires": { + "@sindresorhus/is": "^0.14.0", + "@szmarczak/http-timer": "^1.1.2", + "cacheable-request": "^6.0.0", + "decompress-response": "^3.3.0", + "duplexer3": "^0.1.4", + "get-stream": "^4.1.0", + "lowercase-keys": "^1.0.1", + "mimic-response": "^1.0.1", + "p-cancelable": "^1.0.0", + "to-readable-stream": "^1.0.0", + "url-parse-lax": "^3.0.0" + }, + "dependencies": { + "decompress-response": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", + "integrity": "sha1-gKTdMjdIOEv6JICDYirt7Jgq3/M=", + "dev": true, + "requires": { + "mimic-response": "^1.0.0" + } + }, + "mimic-response": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", + "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==", + "dev": true + } + } + }, "graceful-fs": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.0.tgz", @@ -2189,6 +2540,12 @@ "integrity": "sha1-uhqPGvKg/DllD1yFA2dwQSIGO0Q=", "dev": true }, + "has-yarn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/has-yarn/-/has-yarn-2.1.0.tgz", + "integrity": "sha512-UqBRqi4ju7T+TqGNdqAO0PaSVGsDGJUBQvk9eUWNGRY1CFGDzYhLWoM7JQEemnlvVcv/YEmc2wNW8BC24EnUsw==", + "dev": true + }, "hasha": { "version": "3.0.0", "resolved": "https://registry.npm.taobao.org/hasha/download/hasha-3.0.0.tgz", @@ -2225,6 +2582,12 @@ "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", "dev": true }, + "http-cache-semantics": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz", + "integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==", + "dev": true + }, "http-errors": { "version": "1.7.2", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", @@ -2305,6 +2668,18 @@ "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", "dev": true }, + "ignore-by-default": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz", + "integrity": "sha1-SMptcvbGo68Aqa1K5odr44ieKwk=", + "dev": true + }, + "import-lazy": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/import-lazy/-/import-lazy-2.1.0.tgz", + "integrity": "sha1-BWmOPUXIjo1+nZLLBYTnfwlvPkM=", + "dev": true + }, "imurmurhash": { "version": "0.1.4", "resolved": "https://registry.npm.taobao.org/imurmurhash/download/imurmurhash-0.1.4.tgz", @@ -2325,6 +2700,12 @@ "resolved": "https://registry.npm.taobao.org/inherits/download/inherits-2.0.3.tgz?cache=0&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Finherits%2Fdownload%2Finherits-2.0.3.tgz", "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" }, + "ini": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.7.tgz", + "integrity": "sha512-iKpRpXP+CrP2jyrxvg1kMUpXDyRUFDWurxbnVT1vQPx+Wz9uCYsMIqYuSBLV+PAaZG/d7kRLKRFc9oDMsH+mFQ==", + "dev": true + }, "inquirer": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-5.2.0.tgz", @@ -2429,6 +2810,15 @@ "integrity": "sha512-r5p9sxJjYnArLjObpjA4xu5EKI3CuKHkJXMhT7kwbpUyIFD1n5PMAsoPvWnvtZiNz7LjkYDRZhd7FlI0eMijEA==", "dev": true }, + "is-ci": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz", + "integrity": "sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==", + "dev": true, + "requires": { + "ci-info": "^2.0.0" + } + }, "is-date-object": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.1.tgz", @@ -2456,12 +2846,40 @@ "is-extglob": "^2.1.1" } }, + "is-installed-globally": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-installed-globally/-/is-installed-globally-0.3.2.tgz", + "integrity": "sha512-wZ8x1js7Ia0kecP/CHM/3ABkAmujX7WPvQk6uu3Fly/Mk44pySulQpnHG46OMjHGXApINnV4QhY3SWnECO2z5g==", + "dev": true, + "requires": { + "global-dirs": "^2.0.1", + "is-path-inside": "^3.0.1" + } + }, + "is-npm": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-npm/-/is-npm-4.0.0.tgz", + "integrity": "sha512-96ECIfh9xtDDlPylNPXhzjsykHsMJZ18ASpaWzQyBr4YRTcVjUvzaHayDAES2oU/3KpljhHUjtSRNiDwi0F0ig==", + "dev": true + }, "is-number": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", "dev": true }, + "is-obj": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", + "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==", + "dev": true + }, + "is-path-inside": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", + "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==", + "dev": true + }, "is-promise": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.1.0.tgz", @@ -2502,6 +2920,12 @@ "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" }, + "is-yarn-global": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/is-yarn-global/-/is-yarn-global-0.3.0.tgz", + "integrity": "sha512-VjSeb/lHmkoyd8ryPVIKvOCn4D1koMqY+vqyjjUfc3xyKtP4dYOxM44sZrnqQSzSds3xyOrUTLTC9LVCVgLngw==", + "dev": true + }, "isarray": { "version": "1.0.0", "resolved": "https://registry.npm.taobao.org/isarray/download/isarray-1.0.0.tgz", @@ -2649,6 +3073,12 @@ "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==", "dev": true }, + "json-buffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.0.tgz", + "integrity": "sha1-Wx85evx11ne96Lz8Dkfh+aPZqJg=", + "dev": true + }, "json-parse-better-errors": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", @@ -2774,6 +3204,15 @@ "safe-buffer": "^5.0.1" } }, + "keyv": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-3.1.0.tgz", + "integrity": "sha512-9ykJ/46SN/9KPM/sichzQ7OvXyGDYKGTaDlKMGCAlg2UK8KRy4jb0d8sFc+0Tt0YYnThq8X2RZgCg74RPxgcVA==", + "dev": true, + "requires": { + "json-buffer": "3.0.0" + } + }, "kuler": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/kuler/-/kuler-1.0.1.tgz", @@ -2782,6 +3221,15 @@ "colornames": "^1.1.1" } }, + "latest-version": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/latest-version/-/latest-version-5.1.0.tgz", + "integrity": "sha512-weT+r0kTkRQdCdYCNtkMwWXQTMEswKrFBkm4ckQOMVhhqhIMI1UT2hMj+1iigIhgSZm5gTmrRXBNoGUgaTY1xA==", + "dev": true, + "requires": { + "package-json": "^6.3.0" + } + }, "lcid": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/lcid/-/lcid-2.0.0.tgz", @@ -2959,6 +3407,12 @@ "js-tokens": "^3.0.0 || ^4.0.0" } }, + "lowercase-keys": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz", + "integrity": "sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA==", + "dev": true + }, "lru-cache": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", @@ -3441,6 +3895,59 @@ "semver": "^5.7.0" } }, + "nodemon": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.7.tgz", + "integrity": "sha512-XHzK69Awgnec9UzHr1kc8EomQh4sjTQ8oRf8TsGrSmHDx9/UmiGG9E/mM3BuTfNeFwdNBvrqQq/RHL0xIeyFOA==", + "dev": true, + "requires": { + "chokidar": "^3.2.2", + "debug": "^3.2.6", + "ignore-by-default": "^1.0.1", + "minimatch": "^3.0.4", + "pstree.remy": "^1.1.7", + "semver": "^5.7.1", + "supports-color": "^5.5.0", + "touch": "^3.1.0", + "undefsafe": "^2.0.3", + "update-notifier": "^4.1.0" + }, + "dependencies": { + "debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, + "requires": { + "ms": "^2.1.1" + } + }, + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true + }, + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "requires": { + "has-flag": "^3.0.0" + } + } + } + }, + "nopt": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-1.0.10.tgz", + "integrity": "sha1-bd0hvSoxQXuScn3Vhfim83YI6+4=", + "dev": true, + "requires": { + "abbrev": "1" + } + }, "normalize-package-data": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz", @@ -3459,6 +3966,12 @@ "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", "dev": true }, + "normalize-url": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.0.tgz", + "integrity": "sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ==", + "dev": true + }, "npm-run-path": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", @@ -3688,6 +4201,12 @@ "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", "dev": true }, + "p-cancelable": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-1.1.0.tgz", + "integrity": "sha512-s73XxOZ4zpt1edZYZzvhqFa6uvQc1vwUa0K0BdtIZgQMAJj9IbebH+JkgKZc9h+B05PKHLOTl4ajG1BmNrVZlw==", + "dev": true + }, "p-defer": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-1.0.0.tgz", @@ -3742,6 +4261,26 @@ "release-zalgo": "^1.0.0" } }, + "package-json": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/package-json/-/package-json-6.5.0.tgz", + "integrity": "sha512-k3bdm2n25tkyxcjSKzB5x8kfVxlMdgsbPr0GkZcwHsLpba6cBjqCt1KlcChKEvxHIcTB1FVMuwoijZ26xex5MQ==", + "dev": true, + "requires": { + "got": "^9.6.0", + "registry-auth-token": "^4.0.0", + "registry-url": "^5.0.0", + "semver": "^6.2.0" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + } + } + }, "parse-json": { "version": "4.0.0", "resolved": "https://registry.npm.taobao.org/parse-json/download/parse-json-4.0.0.tgz?cache=0&sync_timestamp=1598129684464&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fparse-json%2Fdownload%2Fparse-json-4.0.0.tgz", @@ -3917,6 +4456,12 @@ "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=", "dev": true }, + "prepend-http": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-2.0.0.tgz", + "integrity": "sha1-6SQ0v6XqjBn0HN/UAddBo8gZ2Jc=", + "dev": true + }, "process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", @@ -3974,6 +4519,12 @@ "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz", "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==" }, + "pstree.remy": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz", + "integrity": "sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==", + "dev": true + }, "pump": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", @@ -3988,6 +4539,15 @@ "resolved": "https://registry.npm.taobao.org/punycode/download/punycode-1.3.2.tgz", "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" }, + "pupa": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/pupa/-/pupa-2.1.1.tgz", + "integrity": "sha512-l1jNAspIBSFqbT+y+5FosojNpVpF94nlI+wDUpqP9enwOTfHx9f0gh5nB96vl+6yTpsJsypeNrwfzPrKuHB41A==", + "dev": true, + "requires": { + "escape-goat": "^2.0.0" + } + }, "qs": { "version": "6.7.0", "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", @@ -4039,6 +4599,18 @@ "unpipe": "1.0.0" } }, + "rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "dev": true, + "requires": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + } + }, "react-is": { "version": "16.8.6", "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.8.6.tgz", @@ -4099,6 +4671,24 @@ "integrity": "sha512-lv0M6+TkDVniA3aD1Eg0DVpfU/booSu7Eev3TDO/mZKHBfVjgCGTV4t4buppESEYDtkArYFOxTJWv6S5C+iaNw==", "dev": true }, + "registry-auth-token": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-4.2.1.tgz", + "integrity": "sha512-6gkSb4U6aWJB4SF2ZvLb76yCBjcvufXBqvvEx1HbmKPkutswjW1xNVRY0+daljIYRbogN7O0etYSlbiaEQyMyw==", + "dev": true, + "requires": { + "rc": "^1.2.8" + } + }, + "registry-url": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/registry-url/-/registry-url-5.1.0.tgz", + "integrity": "sha512-8acYXXTI0AkQv6RAOjE3vOaIXZkT9wo4LOFbBKYQEEnnMNBpKqdUrI6S4NT0KPIo/WVvJ5tE/X5LF/TQUf0ekw==", + "dev": true, + "requires": { + "rc": "^1.2.8" + } + }, "release-zalgo": { "version": "1.0.0", "resolved": "https://registry.npm.taobao.org/release-zalgo/download/release-zalgo-1.0.0.tgz", @@ -4197,6 +4787,15 @@ "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", "dev": true }, + "responselike": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/responselike/-/responselike-1.0.2.tgz", + "integrity": "sha1-kYcg7ztjHFZCvgaPFa3lpG9Loec=", + "dev": true, + "requires": { + "lowercase-keys": "^1.0.0" + } + }, "restore-cursor": { "version": "2.0.0", "resolved": "https://registry.npm.taobao.org/restore-cursor/download/restore-cursor-2.0.0.tgz", @@ -4271,6 +4870,23 @@ "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", "integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==" }, + "semver-diff": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/semver-diff/-/semver-diff-3.1.1.tgz", + "integrity": "sha512-GX0Ix/CJcHyB8c4ykpHGIAvLyOwOobtM/8d+TQkAd81/bEjgPHrfba41Vpesr7jX/t8Uh+R3EX9eAS5be+jQYg==", + "dev": true, + "requires": { + "semver": "^6.3.0" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + } + } + }, "send": { "version": "0.17.1", "resolved": "https://registry.npmjs.org/send/-/send-0.17.1.tgz", @@ -5003,6 +5619,12 @@ "request": "^2.88.0" } }, + "term-size": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/term-size/-/term-size-2.2.1.tgz", + "integrity": "sha512-wK0Ri4fOGjv/XPy8SBHZChl8CM7uMc5VML7SqiQ0zG7+J5Vr+RMQDoHa2CNT6KHUnTGIXH34UDMkPzAUyapBZg==", + "dev": true + }, "test-exclude": { "version": "5.2.3", "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-5.2.3.tgz", @@ -5047,6 +5669,12 @@ "integrity": "sha1-3F5pjL0HkmW8c+A3doGk5Og/YW4=", "dev": true }, + "to-readable-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/to-readable-stream/-/to-readable-stream-1.0.0.tgz", + "integrity": "sha512-Iq25XBt6zD5npPhlLVXGFN3/gyR2/qODcKNNyTMd4vbm39HUaOiAM4PMq0eMVC/Tkxz+Zjdsc55g9yyz+Yq00Q==", + "dev": true + }, "to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -5069,6 +5697,15 @@ "express": "^4.16.3" } }, + "touch": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/touch/-/touch-3.1.0.tgz", + "integrity": "sha512-WBx8Uy5TLtOSRtIq+M03/sKDrXCLHxwDcquSP2c43Le03/9serjQBIztjRz6FkJez9D/hleyAXTBGLwwZUw9lA==", + "dev": true, + "requires": { + "nopt": "~1.0.10" + } + }, "tough-cookie": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", @@ -5129,6 +5766,12 @@ "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", "dev": true }, + "type-fest": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz", + "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==", + "dev": true + }, "type-is": { "version": "1.6.18", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", @@ -5138,17 +5781,116 @@ "mime-types": "~2.1.24" } }, + "typedarray-to-buffer": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz", + "integrity": "sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==", + "dev": true, + "requires": { + "is-typedarray": "^1.0.0" + } + }, + "undefsafe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.3.tgz", + "integrity": "sha512-nrXZwwXrD/T/JXeygJqdCO6NZZ1L66HrxM/Z7mIq2oPanoN0F1nLx3lwJMu6AwJY69hdixaFQOuoYsMjE5/C2A==", + "dev": true, + "requires": { + "debug": "^2.2.0" + } + }, "uniq": { "version": "1.0.1", "resolved": "https://registry.npm.taobao.org/uniq/download/uniq-1.0.1.tgz", "integrity": "sha1-sxxa6CVIRKOoKBVBzisEuGWnNP8=", "dev": true }, + "unique-string": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unique-string/-/unique-string-2.0.0.tgz", + "integrity": "sha512-uNaeirEPvpZWSgzwsPGtU2zVSTrn/8L5q/IexZmH0eH6SA73CmAA5U4GwORTxQAZs95TAXLNqeLoPPNO5gZfWg==", + "dev": true, + "requires": { + "crypto-random-string": "^2.0.0" + } + }, "unpipe": { "version": "1.0.0", "resolved": "https://registry.npm.taobao.org/unpipe/download/unpipe-1.0.0.tgz", "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" }, + "update-notifier": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/update-notifier/-/update-notifier-4.1.3.tgz", + "integrity": "sha512-Yld6Z0RyCYGB6ckIjffGOSOmHXj1gMeE7aROz4MG+XMkmixBX4jUngrGXNYz7wPKBmtoD4MnBa2Anu7RSKht/A==", + "dev": true, + "requires": { + "boxen": "^4.2.0", + "chalk": "^3.0.0", + "configstore": "^5.0.1", + "has-yarn": "^2.1.0", + "import-lazy": "^2.1.0", + "is-ci": "^2.0.0", + "is-installed-globally": "^0.3.1", + "is-npm": "^4.0.0", + "is-yarn-global": "^0.3.0", + "latest-version": "^5.0.0", + "pupa": "^2.0.1", + "semver-diff": "^3.1.1", + "xdg-basedir": "^4.0.0" + }, + "dependencies": { + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "chalk": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz", + "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + } + } + }, "uri-js": { "version": "4.2.2", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", @@ -5173,6 +5915,15 @@ "querystring": "0.2.0" } }, + "url-parse-lax": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-3.0.0.tgz", + "integrity": "sha1-FrXK/Afb42dsGxmZF3gj1lA6yww=", + "dev": true, + "requires": { + "prepend-http": "^2.0.0" + } + }, "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npm.taobao.org/util-deprecate/download/util-deprecate-1.0.2.tgz", @@ -5237,6 +5988,55 @@ "string-width": "^1.0.2 || 2" } }, + "widest-line": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-3.1.0.tgz", + "integrity": "sha512-NsmoXalsWVDMGupxZ5R08ka9flZjjiLvHVAWYOKtiKM8ujtZWr9cRffak+uSE48+Ob8ObalXpwyeUiyDD6QFgg==", + "dev": true, + "requires": { + "string-width": "^4.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "dev": true + }, + "emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true + }, + "string-width": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz", + "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==", + "dev": true, + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.0" + } + }, + "strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "dev": true, + "requires": { + "ansi-regex": "^5.0.0" + } + } + } + }, "winston": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/winston/-/winston-3.2.1.tgz", @@ -5351,6 +6151,12 @@ "lodash": "^4.17.11" } }, + "xdg-basedir": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-4.0.0.tgz", + "integrity": "sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q==", + "dev": true + }, "xml2js": { "version": "0.4.19", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", diff --git a/package.json b/package.json index 73f7cc9..f857c4e 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,7 @@ "main": "src/app.js", "scripts": { "start": "node src/app.js", + "dev": "nodemon src/app.js", "lint": "standard", "lint:fix": "standard --fix", "create-index": "node src/scripts/createIndex.js", @@ -22,6 +23,7 @@ "mocha": "^7.1.2", "mocha-prepare": "^0.1.0", "nock": "^12.0.3", + "nodemon": "^2.0.7", "nyc": "^14.1.1", "should": "^13.2.3", "sinon": "^10.0.1", @@ -58,4 +60,4 @@ "test/e2e/*.js" ] } -} +} \ No newline at end of file diff --git a/src/scripts/createIndex.js b/src/scripts/createIndex.js index dc0d213..1d6d6b1 100644 --- a/src/scripts/createIndex.js +++ b/src/scripts/createIndex.js @@ -87,37 +87,35 @@ async function createIndex () { customerRate: { type: 'float' }, rateType: { type: 'keyword' }, billingAccountId: { type: 'integer' }, - createdAt: { type: 'date' }, - createdBy: { type: 'keyword' }, - updatedAt: { type: 'date' }, - updatedBy: { type: 'keyword' } - } - } - } - }, - { - index: config.get('esConfig.ES_INDEX_WORK_PERIOD'), - body: { - mappings: { - properties: { - resourceBookingId: { type: 'keyword' }, - userHandle: { type: 'keyword' }, - projectId: { type: 'integer' }, - userId: { type: 'keyword' }, - startDate: { type: 'date', format: 'yyyy-MM-dd' }, - endDate: { type: 'date', format: 'yyyy-MM-dd' }, - daysWorked: { type: 'integer' }, - memberRate: { type: 'float' }, - customerRate: { type: 'float' }, - paymentStatus: { type: 'keyword' }, - payments: { + workPeriods: { type: 'nested', properties: { - workPeriodId: { type: 'keyword' }, - challengeId: { type: 'keyword' }, - amount: { type: 'float' }, - status: { type: 'keyword' }, - billingAccountId: { type: 'integer' }, + id: { type: 'keyword' }, + resourceBookingId: { type: 'keyword' }, + userHandle: { type: 'keyword' }, + projectId: { type: 'integer' }, + userId: { type: 'keyword' }, + startDate: { type: 'date', format: 'yyyy-MM-dd' }, + endDate: { type: 'date', format: 'yyyy-MM-dd' }, + daysWorked: { type: 'integer' }, + memberRate: { type: 'float' }, + customerRate: { type: 'float' }, + paymentStatus: { type: 'keyword' }, + payments: { + type: 'nested', + properties: { + id: { type: 'keyword' }, + workPeriodId: { type: 'keyword' }, + challengeId: { type: 'keyword' }, + amount: { type: 'float' }, + status: { type: 'keyword' }, + billingAccountId: { type: 'integer' }, + createdAt: { type: 'date' }, + createdBy: { type: 'keyword' }, + updatedAt: { type: 'date' }, + updatedBy: { type: 'keyword' } + } + }, createdAt: { type: 'date' }, createdBy: { type: 'keyword' }, updatedAt: { type: 'date' }, @@ -131,7 +129,8 @@ async function createIndex () { } } } - }] + } + ] for (const index of indices) { await esClient.indices.create(index) diff --git a/src/scripts/deleteIndex.js b/src/scripts/deleteIndex.js index 69594b4..9c3a1bb 100644 --- a/src/scripts/deleteIndex.js +++ b/src/scripts/deleteIndex.js @@ -11,8 +11,7 @@ async function deleteIndex () { const esClient = helper.getESClient() const indices = [config.get('esConfig.ES_INDEX_JOB'), config.get('esConfig.ES_INDEX_JOB_CANDIDATE'), - config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), - config.get('esConfig.ES_INDEX_WORK_PERIOD')] + config.get('esConfig.ES_INDEX_RESOURCE_BOOKING')] for (const index of indices) { await esClient.indices.delete({ index diff --git a/src/scripts/view-data.js b/src/scripts/view-data.js index 9c3d0ce..1db36d5 100644 --- a/src/scripts/view-data.js +++ b/src/scripts/view-data.js @@ -11,8 +11,7 @@ const esClient = helper.getESClient() const modelIndexMapping = { Job: 'ES_INDEX_JOB', JobCandidate: 'ES_INDEX_JOB_CANDIDATE', - ResourceBooking: 'ES_INDEX_RESOURCE_BOOKING', - WorkPeriod: 'ES_INDEX_WORK_PERIOD' + ResourceBooking: 'ES_INDEX_RESOURCE_BOOKING' } async function showESData () { diff --git a/src/services/JobCandidateProcessorService.js b/src/services/JobCandidateProcessorService.js index 051af9a..48d8660 100644 --- a/src/services/JobCandidateProcessorService.js +++ b/src/services/JobCandidateProcessorService.js @@ -25,8 +25,8 @@ async function updateCandidateStatus ({ type, payload, previousData }) { localLogger.debug({ context: 'updateCandidateStatus', message: `jobCandidate is already in status: ${payload.status}` }) return } - //if (!['rejected', 'shortlist',].includes(payload.status)) { - if (!['client rejected - screening', 'client rejected - interview','interview','selected'].includes(payload.status)) { + // if (!['rejected', 'shortlist',].includes(payload.status)) { + if (!['client rejected - screening', 'client rejected - interview', 'interview', 'selected'].includes(payload.status)) { localLogger.debug({ context: 'updateCandidateStatus', message: `not interested status: ${payload.status}` }) return } diff --git a/src/services/WorkPeriodPaymentProcessorService.js b/src/services/WorkPeriodPaymentProcessorService.js index d336379..7f30b6e 100644 --- a/src/services/WorkPeriodPaymentProcessorService.js +++ b/src/services/WorkPeriodPaymentProcessorService.js @@ -18,19 +18,39 @@ const esClient = helper.getESClient() */ async function processCreate (message, transactionId) { const data = message.payload - const workPeriod = await esClient.getExtra({ - index: config.get('esConfig.ES_INDEX_WORK_PERIOD'), - id: data.workPeriodId + // find related resourceBooking + const result = await esClient.search({ + index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), + body: { + query: { + nested: { + path: 'workPeriods', + query: { + match: { 'workPeriods.id': data.workPeriodId } + } + } + } + } }) - const payments = _.isArray(workPeriod.body.payments) ? workPeriod.body.payments : [] + if (!result.body.hits.total.value) { + throw new Error(`id: ${data.workPeriodId} "WorkPeriod" not found`) + } + const resourceBooking = result.body.hits.hits[0]._source + // find related workPeriod record + const workPeriod = _.find(resourceBooking.workPeriods, ['id', data.workPeriodId]) + // Get workPeriod's existing payments + const payments = _.isArray(workPeriod.payments) ? workPeriod.payments : [] + // Append new payment payments.push(data) - - return esClient.updateExtra({ - index: config.get('esConfig.ES_INDEX_WORK_PERIOD'), - id: data.workPeriodId, + // Assign new payments array to workPeriod + workPeriod.payments = payments + // Update ResourceBooking's workPeriods property + await esClient.updateExtra({ + index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), + id: resourceBooking.id, transactionId, body: { - doc: _.assign(workPeriod.body, { payments }) + doc: { workPeriods: resourceBooking.workPeriods } }, refresh: constants.esRefreshOption }) @@ -65,64 +85,103 @@ processCreate.schema = { */ async function processUpdate (message, transactionId) { const data = message.payload - let workPeriod = await esClient.search({ - index: config.get('esConfig.ES_INDEX_WORK_PERIOD'), + // find workPeriodPayment in it's parent ResourceBooking + let result = await esClient.search({ + index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), body: { query: { nested: { - path: 'payments', + path: 'workPeriods.payments', query: { - match: { 'payments.id': data.id } + match: { 'workPeriods.payments.id': data.id } } } } } }) - if (!workPeriod.body.hits.total.value) { - throw new Error(`id: ${data.id} "WorkPeriodPayments" not found`) + if (!result.body.hits.total.value) { + throw new Error(`id: ${data.id} "WorkPeriodPayment" not found`) } + const resourceBooking = _.cloneDeep(result.body.hits.hits[0]._source) + let workPeriod = null + let payment = null + let paymentIndex = null + // find workPeriod and workPeriodPayment records + _.forEach(resourceBooking.workPeriods, wp => { + _.forEach(wp.payments, (p, pi) => { + if (p.id === data.id) { + payment = p + paymentIndex = pi + return false + } + }) + if (payment) { + workPeriod = wp + return false + } + }) let payments // if WorkPeriodPayment's workPeriodId changed then it must be deleted from the old WorkPeriod // and added to the new WorkPeriod - if (workPeriod.body.hits.hits[0]._source.id !== data.workPeriodId) { - payments = _.filter(workPeriod.body.hits.hits[0]._source.payments, (payment) => payment.id !== data.id) + if (payment.workPeriodId !== data.workPeriodId) { + // remove payment from payments + payments = _.filter(workPeriod.payments, p => p.id !== data.id) + // assign payments to workPeriod record + workPeriod.payments = payments + // Update old ResourceBooking's workPeriods property await esClient.updateExtra({ - index: config.get('esConfig.ES_INDEX_WORK_PERIOD'), - id: workPeriod.body.hits.hits[0]._source.id, + index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), + id: resourceBooking.id, transactionId, body: { - doc: _.assign(workPeriod.body.hits.hits[0]._source, { payments }) - } + doc: { workPeriods: resourceBooking.workPeriods } + }, + refresh: constants.esRefreshOption }) - workPeriod = await esClient.getExtra({ - index: config.get('esConfig.ES_INDEX_WORK_PERIOD'), - id: data.workPeriodId + // find workPeriodPayment's new parent WorkPeriod + result = await esClient.search({ + index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), + body: { + query: { + nested: { + path: 'workPeriods', + query: { + match: { 'workPeriods.id': data.workPeriodId } + } + } + } + } }) - payments = _.isArray(workPeriod.body.payments) ? workPeriod.body.payments : [] - payments.push(data) - return esClient.updateExtra({ - index: config.get('esConfig.ES_INDEX_WORK_PERIOD'), - id: data.workPeriodId, + const newResourceBooking = result.body.hits.hits[0]._source + // find WorkPeriod record in ResourceBooking + const newWorkPeriod = _.find(newResourceBooking.workPeriods, ['id', data.workPeriodId]) + // Get WorkPeriod's existing payments + const newPayments = _.isArray(newWorkPeriod.payments) ? newWorkPeriod.payments : [] + // Append new payment + newPayments.push(data) + // Assign new payments array to workPeriod + newWorkPeriod.payments = newPayments + // Update new ResourceBooking's workPeriods property + await esClient.updateExtra({ + index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), + id: newResourceBooking.id, transactionId, body: { - doc: _.assign(workPeriod.body, { payments }) - } + doc: { workPeriods: newResourceBooking.workPeriods } + }, + refresh: constants.esRefreshOption }) + return } - - payments = _.map(workPeriod.body.hits.hits[0]._source.payments, (payment) => { - if (payment.id === data.id) { - return _.assign(payment, data) - } - return payment - }) - - return esClient.updateExtra({ - index: config.get('esConfig.ES_INDEX_WORK_PERIOD'), - id: data.workPeriodId, + // update payment record + workPeriod.payments[paymentIndex] = data + // Update ResourceBooking's workPeriods property + await esClient.updateExtra({ + index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), + id: resourceBooking.id, transactionId, body: { - doc: _.assign(workPeriod.body.hits.hits[0]._source, { payments }) + doc: { workPeriods: resourceBooking.workPeriods } }, refresh: constants.esRefreshOption }) diff --git a/src/services/WorkPeriodProcessorService.js b/src/services/WorkPeriodProcessorService.js index 568c746..59b8d4b 100644 --- a/src/services/WorkPeriodProcessorService.js +++ b/src/services/WorkPeriodProcessorService.js @@ -7,7 +7,7 @@ const logger = require('../common/logger') const helper = require('../common/helper') const constants = require('../common/constants') const config = require('config') - +const _ = require('lodash') const esClient = helper.getESClient() /** @@ -17,11 +17,23 @@ const esClient = helper.getESClient() */ async function processCreate (message, transactionId) { const workPeriod = message.payload - await esClient.createExtra({ - index: config.get('esConfig.ES_INDEX_WORK_PERIOD'), - id: workPeriod.id, + // Find related resourceBooking + const resourceBooking = await esClient.getExtra({ + index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), + id: workPeriod.resourceBookingId + }) + // Get ResourceBooking's existing workPeriods + const workPeriods = _.isArray(resourceBooking.body.workPeriods) ? resourceBooking.body.workPeriods : [] + // Append new workPeriod + workPeriods.push(workPeriod) + // Update ResourceBooking's workPeriods property + await esClient.updateExtra({ + index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), + id: workPeriod.resourceBookingId, transactionId, - body: workPeriod, + body: { + doc: { workPeriods } + }, refresh: constants.esRefreshOption }) } @@ -59,12 +71,78 @@ processCreate.schema = { */ async function processUpdate (message, transactionId) { const data = message.payload + // find workPeriod in it's parent ResourceBooking + let resourceBooking = await esClient.search({ + index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), + body: { + query: { + nested: { + path: 'workPeriods', + query: { + match: { 'workPeriods.id': data.id } + } + } + } + } + }) + if (!resourceBooking.body.hits.total.value) { + throw new Error(`id: ${data.id} "WorkPeriod" not found`) + } + let workPeriods + // if WorkPeriod's resourceBookingId changed then it must be deleted from the old ResourceBooking + // and added to the new ResourceBooking + if (resourceBooking.body.hits.hits[0]._source.id !== data.resourceBookingId) { + // find old workPeriod record, so we can keep it's existing nested payments field + let oldWorkPeriod = _.find(resourceBooking.body.hits.hits[0]._source.workPeriods, ['id', data.id]) + // remove workPeriod from it's old parent + workPeriods = _.filter(resourceBooking.body.hits.hits[0]._source.workPeriods, (workPeriod) => workPeriod.id !== data.id) + // Update old ResourceBooking's workPeriods property + await esClient.updateExtra({ + index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), + id: resourceBooking.body.hits.hits[0]._source.id, + transactionId, + body: { + doc: { workPeriods } + }, + refresh: constants.esRefreshOption + }) + // find workPeriod's new parent ResourceBooking + resourceBooking = await esClient.getExtra({ + index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), + id: data.resourceBookingId + }) + // Get ResourceBooking's existing workPeriods + workPeriods = _.isArray(resourceBooking.body.workPeriods) ? resourceBooking.body.workPeriods : [] + // Update workPeriod record + const newData = _.assign(oldWorkPeriod, data) + // Append updated workPeriod to workPeriods + workPeriods.push(newData) + // Update new ResourceBooking's workPeriods property + await esClient.updateExtra({ + index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), + id: data.resourceBookingId, + transactionId, + body: { + doc: { workPeriods } + }, + refresh: constants.esRefreshOption + }) + return + } + // Update workPeriod record + workPeriods = _.map(resourceBooking.body.hits.hits[0]._source.workPeriods, (workPeriod) => { + if (workPeriod.id === data.id) { + return _.assign(workPeriod, data) + } + return workPeriod + }) + // Update ResourceBooking's workPeriods property await esClient.updateExtra({ - index: config.get('esConfig.ES_INDEX_WORK_PERIOD'), - id: data.id, + index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), + id: data.resourceBookingId, transactionId, body: { - doc: data + doc: { workPeriods } }, refresh: constants.esRefreshOption }) @@ -78,11 +156,34 @@ processUpdate.schema = processCreate.schema * @param {String} transactionId */ async function processDelete (message, transactionId) { - const id = message.payload.id - await esClient.deleteExtra({ - index: config.get('esConfig.ES_INDEX_WORK_PERIOD'), - id, + const data = message.payload + // Find related ResourceBooking + const resourceBooking = await esClient.search({ + index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), + body: { + query: { + nested: { + path: 'workPeriods', + query: { + match: { 'workPeriods.id': data.id } + } + } + } + } + }) + if (!resourceBooking.body.hits.total.value) { + throw new Error(`id: ${data.id} "WorkPeriod" not found`) + } + // Remove workPeriod from workPeriods + const workPeriods = _.filter(resourceBooking.body.hits.hits[0]._source.workPeriods, (workPeriod) => workPeriod.id !== data.id) + // Update ResourceBooking's workPeriods property + await esClient.updateExtra({ + index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), + id: resourceBooking.body.hits.hits[0]._source.id, transactionId, + body: { + doc: { workPeriods } + }, refresh: constants.esRefreshOption }) } diff --git a/test/common/testData.js b/test/common/testData.js index 7e18dde..e583618 100644 --- a/test/common/testData.js +++ b/test/common/testData.js @@ -17,6 +17,15 @@ const messages = { update: { topic: 'taas.resourcebooking.update', message: require('../messages/taas.resourcebooking.update.event.json') }, delete: { topic: 'taas.resourcebooking.delete', message: require('../messages/taas.resourcebooking.delete.event.json') } }, + WorkPeriod: { + create: { topic: 'taas.workperiod.create', message: require('../messages/taas.workperiod.create.event.json') }, + update: { topic: 'taas.workperiod.update', message: require('../messages/taas.workperiod.update.event.json') }, + delete: { topic: 'taas.workperiod.delete', message: require('../messages/taas.workperiod.delete.event.json') } + }, + WorkPeriodPayment: { + create: { topic: 'taas.workperiodpayment.create', message: require('../messages/taas.workperiodpayment.create.event.json') }, + update: { topic: 'taas.workperiodpayment.update', message: require('../messages/taas.workperiodpayment.update.event.json') } + }, messageInvalid: '{ "topic": "taas.job.create", }' } diff --git a/test/e2e/test.js b/test/e2e/test.js index 1e09ccc..020b060 100644 --- a/test/e2e/test.js +++ b/test/e2e/test.js @@ -10,6 +10,7 @@ const should = require('should') const logger = require('../../src/common/logger') const testData = require('../common/testData') const testHelper = require('../common/testHelper') +const _ = require('lodash') describe('Taas ES Processor E2E Test', () => { let infoLogs = [] @@ -54,102 +55,233 @@ describe('Taas ES Processor E2E Test', () => { errorLogs = [] await testHelper.clearES() }) + describe('General Logic Tests', () => { + it('Should setup healthcheck with check on kafka connection', async () => { + const healthcheckEndpoint = `http://localhost:${config.PORT}/health` + const result = await request.get(healthcheckEndpoint) + should.equal(result.status, 200) + should.deepEqual(result.body, { checksRun: 1 }) + }) - it('Should setup healthcheck with check on kafka connection', async () => { - const healthcheckEndpoint = `http://localhost:${config.PORT}/health` - const result = await request.get(healthcheckEndpoint) - should.equal(result.status, 200) - should.deepEqual(result.body, { checksRun: 1 }) - }) + it('Should handle invalid json message', async () => { + await testHelper.sendMessage(testData.messages.messageInvalid, config.topics.TAAS_JOB_CREATE_TOPIC) + await waitForMessageHandled() + errorLogs[0].should.match(/Invalid message JSON/) + }) - it('Should handle invalid json message', async () => { - await testHelper.sendMessage(testData.messages.messageInvalid, config.topics.TAAS_JOB_CREATE_TOPIC) - await waitForMessageHandled() - errorLogs[0].should.match(/Invalid message JSON/) + it('Should handle incorrect topic field message', async () => { + await testHelper.sendMessage(testData.messages.Job.create.message, config.topics.TAAS_JOB_UPDATE_TOPIC) + await waitForMessageHandled() + should.equal(errorLogs[0], `The message topic ${testData.messages.Job.create.topic} doesn't match the Kafka topic ${config.topics.TAAS_JOB_UPDATE_TOPIC}.`) + }) }) + describe('Job, JobCandidate, ResourceBooking tests', () => { + for (const [index, model] of [ + [config.esConfig.ES_INDEX_JOB, 'Job'], + [config.esConfig.ES_INDEX_JOB_CANDIDATE, 'JobCandidate'], + [config.esConfig.ES_INDEX_RESOURCE_BOOKING, 'ResourceBooking'] + ]) { + const modelInSpaceCase = stringcase.spacecase(model) - it('Should handle incorrect topic field message', async () => { - await testHelper.sendMessage(testData.messages.Job.create.message, config.topics.TAAS_JOB_UPDATE_TOPIC) - await waitForMessageHandled() - should.equal(errorLogs[0], `The message topic ${testData.messages.Job.create.topic} doesn't match the Kafka topic ${config.topics.TAAS_JOB_UPDATE_TOPIC}.`) - }) + it(`Should handle ${modelInSpaceCase} creation message`, async () => { + await testHelper.sendMessage(testData.messages[model].create.message) + await waitForMessageHandled() + const doc = await testHelper.esClient.get({ + index, + id: testData.messages[model].create.message.payload.id + }) + should.deepEqual(doc.body._source, testData.messages[model].create.message.payload, ['id']) + }) + + it(`Should handle ${modelInSpaceCase} updating message`, async () => { + await testHelper.esClient.create({ + index, + id: testData.messages[model].create.message.payload.id, + body: testData.messages[model].create.message.payload, + refresh: 'true' + }) + await testHelper.sendMessage(testData.messages[model].update.message) + await waitForMessageHandled() + const doc = await testHelper.esClient.get({ + index, + id: testData.messages[model].update.message.payload.id + }) + should.deepEqual(doc.body._source, testData.messages[model].update.message.payload) + }) + + it(`Should handle ${modelInSpaceCase} deletion message`, async () => { + await testHelper.esClient.create({ + index, + id: testData.messages[model].create.message.payload.id, + body: testData.messages[model].create.message.payload, + refresh: 'true' + }) + await testHelper.sendMessage(testData.messages[model].delete.message) + await waitForMessageHandled() + const doc = await testHelper.esClient.get({ + index, + id: testData.messages[model].delete.message.payload.id + }).catch(err => { + if (err.statusCode === 404) { + return + } + throw err + }) + should.not.exist(doc) + }) + + it(`Failure - creation message - ${modelInSpaceCase} already exists`, async () => { + await testHelper.esClient.create({ + index, + id: testData.messages[model].create.message.payload.id, + body: testData.messages[model].create.message.payload, + refresh: 'true' + }) + await testHelper.sendMessage(testData.messages[model].create.message) + await waitForMessageHandled() + should.equal(errorLogs[0], `id: ${testData.messages[model].create.message.payload.id} "${index}" already exists`) + }) + + it(`Failure - updating message - ${modelInSpaceCase} not found`, async () => { + await testHelper.sendMessage(testData.messages[model].update.message) + await waitForMessageHandled() + should.equal(errorLogs[0], `id: ${testData.messages[model].update.message.payload.id} "${index}" not found`) + }) - for (const [index, model] of [ - [config.esConfig.ES_INDEX_JOB, 'Job'], - [config.esConfig.ES_INDEX_JOB_CANDIDATE, 'JobCandidate'], - [config.esConfig.ES_INDEX_RESOURCE_BOOKING, 'ResourceBooking'] - ]) { + it(`Failure - deletion message - ${modelInSpaceCase} not found`, async () => { + await testHelper.sendMessage(testData.messages[model].delete.message) + await waitForMessageHandled() + should.equal(errorLogs[0], `id: ${testData.messages[model].delete.message.payload.id} "${index}" not found`) + }) + } + }) + describe('Nested WorkPeriod tests', () => { + const index = config.esConfig.ES_INDEX_RESOURCE_BOOKING + const model = 'WorkPeriod' + const parentModel = 'ResourceBooking' + const nestedName = 'workPeriods' const modelInSpaceCase = stringcase.spacecase(model) it(`Should handle ${modelInSpaceCase} creation message`, async () => { + await testHelper.esClient.create({ + index, + id: testData.messages[parentModel].create.message.payload.id, + body: testData.messages[parentModel].create.message.payload, + refresh: 'true' + }) await testHelper.sendMessage(testData.messages[model].create.message) await waitForMessageHandled() const doc = await testHelper.esClient.get({ index, - id: testData.messages[model].create.message.payload.id + id: testData.messages[parentModel].create.message.payload.id }) - should.deepEqual(doc.body._source, testData.messages[model].create.message.payload, ['id']) + should.deepEqual(doc.body._source, + _.assign(testData.messages[parentModel].create.message.payload, { [nestedName]: [testData.messages[model].create.message.payload] })) }) it(`Should handle ${modelInSpaceCase} updating message`, async () => { await testHelper.esClient.create({ index, - id: testData.messages[model].create.message.payload.id, - body: testData.messages[model].create.message.payload, + id: testData.messages[parentModel].create.message.payload.id, + body: _.assign(testData.messages[parentModel].create.message.payload, { [nestedName]: [testData.messages[model].create.message.payload] }), refresh: 'true' }) await testHelper.sendMessage(testData.messages[model].update.message) await waitForMessageHandled() const doc = await testHelper.esClient.get({ index, - id: testData.messages[model].update.message.payload.id + id: testData.messages[parentModel].create.message.payload.id }) - should.deepEqual(doc.body._source, testData.messages[model].update.message.payload) + should.deepEqual(doc.body._source, + _.assign(testData.messages[parentModel].create.message.payload, { [nestedName]: [testData.messages[model].update.message.payload] })) }) it(`Should handle ${modelInSpaceCase} deletion message`, async () => { await testHelper.esClient.create({ index, - id: testData.messages[model].create.message.payload.id, - body: testData.messages[model].create.message.payload, + id: testData.messages[parentModel].create.message.payload.id, + body: _.assign(testData.messages[parentModel].create.message.payload, { [nestedName]: [testData.messages[model].create.message.payload] }), refresh: 'true' }) await testHelper.sendMessage(testData.messages[model].delete.message) await waitForMessageHandled() const doc = await testHelper.esClient.get({ index, - id: testData.messages[model].delete.message.payload.id - }).catch(err => { - if (err.statusCode === 404) { - return - } - throw err - }) - should.not.exist(doc) + id: testData.messages[parentModel].create.message.payload.id + }) + should.deepEqual(doc.body._source, + _.assign(testData.messages[parentModel].create.message.payload, { [nestedName]: [] })) + }) + + it(`Failure - creation message - ${modelInSpaceCase} not found`, async () => { + await testHelper.sendMessage(testData.messages[model].create.message) + await waitForMessageHandled() + should.equal(errorLogs[0], `id: ${testData.messages[parentModel].create.message.payload.id} "${index}" not found`) + }) + + it(`Failure - updating message - ${modelInSpaceCase} not found`, async () => { + await testHelper.sendMessage(testData.messages[model].update.message) + await waitForMessageHandled() + should.equal(errorLogs[0], `id: ${testData.messages[model].update.message.payload.id} "${model}" not found`) + }) + + it(`Failure - deletion message - ${modelInSpaceCase} not found`, async () => { + await testHelper.sendMessage(testData.messages[model].delete.message) + await waitForMessageHandled() + should.equal(errorLogs[0], `id: ${testData.messages[model].delete.message.payload.id} "${model}" not found`) }) + }) + describe('Nested WorkPeriodPayment tests', () => { + const index = config.esConfig.ES_INDEX_RESOURCE_BOOKING + const model = 'WorkPeriodPayment' + const parentModel = 'WorkPeriod' + const rootModel = 'ResourceBooking' + const nestedName = 'payments' + const parentNestedName = 'workPeriods' + const modelInSpaceCase = stringcase.spacecase(model) - it(`Failure - creation message - ${modelInSpaceCase} already exists`, async () => { + it(`Should handle ${modelInSpaceCase} creation message`, async () => { await testHelper.esClient.create({ index, - id: testData.messages[model].create.message.payload.id, - body: testData.messages[model].create.message.payload, + id: testData.messages[rootModel].create.message.payload.id, + body: _.assign(testData.messages[rootModel].create.message.payload, { [parentNestedName]: [testData.messages[parentModel].create.message.payload] }), refresh: 'true' }) await testHelper.sendMessage(testData.messages[model].create.message) await waitForMessageHandled() - should.equal(errorLogs[0], `id: ${testData.messages[model].create.message.payload.id} "${index}" already exists`) + const doc = await testHelper.esClient.get({ + index, + id: testData.messages[rootModel].create.message.payload.id + }) + should.deepEqual(doc.body._source, _.assign(testData.messages[rootModel].create.message.payload, { [parentNestedName]: [_.assign(testData.messages[parentModel].create.message.payload, { [nestedName]: [testData.messages[model].create.message.payload] })] })) }) - it(`Failure - updating message - ${modelInSpaceCase} not found`, async () => { + it(`Should handle ${modelInSpaceCase} updating message`, async () => { + await testHelper.esClient.create({ + index, + id: testData.messages[rootModel].create.message.payload.id, + body: _.assign(testData.messages[rootModel].create.message.payload, { [parentNestedName]: [_.assign(testData.messages[parentModel].create.message.payload, { [nestedName]: [testData.messages[model].create.message.payload] })] }), + refresh: 'true' + }) await testHelper.sendMessage(testData.messages[model].update.message) await waitForMessageHandled() - should.equal(errorLogs[0], `id: ${testData.messages[model].update.message.payload.id} "${index}" not found`) + const doc = await testHelper.esClient.get({ + index, + id: testData.messages[rootModel].create.message.payload.id + }) + should.deepEqual(doc.body._source, _.assign(testData.messages[rootModel].create.message.payload, { [parentNestedName]: [_.assign(testData.messages[parentModel].create.message.payload, { [nestedName]: [testData.messages[model].update.message.payload] })] })) }) - it(`Failure - deletion message - ${modelInSpaceCase} not found`, async () => { - await testHelper.sendMessage(testData.messages[model].delete.message) + it(`Failure - creation message - ${modelInSpaceCase} not found`, async () => { + await testHelper.sendMessage(testData.messages[model].create.message) await waitForMessageHandled() - should.equal(errorLogs[0], `id: ${testData.messages[model].delete.message.payload.id} "${index}" not found`) + should.equal(errorLogs[0], `id: ${testData.messages[parentModel].create.message.payload.id} "${parentModel}" not found`) }) - } + + it(`Failure - updating message - ${modelInSpaceCase} not found`, async () => { + await testHelper.sendMessage(testData.messages[model].update.message) + await waitForMessageHandled() + should.equal(errorLogs[0], `id: ${testData.messages[model].update.message.payload.id} "${model}" not found`) + }) + }) }) diff --git a/test/messages/taas.job.create.event.json b/test/messages/taas.job.create.event.json index e2b12bb..ebc5d15 100644 --- a/test/messages/taas.job.create.event.json +++ b/test/messages/taas.job.create.event.json @@ -1 +1,29 @@ -{"topic":"taas.job.create","originator":"taas-api","timestamp":"2020-11-05T19:00:17.563Z","mime-type":"application/json","payload":{"title":"Job Title","projectId":21,"externalId":"1212","description":"Dummy Description","startDate":"2020-09-27T04:17:23.131Z","duration":17,"numPositions":13,"resourceType":"Dummy Resource Type","rateType":"hourly","skills":["56fdc405-eccc-4189-9e83-c78abf844f50","f91ae184-aba2-4485-a8cb-9336988c05ab","edfc7b4f-636f-44bd-96fc-949ffc58e38b","4ca63bb6-f515-4ab0-a6bc-c2d8531e084f","ee03c041-d53b-4c08-b7d9-80d7461da3e4"],"id":"ffbc24f7-301e-48d3-bf01-c056916056a2","createdAt":"2020-11-05T19:00:16.268Z","createdBy":"a55fe1bc-1754-45fa-9adc-cf3d6d7c377a","status":"sourcing","isApplicationPageActive":false}} \ No newline at end of file +{ + "topic": "taas.job.create", + "originator": "taas-api", + "timestamp": "2020-11-05T19:00:17.563Z", + "mime-type": "application/json", + "payload": { + "title": "Job Title", + "projectId": 21, + "externalId": "1212", + "description": "Dummy Description", + "startDate": "2020-09-27T04:17:23.131Z", + "duration": 17, + "numPositions": 13, + "resourceType": "Dummy Resource Type", + "rateType": "hourly", + "skills": [ + "56fdc405-eccc-4189-9e83-c78abf844f50", + "f91ae184-aba2-4485-a8cb-9336988c05ab", + "edfc7b4f-636f-44bd-96fc-949ffc58e38b", + "4ca63bb6-f515-4ab0-a6bc-c2d8531e084f", + "ee03c041-d53b-4c08-b7d9-80d7461da3e4" + ], + "id": "ffbc24f7-301e-48d3-bf01-c056916056a2", + "createdAt": "2020-11-05T19:00:16.268Z", + "createdBy": "a55fe1bc-1754-45fa-9adc-cf3d6d7c377a", + "status": "sourcing", + "isApplicationPageActive": false + } +} \ No newline at end of file diff --git a/test/messages/taas.job.delete.event.json b/test/messages/taas.job.delete.event.json index 00ddbe9..9ad9844 100644 --- a/test/messages/taas.job.delete.event.json +++ b/test/messages/taas.job.delete.event.json @@ -1 +1,9 @@ -{"topic":"taas.job.delete","originator":"taas-api","timestamp":"2020-11-05T19:00:19.035Z","mime-type":"application/json","payload":{"id":"ffbc24f7-301e-48d3-bf01-c056916056a2"}} \ No newline at end of file +{ + "topic": "taas.job.delete", + "originator": "taas-api", + "timestamp": "2020-11-05T19:00:19.035Z", + "mime-type": "application/json", + "payload": { + "id": "ffbc24f7-301e-48d3-bf01-c056916056a2" + } +} \ No newline at end of file diff --git a/test/messages/taas.job.update.event.json b/test/messages/taas.job.update.event.json index fced890..3be8597 100644 --- a/test/messages/taas.job.update.event.json +++ b/test/messages/taas.job.update.event.json @@ -1 +1,28 @@ -{"topic":"taas.job.update","originator":"taas-api","timestamp":"2020-11-05T19:00:19.015Z","mime-type":"application/json","payload":{"id":"ffbc24f7-301e-48d3-bf01-c056916056a2","title":"Job Title Updated","projectId":21,"externalId":"1212","description":"Dummy Description","startDate":"2020-09-27T04:17:23.131Z","duration":19,"numPositions":13,"resourceType":"Dummy Resource Type","rateType":"hourly","skills":["3fa85f64-5717-4562-b3fc-2c963f66afa6","cc41ddc4-cacc-4570-9bdb-1229c12b9784"],"status":"sourcing","updatedAt":"2020-11-05T19:00:17.612Z","updatedBy":"a55fe1bc-1754-45fa-9adc-cf3d6d7c377a","createdAt":"2020-11-05T19:00:16.268Z","createdBy":"a55fe1bc-1754-45fa-9adc-cf3d6d7c377a","isApplicationPageActive":false}} \ No newline at end of file +{ + "topic": "taas.job.update", + "originator": "taas-api", + "timestamp": "2020-11-05T19:00:19.015Z", + "mime-type": "application/json", + "payload": { + "id": "ffbc24f7-301e-48d3-bf01-c056916056a2", + "title": "Job Title Updated", + "projectId": 21, + "externalId": "1212", + "description": "Dummy Description", + "startDate": "2020-09-27T04:17:23.131Z", + "duration": 19, + "numPositions": 13, + "resourceType": "Dummy Resource Type", + "rateType": "hourly", + "skills": [ + "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "cc41ddc4-cacc-4570-9bdb-1229c12b9784" + ], + "status": "sourcing", + "updatedAt": "2020-11-05T19:00:17.612Z", + "updatedBy": "a55fe1bc-1754-45fa-9adc-cf3d6d7c377a", + "createdAt": "2020-11-05T19:00:16.268Z", + "createdBy": "a55fe1bc-1754-45fa-9adc-cf3d6d7c377a", + "isApplicationPageActive": false + } +} \ No newline at end of file diff --git a/test/messages/taas.jobcandidate.create.event.json b/test/messages/taas.jobcandidate.create.event.json index 18cabc3..822dc80 100644 --- a/test/messages/taas.jobcandidate.create.event.json +++ b/test/messages/taas.jobcandidate.create.event.json @@ -1 +1,14 @@ -{"topic":"taas.jobcandidate.create","originator":"taas-api","timestamp":"2020-11-05T19:00:21.597Z","mime-type":"application/json","payload":{"jobId":"ffbc24f7-301e-48d3-bf01-c056916056a2","userId":"a55fe1bc-1754-45fa-9adc-cf3d6d7c377a","id":"0cb99adb-8bcd-4952-9203-9867dd45ef6f","createdAt":"2020-11-05T19:00:19.052Z","createdBy":"a55fe1bc-1754-45fa-9adc-cf3d6d7c377a","status":"open"}} \ No newline at end of file +{ + "topic": "taas.jobcandidate.create", + "originator": "taas-api", + "timestamp": "2020-11-05T19:00:21.597Z", + "mime-type": "application/json", + "payload": { + "jobId": "ffbc24f7-301e-48d3-bf01-c056916056a2", + "userId": "a55fe1bc-1754-45fa-9adc-cf3d6d7c377a", + "id": "0cb99adb-8bcd-4952-9203-9867dd45ef6f", + "createdAt": "2020-11-05T19:00:19.052Z", + "createdBy": "a55fe1bc-1754-45fa-9adc-cf3d6d7c377a", + "status": "open" + } +} \ No newline at end of file diff --git a/test/messages/taas.jobcandidate.delete.event.json b/test/messages/taas.jobcandidate.delete.event.json index 714cce6..0aa13d5 100644 --- a/test/messages/taas.jobcandidate.delete.event.json +++ b/test/messages/taas.jobcandidate.delete.event.json @@ -1 +1,9 @@ -{"topic":"taas.jobcandidate.delete","originator":"taas-api","timestamp":"2020-11-05T19:00:23.021Z","mime-type":"application/json","payload":{"id":"0cb99adb-8bcd-4952-9203-9867dd45ef6f"}} \ No newline at end of file +{ + "topic": "taas.jobcandidate.delete", + "originator": "taas-api", + "timestamp": "2020-11-05T19:00:23.021Z", + "mime-type": "application/json", + "payload": { + "id": "0cb99adb-8bcd-4952-9203-9867dd45ef6f" + } +} \ No newline at end of file diff --git a/test/messages/taas.jobcandidate.update.event.json b/test/messages/taas.jobcandidate.update.event.json index 79530f9..1254087 100644 --- a/test/messages/taas.jobcandidate.update.event.json +++ b/test/messages/taas.jobcandidate.update.event.json @@ -1 +1,16 @@ -{"topic":"taas.jobcandidate.update","originator":"taas-api","timestamp":"2020-11-05T19:00:23.003Z","mime-type":"application/json","payload":{"id":"0cb99adb-8bcd-4952-9203-9867dd45ef6f","jobId":"ffbc24f7-301e-48d3-bf01-c056916056a2","userId":"a55fe1bc-1754-45fa-9adc-cf3d6d7c377a","status":"selected","updatedAt":"2020-11-05T19:00:21.625Z","updatedBy":"a55fe1bc-1754-45fa-9adc-cf3d6d7c377a","createdAt":"2020-11-05T19:00:16.268Z","createdBy":"a55fe1bc-1754-45fa-9adc-cf3d6d7c377a"}} \ No newline at end of file +{ + "topic": "taas.jobcandidate.update", + "originator": "taas-api", + "timestamp": "2020-11-05T19:00:23.003Z", + "mime-type": "application/json", + "payload": { + "id": "0cb99adb-8bcd-4952-9203-9867dd45ef6f", + "jobId": "ffbc24f7-301e-48d3-bf01-c056916056a2", + "userId": "a55fe1bc-1754-45fa-9adc-cf3d6d7c377a", + "status": "selected", + "updatedAt": "2020-11-05T19:00:21.625Z", + "updatedBy": "a55fe1bc-1754-45fa-9adc-cf3d6d7c377a", + "createdAt": "2020-11-05T19:00:16.268Z", + "createdBy": "a55fe1bc-1754-45fa-9adc-cf3d6d7c377a" + } +} \ No newline at end of file diff --git a/test/messages/taas.resourcebooking.create.event.json b/test/messages/taas.resourcebooking.create.event.json index 2f3de0a..2d7fd00 100644 --- a/test/messages/taas.resourcebooking.create.event.json +++ b/test/messages/taas.resourcebooking.create.event.json @@ -15,7 +15,7 @@ "id": "60d97713-8621-476e-b006-7cb9589c7777", "createdAt": "2020-11-05T19:00:23.036Z", "createdBy": "a55fe1bc-1754-45fa-9adc-cf3d6d7c377a", - "status": "assigned", + "status": "placed", "billingAccountId": 80000071 } -} +} \ No newline at end of file diff --git a/test/messages/taas.resourcebooking.delete.event.json b/test/messages/taas.resourcebooking.delete.event.json index 46a5e6f..644037e 100644 --- a/test/messages/taas.resourcebooking.delete.event.json +++ b/test/messages/taas.resourcebooking.delete.event.json @@ -1 +1,9 @@ -{"topic":"taas.resourcebooking.delete","originator":"taas-api","timestamp":"2020-11-05T19:00:26.433Z","mime-type":"application/json","payload":{"id":"60d97713-8621-476e-b006-7cb9589c7777"}} \ No newline at end of file +{ + "topic": "taas.resourcebooking.delete", + "originator": "taas-api", + "timestamp": "2020-11-05T19:00:26.433Z", + "mime-type": "application/json", + "payload": { + "id": "60d97713-8621-476e-b006-7cb9589c7777" + } +} \ No newline at end of file diff --git a/test/messages/taas.resourcebooking.update.event.json b/test/messages/taas.resourcebooking.update.event.json index 2b96229..036b6c6 100644 --- a/test/messages/taas.resourcebooking.update.event.json +++ b/test/messages/taas.resourcebooking.update.event.json @@ -13,11 +13,11 @@ "memberRate": 13.23, "customerRate": 13, "rateType": "hourly", - "status": "assigned", + "status": "placed", "billingAccountId": 80000071, "updatedAt": "2020-11-05T19:00:25.062Z", "updatedBy": "a55fe1bc-1754-45fa-9adc-cf3d6d7c377a", "createdAt": "2020-11-05T19:00:16.268Z", "createdBy": "a55fe1bc-1754-45fa-9adc-cf3d6d7c377a" } -} +} \ No newline at end of file diff --git a/test/messages/taas.workperiod.create.event.json b/test/messages/taas.workperiod.create.event.json index 3c2d286..eb26743 100644 --- a/test/messages/taas.workperiod.create.event.json +++ b/test/messages/taas.workperiod.create.event.json @@ -4,7 +4,7 @@ "timestamp": "2021-03-30T20:24:17.555Z", "mime-type": "application/json", "payload": { - "resourceBookingId": "6cf2edf6-4b2c-40ef-96db-e1ddb771fdd3", + "resourceBookingId": "60d97713-8621-476e-b006-7cb9589c7777", "startDate": "2021-03-14", "endDate": "2021-03-20", "daysWorked": 3, @@ -14,10 +14,9 @@ "projectId": 111, "userHandle": "pshah_manager", "id": "926040c4-1709-4de2-b2b6-52adf6e5e72d", - "billingAccountId": 80000071 "createdBy": "00000000-0000-0000-0000-000000000000", "updatedAt": "2021-03-30T20:24:17.541Z", "createdAt": "2021-03-30T20:24:17.541Z", "updatedBy": null } -} +} \ No newline at end of file diff --git a/test/messages/taas.workperiod.update.event.json b/test/messages/taas.workperiod.update.event.json index 0e8ca71..e8798dd 100644 --- a/test/messages/taas.workperiod.update.event.json +++ b/test/messages/taas.workperiod.update.event.json @@ -4,18 +4,19 @@ "timestamp": "2021-03-30T20:13:53.179Z", "mime-type": "application/json", "payload": { - "id": "926040c4-1709-4de2-b2b6-52adf6e5e72d", - "resourceBookingId": "79317ff6-5b30-45c2-ace8-b97282b042a8", - "startDate": "2021-03-14", - "endDate": "2021-03-20", - "daysWorked": 3, + "resourceBookingId": "60d97713-8621-476e-b006-7cb9589c7777", + "startDate": "2021-03-21", + "endDate": "2021-03-28", + "daysWorked": 4, "memberRate": 13.13, "customerRate": 13.13, - "paymentStatus": "pending", + "paymentStatus": "cancelled", "projectId": 111, "userHandle": "pshah_manager", + "id": "926040c4-1709-4de2-b2b6-52adf6e5e72d", "createdBy": "00000000-0000-0000-0000-000000000000", - "createdAt": "2021-03-30T20:13:34.670Z", - "updatedAt": "2021-03-30T20:13:45.354Z" + "updatedAt": "2021-03-30T20:24:17.541Z", + "createdAt": "2021-03-30T20:24:17.541Z", + "updatedBy": "00000000-0000-0000-0000-000000000000" } } \ No newline at end of file diff --git a/test/messages/taas.workperiodpayment.create.event.json b/test/messages/taas.workperiodpayment.create.event.json index 25ab9cc..1ecbc84 100644 --- a/test/messages/taas.workperiodpayment.create.event.json +++ b/test/messages/taas.workperiodpayment.create.event.json @@ -1 +1,18 @@ -{"topic":"taas.workperiodpayment.create","originator":"taas-api","timestamp":"2021-04-09T20:10:33.770Z","mime-type":"application/json","payload":{"challengeId":"00000000-0000-0000-0000-000000000000","workPeriodId":"140b7407-540d-40c3-ad23-905d932aa9c8","amount":600,"status":"completed","id":"09c80ee6-21be-45a4-9c3c-7ec4c75ece79","createdBy":"57646ff9-1cd3-4d3c-88ba-eb09a395366c","updatedAt":"2021-04-09T20:10:33.755Z","createdAt":"2021-04-09T20:10:33.755Z","updatedBy":null}} \ No newline at end of file +{ + "topic": "taas.workperiodpayment.create", + "originator": "taas-api", + "timestamp": "2021-04-09T20:10:33.770Z", + "mime-type": "application/json", + "payload": { + "challengeId": "00000000-0000-0000-0000-000000000000", + "workPeriodId": "926040c4-1709-4de2-b2b6-52adf6e5e72d", + "amount": 600, + "status": "completed", + "id": "09c80ee6-21be-45a4-9c3c-7ec4c75ece79", + "billingAccountId": 80000071, + "createdBy": "57646ff9-1cd3-4d3c-88ba-eb09a395366c", + "updatedAt": "2021-04-09T20:10:33.755Z", + "createdAt": "2021-04-09T20:10:33.755Z", + "updatedBy": null + } +} \ No newline at end of file diff --git a/test/messages/taas.workperiodpayment.update.event.json b/test/messages/taas.workperiodpayment.update.event.json index 66e5bce..ea1e859 100644 --- a/test/messages/taas.workperiodpayment.update.event.json +++ b/test/messages/taas.workperiodpayment.update.event.json @@ -1 +1,18 @@ -{"topic":"taas.workperiodpayment.update","originator":"taas-api","timestamp":"2021-04-09T20:12:26.994Z","mime-type":"application/json","payload":{"id":"09c80ee6-21be-45a4-9c3c-7ec4c75ece79","workPeriodId":"140b7407-540d-40c3-ad23-905d932aa9c8","challengeId":"00000000-0000-0000-0000-000000000000","amount":1600,"status":"completed","createdBy":"57646ff9-1cd3-4d3c-88ba-eb09a395366c","updatedBy":"57646ff9-1cd3-4d3c-88ba-eb09a395366c","createdAt":"2021-04-09T20:10:33.755Z","updatedAt":"2021-04-09T20:12:26.966Z"}} \ No newline at end of file +{ + "topic": "taas.workperiodpayment.update", + "originator": "taas-api", + "timestamp": "2021-04-09T20:12:26.994Z", + "mime-type": "application/json", + "payload": { + "id": "09c80ee6-21be-45a4-9c3c-7ec4c75ece79", + "workPeriodId": "926040c4-1709-4de2-b2b6-52adf6e5e72d", + "challengeId": "00000000-0000-0000-0000-000000000000", + "amount": 1600, + "status": "completed", + "billingAccountId": 80000071, + "createdBy": "57646ff9-1cd3-4d3c-88ba-eb09a395366c", + "updatedBy": "57646ff9-1cd3-4d3c-88ba-eb09a395366c", + "createdAt": "2021-04-09T20:10:33.755Z", + "updatedAt": "2021-04-09T20:12:26.966Z" + } +} \ No newline at end of file diff --git a/test/unit/prepare.js b/test/unit/prepare.js index 320b0a8..f7966ed 100644 --- a/test/unit/prepare.js +++ b/test/unit/prepare.js @@ -32,7 +32,7 @@ prepare(function (done) { .reply((uri, body) => { const id = idFromUri(uri) if (testData.esStorage.content[id]) { - testData.esStorage.content[id] = body.doc + _.assign(testData.esStorage.content[id], body.doc) return [200] } else { return [404] @@ -59,6 +59,31 @@ prepare(function (done) { return [404] } }) + .post(uri => uri.includes('_search')) + .query(true) + .reply(uri => { + if (Object.keys(testData.esStorage.content).length > 0) { + return [200, { + hits: { + total: { + value: 1 + }, + hits: [{ + _source: testData.esStorage.content[Object.keys(testData.esStorage.content)[0]] + }] + } + } ] + } else { + return [200, { + hits: { + total: { + value: 0 + }, + hits: [] + } + } ] + } + }) done() }, function (done) { nock.cleanAll() diff --git a/test/unit/test.js b/test/unit/test.js index 6f9ca67..d2d3f66 100644 --- a/test/unit/test.js +++ b/test/unit/test.js @@ -15,7 +15,9 @@ const constants = require('../../src/common/constants') const services = { JobProcessorService: require('../../src/services/JobProcessorService'), JobCandidateProcessorService: require('../../src/services/JobCandidateProcessorService'), - ResourceBookingProcessorService: require('../../src/services/ResourceBookingProcessorService') + ResourceBookingProcessorService: require('../../src/services/ResourceBookingProcessorService'), + WorkPeriodProcessorService: require('../../src/services/WorkPeriodProcessorService'), + WorkPeriodPaymentProcessorService: require('../../src/services/WorkPeriodPaymentProcessorService') } // random transaction id here @@ -48,58 +50,133 @@ describe('General Logic Tests', () => { sandbox.restore() }) + describe('Job, JobCandidate, ResourceBooking tests', () => { + for (const [index, model] of [ + [config.esConfig.ES_INDEX_JOB, 'Job'], + [config.esConfig.ES_INDEX_JOB_CANDIDATE, 'JobCandidate'], + [config.esConfig.ES_INDEX_RESOURCE_BOOKING, 'ResourceBooking'] + ]) { + const modelInSpaceCase = stringcase.spacecase(model) + it(`processCreate - ${modelInSpaceCase} success`, async () => { + await services[`${model}ProcessorService`].processCreate(testData.messages[model].create.message, transactionId) + should.deepEqual( + testData.esStorage.content[testData.messages[model].create.message.payload.id], + testData.messages[model].create.message.payload + ) + }) + + it(`processUpdate - ${modelInSpaceCase} success`, async () => { + await testHelper.esClient.create({ + index, + id: testData.messages[model].create.message.payload.id, + body: testData.messages[model].create.message.payload, + refresh: 'true' + }) + await services[`${model}ProcessorService`].processUpdate(testData.messages[model].update.message, transactionId) + should.deepEqual( + testData.esStorage.content[testData.messages[model].create.message.payload.id], + testData.messages[model].update.message.payload + ) + }) + + it(`processDelete - ${modelInSpaceCase} success`, async () => { + await testHelper.esClient.create({ + index, + id: testData.messages[model].create.message.payload.id, + body: testData.messages[model].create.message.payload, + refresh: 'true' + }) + await services[`${model}ProcessorService`].processDelete(testData.messages[model].delete.message, transactionId) + should.not.exist(testData.esStorage.content[testData.messages[model].create.message.payload.id]) + }) - for (const [index, model] of [ - [config.esConfig.ES_INDEX_JOB, 'Job'], - [config.esConfig.ES_INDEX_JOB_CANDIDATE, 'JobCandidate'], - [config.esConfig.ES_INDEX_RESOURCE_BOOKING, 'ResourceBooking'] - ]) { + it(`Failure - processCreate - ${modelInSpaceCase} already exists`, async () => { + await testHelper.esClient.create({ + index, + id: testData.messages[model].create.message.payload.id, + body: testData.messages[model].create.message.payload, + refresh: 'true' + }) + try { + await services[`${model}ProcessorService`].processCreate(testData.messages[model].create.message, transactionId) + throw new Error() + } catch (err) { + should.equal(err.message, `id: ${testData.messages[model].create.message.payload.id} "${index}" already exists`) + } + }) + + it(`Failure - processUpdate - ${modelInSpaceCase} not found`, async () => { + try { + await services[`${model}ProcessorService`].processUpdate(testData.messages[model].update.message, transactionId) + throw new Error() + } catch (err) { + should.equal(err.message, `id: ${testData.messages[model].update.message.payload.id} "${index}" not found`) + } + }) + + it(`Failure - processDelete - ${modelInSpaceCase} not found`, async () => { + try { + await services[`${model}ProcessorService`].processDelete(testData.messages[model].delete.message, transactionId) + throw new Error() + } catch (err) { + should.equal(err.message, `id: ${testData.messages[model].delete.message.payload.id} "${index}" not found`) + } + }) + } + }) + describe('Nested WorkPeriod tests', () => { + const index = config.esConfig.ES_INDEX_RESOURCE_BOOKING + const model = 'WorkPeriod' + const parentModel = 'ResourceBooking' + const nestedName = 'workPeriods' const modelInSpaceCase = stringcase.spacecase(model) - it('processCreate - success', async () => { + it(`processCreate - ${modelInSpaceCase} success`, async () => { + await testHelper.esClient.create({ + index, + id: testData.messages[parentModel].create.message.payload.id, + body: testData.messages[parentModel].create.message.payload, + refresh: 'true' + }) await services[`${model}ProcessorService`].processCreate(testData.messages[model].create.message, transactionId) should.deepEqual( - testData.esStorage.content[testData.messages[model].create.message.payload.id], - testData.messages[model].create.message.payload + testData.esStorage.content[testData.messages[parentModel].create.message.payload.id], + _.assign(testData.messages[parentModel].create.message.payload, { [nestedName]: [testData.messages[model].create.message.payload] }) ) }) - it('processUpdate - success', async () => { + it(`processUpdate - ${modelInSpaceCase} success`, async () => { await testHelper.esClient.create({ index, - id: testData.messages[model].create.message.payload.id, - body: testData.messages[model].create.message.payload, + id: testData.messages[parentModel].create.message.payload.id, + body: _.assign(testData.messages[parentModel].create.message.payload, { [nestedName]: [testData.messages[model].create.message.payload] }), refresh: 'true' }) await services[`${model}ProcessorService`].processUpdate(testData.messages[model].update.message, transactionId) should.deepEqual( - testData.esStorage.content[testData.messages[model].create.message.payload.id], - testData.messages[model].update.message.payload + testData.esStorage.content[testData.messages[parentModel].create.message.payload.id], + _.assign(testData.messages[parentModel].create.message.payload, { [nestedName]: [testData.messages[model].update.message.payload] }) ) }) - it('processDelete - success', async () => { + it(`processDelete - ${modelInSpaceCase} success`, async () => { await testHelper.esClient.create({ index, - id: testData.messages[model].create.message.payload.id, - body: testData.messages[model].create.message.payload, + id: testData.messages[parentModel].create.message.payload.id, + body: _.assign(testData.messages[parentModel].create.message.payload, { [nestedName]: [testData.messages[model].create.message.payload] }), refresh: 'true' }) await services[`${model}ProcessorService`].processDelete(testData.messages[model].delete.message, transactionId) - should.not.exist(testData.esStorage.content[testData.messages[model].create.message.payload.id]) + should.deepEqual( + testData.esStorage.content[testData.messages[parentModel].create.message.payload.id], + _.assign(testData.messages[parentModel].create.message.payload, { [nestedName]: [] }) + ) }) - - it(`Failure - processCreate - ${modelInSpaceCase} already exists`, async () => { - await testHelper.esClient.create({ - index, - id: testData.messages[model].create.message.payload.id, - body: testData.messages[model].create.message.payload, - refresh: 'true' - }) + it(`Failure - processCreate - ${modelInSpaceCase} not found`, async () => { try { await services[`${model}ProcessorService`].processCreate(testData.messages[model].create.message, transactionId) throw new Error() } catch (err) { - should.equal(err.message, `id: ${testData.messages[model].create.message.payload.id} "${index}" already exists`) + should.equal(err.message, `id: ${testData.messages[parentModel].create.message.payload.id} "${index}" not found`) } }) @@ -108,7 +185,7 @@ describe('General Logic Tests', () => { await services[`${model}ProcessorService`].processUpdate(testData.messages[model].update.message, transactionId) throw new Error() } catch (err) { - should.equal(err.message, `id: ${testData.messages[model].update.message.payload.id} "${index}" not found`) + should.equal(err.message, `id: ${testData.messages[model].update.message.payload.id} "${model}" not found`) } }) @@ -117,10 +194,64 @@ describe('General Logic Tests', () => { await services[`${model}ProcessorService`].processDelete(testData.messages[model].delete.message, transactionId) throw new Error() } catch (err) { - should.equal(err.message, `id: ${testData.messages[model].delete.message.payload.id} "${index}" not found`) + should.equal(err.message, `id: ${testData.messages[model].delete.message.payload.id} "${model}" not found`) } }) - } + }) + describe('Nested WorkPeriodPayment tests', () => { + const index = config.esConfig.ES_INDEX_RESOURCE_BOOKING + const model = 'WorkPeriodPayment' + const parentModel = 'WorkPeriod' + const rootModel = 'ResourceBooking' + const nestedName = 'payments' + const parentNestedName = 'workPeriods' + const modelInSpaceCase = stringcase.spacecase(model) + it(`processCreate - ${modelInSpaceCase} success`, async () => { + await testHelper.esClient.create({ + index, + id: testData.messages[rootModel].create.message.payload.id, + body: _.assign(testData.messages[rootModel].create.message.payload, { [parentNestedName]: [testData.messages[parentModel].create.message.payload] }), + refresh: 'true' + }) + await services[`${model}ProcessorService`].processCreate(testData.messages[model].create.message, transactionId) + should.deepEqual( + testData.esStorage.content[testData.messages[rootModel].create.message.payload.id], + _.assign(testData.messages[rootModel].create.message.payload, { [parentNestedName]: [_.assign(testData.messages[parentModel].create.message.payload, { [nestedName]: [testData.messages[model].create.message.payload] })] }) + ) + }) + + it(`processUpdate - ${modelInSpaceCase} success`, async () => { + await testHelper.esClient.create({ + index, + id: testData.messages[rootModel].create.message.payload.id, + body: _.assign(testData.messages[rootModel].create.message.payload, { [parentNestedName]: [_.assign(testData.messages[parentModel].create.message.payload, { [nestedName]: [testData.messages[model].create.message.payload] })] }), + refresh: 'true' + }) + await services[`${model}ProcessorService`].processUpdate(testData.messages[model].update.message, transactionId) + should.deepEqual( + testData.esStorage.content[testData.messages[rootModel].create.message.payload.id], + _.assign(testData.messages[rootModel].create.message.payload, { [parentNestedName]: [_.assign(testData.messages[parentModel].create.message.payload, { [nestedName]: [testData.messages[model].update.message.payload] })] }) + ) + }) + + it(`Failure - processCreate - ${modelInSpaceCase} not found`, async () => { + try { + await services[`${model}ProcessorService`].processCreate(testData.messages[model].create.message, transactionId) + throw new Error() + } catch (err) { + should.equal(err.message, `id: ${testData.messages[parentModel].create.message.payload.id} "${parentModel}" not found`) + } + }) + + it(`Failure - processUpdate - ${modelInSpaceCase} not found`, async () => { + try { + await services[`${model}ProcessorService`].processUpdate(testData.messages[model].update.message, transactionId) + throw new Error() + } catch (err) { + should.equal(err.message, `id: ${testData.messages[model].update.message.payload.id} "${model}" not found`) + } + }) + }) }) describe('Zapier Logic Tests', () => { @@ -158,10 +289,10 @@ describe('Zapier Logic Tests', () => { }) describe('Job Candidate Update', () => { - it('should post to Zapier if status is changed to "rejected"', async () => { + it('should post to Zapier if status is changed to "client rejected - screening"', async () => { const previousData = _.assign({}, testData.messages.JobCandidate.create.message.payload, { status: 'open', externalId: '123' }) const updateMessage = _.assign({}, testData.messages.JobCandidate.update.message, { - payload: _.assign({}, testData.messages.JobCandidate.update.message.payload, { status: 'rejected', externalId: '123' }) + payload: _.assign({}, testData.messages.JobCandidate.update.message.payload, { status: 'client rejected - screening', externalId: '123' }) }) await testHelper.esClient.create({ @@ -181,10 +312,10 @@ describe('Zapier Logic Tests', () => { helper.postMessageViaWebhook.callCount.should.equal(1) }) - it('should post to Zapier if status is changed to "shortlist"', async () => { + it('should post to Zapier if status is changed to "interview"', async () => { const previousData = _.assign({}, testData.messages.JobCandidate.create.message.payload, { status: 'open', externalId: '123' }) const updateMessage = _.assign({}, testData.messages.JobCandidate.update.message, { - payload: _.assign({}, testData.messages.JobCandidate.update.message.payload, { status: 'shortlist', externalId: '123' }) + payload: _.assign({}, testData.messages.JobCandidate.update.message.payload, { status: 'interview', externalId: '123' }) }) await testHelper.esClient.create({ @@ -204,10 +335,10 @@ describe('Zapier Logic Tests', () => { helper.postMessageViaWebhook.callCount.should.equal(1) }) - it('should not post to Zapier if status was already "rejected"', async () => { - const previousData = _.assign({}, testData.messages.JobCandidate.create.message.payload, { status: 'rejected', externalId: '123' }) + it('should not post to Zapier if status was already "client rejected - screening"', async () => { + const previousData = _.assign({}, testData.messages.JobCandidate.create.message.payload, { status: 'client rejected - screening', externalId: '123' }) const updateMessage = _.assign({}, testData.messages.JobCandidate.update.message, { - payload: _.assign({}, testData.messages.JobCandidate.update.message.payload, { status: 'rejected', externalId: '123' }) + payload: _.assign({}, testData.messages.JobCandidate.update.message.payload, { status: 'client rejected - screening', externalId: '123' }) }) await testHelper.esClient.create({ @@ -227,10 +358,10 @@ describe('Zapier Logic Tests', () => { helper.postMessageViaWebhook.callCount.should.equal(0) }) - it('should not post to Zapier if status was already "shortlist"', async () => { - const previousData = _.assign({}, testData.messages.JobCandidate.create.message.payload, { status: 'shortlist', externalId: '123' }) + it('should not post to Zapier if status was already "interview"', async () => { + const previousData = _.assign({}, testData.messages.JobCandidate.create.message.payload, { status: 'interview', externalId: '123' }) const updateMessage = _.assign({}, testData.messages.JobCandidate.update.message, { - payload: _.assign({}, testData.messages.JobCandidate.update.message.payload, { status: 'shortlist', externalId: '123' }) + payload: _.assign({}, testData.messages.JobCandidate.update.message.payload, { status: 'interview', externalId: '123' }) }) await testHelper.esClient.create({ @@ -250,10 +381,10 @@ describe('Zapier Logic Tests', () => { helper.postMessageViaWebhook.callCount.should.equal(0) }) - it('should not post to Zapier if status is changed to "interview" (not "rejected" or "shortlist")', async () => { + it('should not post to Zapier if status is changed to "placed" (not "rejected" or "shortlist")', async () => { const previousData = _.assign({}, testData.messages.JobCandidate.create.message.payload, { status: 'open', externalId: '123' }) const updateMessage = _.assign({}, testData.messages.JobCandidate.update.message, { - payload: _.assign({}, testData.messages.JobCandidate.update.message.payload, { status: 'interview', externalId: '123' }) + payload: _.assign({}, testData.messages.JobCandidate.update.message.payload, { status: 'placed', externalId: '123' }) }) await testHelper.esClient.create({ From d9aff64a836d94226019abc1cd44fac366712311 Mon Sep 17 00:00:00 2001 From: urwithat Date: Tue, 11 May 2021 21:36:31 +0530 Subject: [PATCH 18/55] Changed xaiId & templateId from UUID to String --- src/services/InterviewProcessorService.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/services/InterviewProcessorService.js b/src/services/InterviewProcessorService.js index 0879589..0994b5a 100644 --- a/src/services/InterviewProcessorService.js +++ b/src/services/InterviewProcessorService.js @@ -58,11 +58,11 @@ processRequestInterview.schema = { 'mime-type': Joi.string().required(), payload: Joi.object().keys({ id: Joi.string().uuid().required(), - xaiId: Joi.string().uuid().allow(null), + xaiId: Joi.string().allow(null), jobCandidateId: Joi.string().uuid().required(), calendarEventId: Joi.string().allow(null), templateUrl: Joi.xaiTemplate().required(), - templateId: Joi.string().uuid().allow(null), + templateId: Joi.string().allow(null), templateType: Joi.string().allow(null), title: Joi.string().uuid().allow(null), locationDetails: Joi.string().uuid().allow(null), From 151c253cec5695af4ef8398c9a91b2a9696fe197 Mon Sep 17 00:00:00 2001 From: urwithat Date: Tue, 11 May 2021 21:50:12 +0530 Subject: [PATCH 19/55] Bug Fix title was GUID --- src/services/InterviewProcessorService.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/InterviewProcessorService.js b/src/services/InterviewProcessorService.js index 0994b5a..b3ae8e2 100644 --- a/src/services/InterviewProcessorService.js +++ b/src/services/InterviewProcessorService.js @@ -64,7 +64,7 @@ processRequestInterview.schema = { templateUrl: Joi.xaiTemplate().required(), templateId: Joi.string().allow(null), templateType: Joi.string().allow(null), - title: Joi.string().uuid().allow(null), + title: Joi.string().allow(null), locationDetails: Joi.string().uuid().allow(null), round: Joi.number().integer().positive().required(), duration: Joi.number().integer().positive().required(), From 2446ba37aefbb57fdcfc4bdb1c67816b438406af Mon Sep 17 00:00:00 2001 From: urwithat Date: Tue, 11 May 2021 21:59:54 +0530 Subject: [PATCH 20/55] Bug Fix locationDetails was GUID --- src/services/InterviewProcessorService.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/InterviewProcessorService.js b/src/services/InterviewProcessorService.js index b3ae8e2..da49774 100644 --- a/src/services/InterviewProcessorService.js +++ b/src/services/InterviewProcessorService.js @@ -65,7 +65,7 @@ processRequestInterview.schema = { templateId: Joi.string().allow(null), templateType: Joi.string().allow(null), title: Joi.string().allow(null), - locationDetails: Joi.string().uuid().allow(null), + locationDetails: Joi.string().allow(null), round: Joi.number().integer().positive().required(), duration: Joi.number().integer().positive().required(), startTimestamp: Joi.date().allow(null), From 4fbacc53794c6c97e75b78001d894824e8939eba Mon Sep 17 00:00:00 2001 From: urwithat Date: Wed, 12 May 2021 19:09:02 +0530 Subject: [PATCH 21/55] Added updatedBy for Interviews --- src/scripts/createIndex.js | 3 ++- src/services/InterviewProcessorService.js | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/scripts/createIndex.js b/src/scripts/createIndex.js index 84325e2..987df1b 100644 --- a/src/scripts/createIndex.js +++ b/src/scripts/createIndex.js @@ -71,7 +71,8 @@ async function createIndex () { createdAt: { type: 'date' }, createdBy: { type: 'keyword' }, updatedAt: { type: 'date' }, - updatedBy: { type: 'keyword' } + updatedBy: { type: 'keyword' }, + deletedAt: { type: 'date' } } }, createdAt: { type: 'date' }, diff --git a/src/services/InterviewProcessorService.js b/src/services/InterviewProcessorService.js index da49774..b7d2e2e 100644 --- a/src/services/InterviewProcessorService.js +++ b/src/services/InterviewProcessorService.js @@ -79,7 +79,8 @@ processRequestInterview.schema = { createdAt: Joi.date().required(), createdBy: Joi.string().uuid().required(), updatedAt: Joi.date().allow(null), - updatedBy: Joi.string().uuid().allow(null) + updatedBy: Joi.string().uuid().allow(null), + deletedAt: Joi.date().allow(null) }).required() }).required(), transactionId: Joi.string().required() From 9502b1be5ffafd8f470223b8ac7caf2fb8991b3c Mon Sep 17 00:00:00 2001 From: eisbilir Date: Wed, 19 May 2021 13:13:19 +0300 Subject: [PATCH 22/55] change refresh option for WP --- src/services/WorkPeriodProcessorService.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/services/WorkPeriodProcessorService.js b/src/services/WorkPeriodProcessorService.js index 59b8d4b..858cc85 100644 --- a/src/services/WorkPeriodProcessorService.js +++ b/src/services/WorkPeriodProcessorService.js @@ -34,7 +34,7 @@ async function processCreate (message, transactionId) { body: { doc: { workPeriods } }, - refresh: constants.esRefreshOption + refresh: true }) } @@ -144,7 +144,7 @@ async function processUpdate (message, transactionId) { body: { doc: { workPeriods } }, - refresh: constants.esRefreshOption + refresh: true }) } @@ -184,7 +184,7 @@ async function processDelete (message, transactionId) { body: { doc: { workPeriods } }, - refresh: constants.esRefreshOption + refresh: true }) } From 0238cac93a05941b5f70805ba4d0f1a1c9281e7f Mon Sep 17 00:00:00 2001 From: Maksym Mykhailenko Date: Wed, 19 May 2021 15:49:18 +0300 Subject: [PATCH 23/55] fix: refresh true --- src/services/WorkPeriodProcessorService.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/services/WorkPeriodProcessorService.js b/src/services/WorkPeriodProcessorService.js index 858cc85..d57c4fe 100644 --- a/src/services/WorkPeriodProcessorService.js +++ b/src/services/WorkPeriodProcessorService.js @@ -34,7 +34,7 @@ async function processCreate (message, transactionId) { body: { doc: { workPeriods } }, - refresh: true + refresh: 'true' }) } @@ -144,7 +144,7 @@ async function processUpdate (message, transactionId) { body: { doc: { workPeriods } }, - refresh: true + refresh: 'true' }) } @@ -184,7 +184,7 @@ async function processDelete (message, transactionId) { body: { doc: { workPeriods } }, - refresh: true + refresh: 'true' }) } From 5d43357da4041957e174de6ba754409bdd391fbf Mon Sep 17 00:00:00 2001 From: Maksym Mykhailenko Date: Wed, 19 May 2021 16:33:28 +0300 Subject: [PATCH 24/55] chore: add consumer handler debug log --- src/app.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/app.js b/src/app.js index 5ebb63c..eb87bc9 100644 --- a/src/app.js +++ b/src/app.js @@ -153,6 +153,7 @@ async function initConsumer () { subscriptions: topics, handler: async (messageSet, topic, partition) => { eventEmitter.emit('start_handling_message') + localLogger.debug(`Consumer handler. Topic: ${topic}, partition: ${partition}, message set length: ${messageSet.length}`) await dataHandler(messageSet, topic, partition) eventEmitter.emit('end_handling_message') } From 642d41c531cf1f1d830ce713060913a3714cdee0 Mon Sep 17 00:00:00 2001 From: nkumar-topcoder <33625707+nkumar-topcoder@users.noreply.github.com> Date: Wed, 19 May 2021 19:52:33 +0530 Subject: [PATCH 25/55] Update WorkPeriodProcessorService.js --- src/services/WorkPeriodProcessorService.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/services/WorkPeriodProcessorService.js b/src/services/WorkPeriodProcessorService.js index d57c4fe..6664eab 100644 --- a/src/services/WorkPeriodProcessorService.js +++ b/src/services/WorkPeriodProcessorService.js @@ -27,6 +27,7 @@ async function processCreate (message, transactionId) { // Append new workPeriod workPeriods.push(workPeriod) // Update ResourceBooking's workPeriods property + console.log("workperiod value-999 : ", workPeriod) await esClient.updateExtra({ index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), id: workPeriod.resourceBookingId, From 9417170a60dd6551d46c9031d83113adabcc644a Mon Sep 17 00:00:00 2001 From: Maksym Mykhailenko Date: Wed, 19 May 2021 18:06:47 +0300 Subject: [PATCH 26/55] chore: add log before adding WP to RB --- src/services/WorkPeriodProcessorService.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/services/WorkPeriodProcessorService.js b/src/services/WorkPeriodProcessorService.js index 6664eab..1cb55c6 100644 --- a/src/services/WorkPeriodProcessorService.js +++ b/src/services/WorkPeriodProcessorService.js @@ -22,12 +22,13 @@ async function processCreate (message, transactionId) { index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), id: workPeriod.resourceBookingId }) + console.log(`[RB value-999] before update: ${JSON.stringify(resourceBooking)}`) // Get ResourceBooking's existing workPeriods const workPeriods = _.isArray(resourceBooking.body.workPeriods) ? resourceBooking.body.workPeriods : [] // Append new workPeriod workPeriods.push(workPeriod) // Update ResourceBooking's workPeriods property - console.log("workperiod value-999 : ", workPeriod) + console.log(`[WP value-999]: ${JSON.stringify(workPeriod)}`) await esClient.updateExtra({ index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), id: workPeriod.resourceBookingId, From f51bb088b76fc28eb7c5d736231a7a8f4ebbbeb8 Mon Sep 17 00:00:00 2001 From: Maksym Mykhailenko Date: Wed, 19 May 2021 18:09:02 +0300 Subject: [PATCH 27/55] fix: use transactionId when get data from ES --- src/services/WorkPeriodProcessorService.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/services/WorkPeriodProcessorService.js b/src/services/WorkPeriodProcessorService.js index 1cb55c6..994b323 100644 --- a/src/services/WorkPeriodProcessorService.js +++ b/src/services/WorkPeriodProcessorService.js @@ -20,6 +20,7 @@ async function processCreate (message, transactionId) { // Find related resourceBooking const resourceBooking = await esClient.getExtra({ index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), + transactionId, id: workPeriod.resourceBookingId }) console.log(`[RB value-999] before update: ${JSON.stringify(resourceBooking)}`) @@ -76,6 +77,7 @@ async function processUpdate (message, transactionId) { // find workPeriod in it's parent ResourceBooking let resourceBooking = await esClient.search({ index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), + transactionId, body: { query: { nested: { @@ -106,11 +108,12 @@ async function processUpdate (message, transactionId) { body: { doc: { workPeriods } }, - refresh: constants.esRefreshOption + refresh: 'true' }) // find workPeriod's new parent ResourceBooking resourceBooking = await esClient.getExtra({ index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), + transactionId, id: data.resourceBookingId }) // Get ResourceBooking's existing workPeriods @@ -127,7 +130,7 @@ async function processUpdate (message, transactionId) { body: { doc: { workPeriods } }, - refresh: constants.esRefreshOption + refresh: 'true' }) return } @@ -162,6 +165,7 @@ async function processDelete (message, transactionId) { // Find related ResourceBooking const resourceBooking = await esClient.search({ index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), + transactionId, body: { query: { nested: { From 471d097211c74c2598a3a410f60fbd2d8b5886b0 Mon Sep 17 00:00:00 2001 From: Maksym Mykhailenko Date: Thu, 20 May 2021 11:48:50 +0300 Subject: [PATCH 28/55] Revert "fix: refresh true" This reverts commit 0238cac93a05941b5f70805ba4d0f1a1c9281e7f. --- src/services/WorkPeriodProcessorService.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/services/WorkPeriodProcessorService.js b/src/services/WorkPeriodProcessorService.js index 994b323..d34f4ba 100644 --- a/src/services/WorkPeriodProcessorService.js +++ b/src/services/WorkPeriodProcessorService.js @@ -37,7 +37,7 @@ async function processCreate (message, transactionId) { body: { doc: { workPeriods } }, - refresh: 'true' + refresh: true }) } @@ -149,7 +149,7 @@ async function processUpdate (message, transactionId) { body: { doc: { workPeriods } }, - refresh: 'true' + refresh: true }) } @@ -190,7 +190,7 @@ async function processDelete (message, transactionId) { body: { doc: { workPeriods } }, - refresh: 'true' + refresh: true }) } From 960d2643b8fa08a3ad31868a2deebba14d81f464 Mon Sep 17 00:00:00 2001 From: Maksym Mykhailenko Date: Thu, 20 May 2021 11:48:55 +0300 Subject: [PATCH 29/55] Revert "Merge pull request #58 from eisbilir/dev" This reverts commit 0428f230ba847a54955d06ca9f91ae7091dbe440, reversing changes made to 9cc0ef1742641dc01460ee1087aa3d91ae195a2e. --- src/services/WorkPeriodProcessorService.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/services/WorkPeriodProcessorService.js b/src/services/WorkPeriodProcessorService.js index d34f4ba..6d12e00 100644 --- a/src/services/WorkPeriodProcessorService.js +++ b/src/services/WorkPeriodProcessorService.js @@ -37,7 +37,7 @@ async function processCreate (message, transactionId) { body: { doc: { workPeriods } }, - refresh: true + refresh: constants.esRefreshOption }) } @@ -149,7 +149,7 @@ async function processUpdate (message, transactionId) { body: { doc: { workPeriods } }, - refresh: true + refresh: constants.esRefreshOption }) } @@ -190,7 +190,7 @@ async function processDelete (message, transactionId) { body: { doc: { workPeriods } }, - refresh: true + refresh: constants.esRefreshOption }) } From 87ede1612b7ae7d84c0a885516b90ca25808cbe3 Mon Sep 17 00:00:00 2001 From: Maksym Mykhailenko Date: Thu, 20 May 2021 11:50:50 +0300 Subject: [PATCH 30/55] fix: revert 'true' to esRefreshOption --- src/services/WorkPeriodProcessorService.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/services/WorkPeriodProcessorService.js b/src/services/WorkPeriodProcessorService.js index 6d12e00..770f774 100644 --- a/src/services/WorkPeriodProcessorService.js +++ b/src/services/WorkPeriodProcessorService.js @@ -108,7 +108,7 @@ async function processUpdate (message, transactionId) { body: { doc: { workPeriods } }, - refresh: 'true' + refresh: constants.esRefreshOption }) // find workPeriod's new parent ResourceBooking resourceBooking = await esClient.getExtra({ @@ -130,7 +130,7 @@ async function processUpdate (message, transactionId) { body: { doc: { workPeriods } }, - refresh: 'true' + refresh: constants.esRefreshOption }) return } From 0f7fa8162a5a9e09e2677d936395f0126648556e Mon Sep 17 00:00:00 2001 From: Sushil Shinde Date: Mon, 24 May 2021 15:22:56 +0530 Subject: [PATCH 31/55] added statuses to the job candidates --- src/bootstrap.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap.js b/src/bootstrap.js index c1df698..be0d1c8 100644 --- a/src/bootstrap.js +++ b/src/bootstrap.js @@ -12,7 +12,7 @@ global.Promise = require('bluebird') Joi.rateType = () => Joi.string().valid('hourly', 'daily', 'weekly', 'monthly') Joi.jobStatus = () => Joi.string().valid('sourcing', 'in-review', 'assigned', 'closed', 'cancelled') Joi.resourceBookingStatus = () => Joi.string().valid('placed', 'closed', 'cancelled') -Joi.jobCandidateStatus = () => Joi.string().valid('open', 'placed', 'selected', 'client rejected - screening', 'client rejected - interview', 'rejected - other', 'cancelled', 'interview', 'topcoder-rejected') +Joi.jobCandidateStatus = () => Joi.string().valid('open', 'placed', 'selected', 'client rejected - screening', 'client rejected - interview', 'rejected - other', 'cancelled', 'interview', 'topcoder-rejected','applied','rejected-pre-screen','skills-test','skills-test','phone-screen','job-closed') Joi.workload = () => Joi.string().valid('full-time', 'fractional') Joi.title = () => Joi.string().max(128) Joi.paymentStatus = () => Joi.string().valid('pending', 'partially-completed', 'completed', 'cancelled') From c7a47a2ca365786e373fa8ac0d59dcb4d3ab7fa8 Mon Sep 17 00:00:00 2001 From: xxcxy Date: Wed, 26 May 2021 23:22:56 +0800 Subject: [PATCH 32/55] Payments - Batch Endpoints --- src/bootstrap.js | 4 ++-- src/services/WorkPeriodPaymentProcessorService.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/bootstrap.js b/src/bootstrap.js index be0d1c8..005795e 100644 --- a/src/bootstrap.js +++ b/src/bootstrap.js @@ -12,13 +12,13 @@ global.Promise = require('bluebird') Joi.rateType = () => Joi.string().valid('hourly', 'daily', 'weekly', 'monthly') Joi.jobStatus = () => Joi.string().valid('sourcing', 'in-review', 'assigned', 'closed', 'cancelled') Joi.resourceBookingStatus = () => Joi.string().valid('placed', 'closed', 'cancelled') -Joi.jobCandidateStatus = () => Joi.string().valid('open', 'placed', 'selected', 'client rejected - screening', 'client rejected - interview', 'rejected - other', 'cancelled', 'interview', 'topcoder-rejected','applied','rejected-pre-screen','skills-test','skills-test','phone-screen','job-closed') +Joi.jobCandidateStatus = () => Joi.string().valid('open', 'placed', 'selected', 'client rejected - screening', 'client rejected - interview', 'rejected - other', 'cancelled', 'interview', 'topcoder-rejected', 'applied', 'rejected-pre-screen', 'skills-test', 'skills-test', 'phone-screen', 'job-closed') Joi.workload = () => Joi.string().valid('full-time', 'fractional') Joi.title = () => Joi.string().max(128) Joi.paymentStatus = () => Joi.string().valid('pending', 'partially-completed', 'completed', 'cancelled') Joi.xaiTemplate = () => Joi.string().valid(...allowedXAITemplates) Joi.interviewStatus = () => Joi.string().valid(...allowedInterviewStatuses) -Joi.workPeriodPaymentStatus = () => Joi.string().valid('completed', 'cancelled') +Joi.workPeriodPaymentStatus = () => Joi.string().valid('completed', 'scheduled', 'cancelled') // Empty string is not allowed by Joi by default and must be enabled with allow(''). // See https://joi.dev/api/?v=17.3.0#string fro details why it's like this. // In many cases we would like to allow empty string to make it easier to create UI for editing data. diff --git a/src/services/WorkPeriodPaymentProcessorService.js b/src/services/WorkPeriodPaymentProcessorService.js index 7f30b6e..736cacb 100644 --- a/src/services/WorkPeriodPaymentProcessorService.js +++ b/src/services/WorkPeriodPaymentProcessorService.js @@ -65,7 +65,7 @@ processCreate.schema = { payload: Joi.object().keys({ id: Joi.string().uuid().required(), workPeriodId: Joi.string().uuid().required(), - challengeId: Joi.string().uuid().required(), + challengeId: Joi.string().uuid().allow(null), amount: Joi.number().greater(0).allow(null), status: Joi.workPeriodPaymentStatus().required(), billingAccountId: Joi.number().allow(null), From 5ce9acabafa4f9ab4cd531f993d4f931e2c87765 Mon Sep 17 00:00:00 2001 From: Sushil Shinde Date: Thu, 27 May 2021 15:51:55 +0530 Subject: [PATCH 33/55] fix: added new 'offered' status --- src/bootstrap.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap.js b/src/bootstrap.js index be0d1c8..c3c6254 100644 --- a/src/bootstrap.js +++ b/src/bootstrap.js @@ -12,7 +12,7 @@ global.Promise = require('bluebird') Joi.rateType = () => Joi.string().valid('hourly', 'daily', 'weekly', 'monthly') Joi.jobStatus = () => Joi.string().valid('sourcing', 'in-review', 'assigned', 'closed', 'cancelled') Joi.resourceBookingStatus = () => Joi.string().valid('placed', 'closed', 'cancelled') -Joi.jobCandidateStatus = () => Joi.string().valid('open', 'placed', 'selected', 'client rejected - screening', 'client rejected - interview', 'rejected - other', 'cancelled', 'interview', 'topcoder-rejected','applied','rejected-pre-screen','skills-test','skills-test','phone-screen','job-closed') +Joi.jobCandidateStatus = () => Joi.string().valid('open', 'placed', 'selected', 'client rejected - screening', 'client rejected - interview', 'rejected - other', 'cancelled', 'interview', 'topcoder-rejected','applied','rejected-pre-screen','skills-test','skills-test','phone-screen','job-closed','offered') Joi.workload = () => Joi.string().valid('full-time', 'fractional') Joi.title = () => Joi.string().max(128) Joi.paymentStatus = () => Joi.string().valid('pending', 'partially-completed', 'completed', 'cancelled') From 7370849c6944343a0e9dbb973432aeb82b1e8601 Mon Sep 17 00:00:00 2001 From: dengjun Date: Sat, 29 May 2021 14:16:15 +0800 Subject: [PATCH 34/55] api-updates challenge:30186701 --- package.json | 2 +- src/bootstrap.js | 2 +- src/scripts/createIndex.js | 7 +++++++ src/services/JobCandidateProcessorService.js | 3 ++- src/services/JobProcessorService.js | 8 +++++++- test/messages/taas.job.create.event.json | 8 +++++++- test/messages/taas.job.update.event.json | 8 +++++++- test/messages/taas.jobcandidate.create.event.json | 3 ++- test/messages/taas.jobcandidate.update.event.json | 1 + 9 files changed, 35 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index f857c4e..7bd0c50 100644 --- a/package.json +++ b/package.json @@ -60,4 +60,4 @@ "test/e2e/*.js" ] } -} \ No newline at end of file +} diff --git a/src/bootstrap.js b/src/bootstrap.js index c3c6254..82287fc 100644 --- a/src/bootstrap.js +++ b/src/bootstrap.js @@ -12,7 +12,7 @@ global.Promise = require('bluebird') Joi.rateType = () => Joi.string().valid('hourly', 'daily', 'weekly', 'monthly') Joi.jobStatus = () => Joi.string().valid('sourcing', 'in-review', 'assigned', 'closed', 'cancelled') Joi.resourceBookingStatus = () => Joi.string().valid('placed', 'closed', 'cancelled') -Joi.jobCandidateStatus = () => Joi.string().valid('open', 'placed', 'selected', 'client rejected - screening', 'client rejected - interview', 'rejected - other', 'cancelled', 'interview', 'topcoder-rejected','applied','rejected-pre-screen','skills-test','skills-test','phone-screen','job-closed','offered') +Joi.jobCandidateStatus = () => Joi.string().valid('open', 'placed', 'selected', 'client rejected - screening', 'client rejected - interview', 'rejected - other', 'cancelled', 'interview', 'topcoder-rejected', 'applied', 'rejected-pre-screen', 'skills-test', 'skills-test', 'phone-screen', 'job-closed', 'offered') Joi.workload = () => Joi.string().valid('full-time', 'fractional') Joi.title = () => Joi.string().max(128) Joi.paymentStatus = () => Joi.string().valid('pending', 'partially-completed', 'completed', 'cancelled') diff --git a/src/scripts/createIndex.js b/src/scripts/createIndex.js index aae5c10..2199dbd 100644 --- a/src/scripts/createIndex.js +++ b/src/scripts/createIndex.js @@ -28,6 +28,12 @@ async function createIndex () { skills: { type: 'keyword' }, status: { type: 'keyword' }, isApplicationPageActive: { type: 'boolean' }, + minSalary: { type: 'integer' }, + maxSalary: { type: 'integer' }, + hoursPerWeek: { type: 'integer' }, + jobLocation: { type: 'keyword' }, + jobTimezone: { type: 'keyword' }, + currency: { type: 'keyword' }, createdAt: { type: 'date' }, createdBy: { type: 'keyword' }, updatedAt: { type: 'date' }, @@ -46,6 +52,7 @@ async function createIndex () { status: { type: 'keyword' }, externalId: { type: 'keyword' }, resume: { type: 'text' }, + remark: { type: 'keyword' }, interviews: { type: 'nested', properties: { diff --git a/src/services/JobCandidateProcessorService.js b/src/services/JobCandidateProcessorService.js index 48d8660..bde7e88 100644 --- a/src/services/JobCandidateProcessorService.js +++ b/src/services/JobCandidateProcessorService.js @@ -101,7 +101,8 @@ processCreate.schema = { updatedBy: Joi.string().uuid().allow(null), status: Joi.jobCandidateStatus().required(), externalId: Joi.string().allow(null), - resume: Joi.string().uri().allow(null) + resume: Joi.string().uri().allow(null), + remark: Joi.string().allow(null) }).required() }).required(), transactionId: Joi.string().required() diff --git a/src/services/JobProcessorService.js b/src/services/JobProcessorService.js index 678a3c3..1064635 100644 --- a/src/services/JobProcessorService.js +++ b/src/services/JobProcessorService.js @@ -83,7 +83,13 @@ processCreate.schema = { updatedAt: Joi.date().allow(null), updatedBy: Joi.string().uuid().allow(null), status: Joi.jobStatus().required(), - isApplicationPageActive: Joi.boolean().required() + isApplicationPageActive: Joi.boolean().required(), + minSalary: Joi.number().integer().required(), + maxSalary: Joi.number().integer().required(), + hoursPerWeek: Joi.number().integer().required(), + jobLocation: Joi.string().required(), + jobTimezone: Joi.string().required(), + currency: Joi.string().required() }).required() }).required(), transactionId: Joi.string().required() diff --git a/test/messages/taas.job.create.event.json b/test/messages/taas.job.create.event.json index ebc5d15..0fd3ef0 100644 --- a/test/messages/taas.job.create.event.json +++ b/test/messages/taas.job.create.event.json @@ -24,6 +24,12 @@ "createdAt": "2020-11-05T19:00:16.268Z", "createdBy": "a55fe1bc-1754-45fa-9adc-cf3d6d7c377a", "status": "sourcing", - "isApplicationPageActive": false + "isApplicationPageActive": false, + "minSalary": 100, + "maxSalary": 200, + "hoursPerWeek": 20, + "jobLocation": "Any location", + "jobTimezone": "GMT", + "currency": "USD" } } \ No newline at end of file diff --git a/test/messages/taas.job.update.event.json b/test/messages/taas.job.update.event.json index 3be8597..0f1cb76 100644 --- a/test/messages/taas.job.update.event.json +++ b/test/messages/taas.job.update.event.json @@ -23,6 +23,12 @@ "updatedBy": "a55fe1bc-1754-45fa-9adc-cf3d6d7c377a", "createdAt": "2020-11-05T19:00:16.268Z", "createdBy": "a55fe1bc-1754-45fa-9adc-cf3d6d7c377a", - "isApplicationPageActive": false + "isApplicationPageActive": false, + "minSalary": 100, + "maxSalary": 200, + "hoursPerWeek": 20, + "jobLocation": "Any location", + "jobTimezone": "GMT", + "currency": "USD" } } \ No newline at end of file diff --git a/test/messages/taas.jobcandidate.create.event.json b/test/messages/taas.jobcandidate.create.event.json index 822dc80..28ba870 100644 --- a/test/messages/taas.jobcandidate.create.event.json +++ b/test/messages/taas.jobcandidate.create.event.json @@ -9,6 +9,7 @@ "id": "0cb99adb-8bcd-4952-9203-9867dd45ef6f", "createdAt": "2020-11-05T19:00:19.052Z", "createdBy": "a55fe1bc-1754-45fa-9adc-cf3d6d7c377a", - "status": "open" + "status": "open", + "remark": "excellent" } } \ No newline at end of file diff --git a/test/messages/taas.jobcandidate.update.event.json b/test/messages/taas.jobcandidate.update.event.json index 1254087..cc6e506 100644 --- a/test/messages/taas.jobcandidate.update.event.json +++ b/test/messages/taas.jobcandidate.update.event.json @@ -8,6 +8,7 @@ "jobId": "ffbc24f7-301e-48d3-bf01-c056916056a2", "userId": "a55fe1bc-1754-45fa-9adc-cf3d6d7c377a", "status": "selected", + "remark": "excellent", "updatedAt": "2020-11-05T19:00:21.625Z", "updatedBy": "a55fe1bc-1754-45fa-9adc-cf3d6d7c377a", "createdAt": "2020-11-05T19:00:16.268Z", From bfd4d48aa6cd5f19e4df6347317aeae5767efed4 Mon Sep 17 00:00:00 2001 From: xxcxy Date: Sat, 29 May 2021 16:08:49 +0800 Subject: [PATCH 35/55] Batch Payments - Part 1 - Scheduler --- src/bootstrap.js | 4 ++-- src/scripts/createIndex.js | 10 ++++++++++ src/services/WorkPeriodPaymentProcessorService.js | 9 ++++++++- test/common/testHelper.js | 3 ++- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/bootstrap.js b/src/bootstrap.js index be0d1c8..2fa2265 100644 --- a/src/bootstrap.js +++ b/src/bootstrap.js @@ -12,13 +12,13 @@ global.Promise = require('bluebird') Joi.rateType = () => Joi.string().valid('hourly', 'daily', 'weekly', 'monthly') Joi.jobStatus = () => Joi.string().valid('sourcing', 'in-review', 'assigned', 'closed', 'cancelled') Joi.resourceBookingStatus = () => Joi.string().valid('placed', 'closed', 'cancelled') -Joi.jobCandidateStatus = () => Joi.string().valid('open', 'placed', 'selected', 'client rejected - screening', 'client rejected - interview', 'rejected - other', 'cancelled', 'interview', 'topcoder-rejected','applied','rejected-pre-screen','skills-test','skills-test','phone-screen','job-closed') +Joi.jobCandidateStatus = () => Joi.string().valid('open', 'placed', 'selected', 'client rejected - screening', 'client rejected - interview', 'rejected - other', 'cancelled', 'interview', 'topcoder-rejected', 'applied', 'rejected-pre-screen', 'skills-test', 'skills-test', 'phone-screen', 'job-closed') Joi.workload = () => Joi.string().valid('full-time', 'fractional') Joi.title = () => Joi.string().max(128) Joi.paymentStatus = () => Joi.string().valid('pending', 'partially-completed', 'completed', 'cancelled') Joi.xaiTemplate = () => Joi.string().valid(...allowedXAITemplates) Joi.interviewStatus = () => Joi.string().valid(...allowedInterviewStatuses) -Joi.workPeriodPaymentStatus = () => Joi.string().valid('completed', 'cancelled') +Joi.workPeriodPaymentStatus = () => Joi.string().valid('completed', 'scheduled', 'in-progress', 'failed', 'cancelled') // Empty string is not allowed by Joi by default and must be enabled with allow(''). // See https://joi.dev/api/?v=17.3.0#string fro details why it's like this. // In many cases we would like to allow empty string to make it easier to create UI for editing data. diff --git a/src/scripts/createIndex.js b/src/scripts/createIndex.js index aae5c10..385c2a4 100644 --- a/src/scripts/createIndex.js +++ b/src/scripts/createIndex.js @@ -120,6 +120,16 @@ async function createIndex () { challengeId: { type: 'keyword' }, amount: { type: 'float' }, status: { type: 'keyword' }, + statusDetails: { + type: 'nested', + properties: { + errorMessage: { type: 'text' }, + errorCode: { type: 'integer' }, + retry: { type: 'integer' }, + step: { type: 'keyword' }, + challengeId: { type: 'keyword' } + } + }, billingAccountId: { type: 'integer' }, createdAt: { type: 'date' }, createdBy: { type: 'keyword' }, diff --git a/src/services/WorkPeriodPaymentProcessorService.js b/src/services/WorkPeriodPaymentProcessorService.js index 7f30b6e..a168ded 100644 --- a/src/services/WorkPeriodPaymentProcessorService.js +++ b/src/services/WorkPeriodPaymentProcessorService.js @@ -65,10 +65,17 @@ processCreate.schema = { payload: Joi.object().keys({ id: Joi.string().uuid().required(), workPeriodId: Joi.string().uuid().required(), - challengeId: Joi.string().uuid().required(), + challengeId: Joi.string().uuid().allow(null), amount: Joi.number().greater(0).allow(null), status: Joi.workPeriodPaymentStatus().required(), billingAccountId: Joi.number().allow(null), + statusDetails: Joi.object().keys({ + errorMessage: Joi.string().required(), + errorCode: Joi.number().integer().allow(null), + retry: Joi.number().integer().allow(null), + step: Joi.string().allow(null), + challengeId: Joi.string().uuid().allow(null) + }).unknown(true).allow(null), createdAt: Joi.date().required(), createdBy: Joi.string().uuid().required(), updatedAt: Joi.date().allow(null), diff --git a/test/common/testHelper.js b/test/common/testHelper.js index fd310d8..6410492 100644 --- a/test/common/testHelper.js +++ b/test/common/testHelper.js @@ -40,7 +40,8 @@ async function clearES () { query: { match_all: {} } - } + }, + refresh: true }) } } From f4e193d2be894c162f89297c7231a9a4cb4c99fd Mon Sep 17 00:00:00 2001 From: Cagdas U Date: Sat, 29 May 2021 11:40:37 +0300 Subject: [PATCH 36/55] feat(job-processor): accept `roles` array * Add `roles` in ES mapping. * Accept `roles` in JobProcessor JOI validation. * Fix lint errors. --- src/bootstrap.js | 2 +- src/scripts/createIndex.js | 1 + src/services/JobProcessorService.js | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/bootstrap.js b/src/bootstrap.js index c3c6254..82287fc 100644 --- a/src/bootstrap.js +++ b/src/bootstrap.js @@ -12,7 +12,7 @@ global.Promise = require('bluebird') Joi.rateType = () => Joi.string().valid('hourly', 'daily', 'weekly', 'monthly') Joi.jobStatus = () => Joi.string().valid('sourcing', 'in-review', 'assigned', 'closed', 'cancelled') Joi.resourceBookingStatus = () => Joi.string().valid('placed', 'closed', 'cancelled') -Joi.jobCandidateStatus = () => Joi.string().valid('open', 'placed', 'selected', 'client rejected - screening', 'client rejected - interview', 'rejected - other', 'cancelled', 'interview', 'topcoder-rejected','applied','rejected-pre-screen','skills-test','skills-test','phone-screen','job-closed','offered') +Joi.jobCandidateStatus = () => Joi.string().valid('open', 'placed', 'selected', 'client rejected - screening', 'client rejected - interview', 'rejected - other', 'cancelled', 'interview', 'topcoder-rejected', 'applied', 'rejected-pre-screen', 'skills-test', 'skills-test', 'phone-screen', 'job-closed', 'offered') Joi.workload = () => Joi.string().valid('full-time', 'fractional') Joi.title = () => Joi.string().max(128) Joi.paymentStatus = () => Joi.string().valid('pending', 'partially-completed', 'completed', 'cancelled') diff --git a/src/scripts/createIndex.js b/src/scripts/createIndex.js index aae5c10..bf4bb0d 100644 --- a/src/scripts/createIndex.js +++ b/src/scripts/createIndex.js @@ -26,6 +26,7 @@ async function createIndex () { rateType: { type: 'keyword' }, workload: { type: 'keyword' }, skills: { type: 'keyword' }, + roles: { type: 'keyword' }, status: { type: 'keyword' }, isApplicationPageActive: { type: 'boolean' }, createdAt: { type: 'date' }, diff --git a/src/services/JobProcessorService.js b/src/services/JobProcessorService.js index 678a3c3..53ca81c 100644 --- a/src/services/JobProcessorService.js +++ b/src/services/JobProcessorService.js @@ -78,6 +78,7 @@ processCreate.schema = { rateType: Joi.rateType().allow(null), workload: Joi.workload().allow(null), skills: Joi.array().items(Joi.string().uuid()).required(), + roles: Joi.array().items(Joi.string().uuid()).allow(null), createdAt: Joi.date().required(), createdBy: Joi.string().uuid().required(), updatedAt: Joi.date().allow(null), From 4b20e8ea79929d88f58c9971877fdec23fad9beb Mon Sep 17 00:00:00 2001 From: eisbilir Date: Sun, 30 May 2021 23:52:24 +0300 Subject: [PATCH 37/55] fix: resource booking search issues --- src/scripts/createIndex.js | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/scripts/createIndex.js b/src/scripts/createIndex.js index bf4bb0d..870f34e 100644 --- a/src/scripts/createIndex.js +++ b/src/scripts/createIndex.js @@ -104,7 +104,8 @@ async function createIndex () { properties: { id: { type: 'keyword' }, resourceBookingId: { type: 'keyword' }, - userHandle: { type: 'keyword' }, + userHandle: { type: 'keyword', + normalizer: 'lowercaseNormalizer' }, projectId: { type: 'integer' }, userId: { type: 'keyword' }, startDate: { type: 'date', format: 'yyyy-MM-dd' }, @@ -145,7 +146,29 @@ async function createIndex () { ] for (const index of indices) { - await esClient.indices.create(index) + await esClient.indices.create({ index: index.index }) + await esClient.indices.close({ index: index.index }) + await esClient.indices.putSettings({ + index: index.index, + body: { + settings: { + analysis: { + normalizer: { + lowercaseNormalizer: { + filter: ['lowercase'] + } + } + } + } + } + }) + await esClient.indices.open({ index: index.index }) + await esClient.indices.putMapping({ + index: index.index, + body: { + properties: index.body.mappings.properties + } + }) logger.info({ component: 'createIndex', message: `ES Index ${index.index} creation succeeded!` }) } process.exit(0) From 97866e282f059f6ea3da2ab29e4c2909a5510d6a Mon Sep 17 00:00:00 2001 From: eisbilir Date: Mon, 31 May 2021 19:06:04 +0300 Subject: [PATCH 38/55] role endpoint added --- README.md | 4 + VERIFICATION.md | 12 ++- config/default.js | 9 +- package-lock.json | 6 +- src/app.js | 7 +- src/bootstrap.js | 1 + src/scripts/createIndex.js | 34 ++++++ src/scripts/deleteIndex.js | 3 +- src/scripts/view-data.js | 3 +- src/services/JobProcessorService.js | 3 +- src/services/RoleProcessorService.js | 119 +++++++++++++++++++++ test/common/testData.js | 5 + test/messages/taas.job.create.event.json | 5 +- test/messages/taas.job.update.event.json | 5 +- test/messages/taas.role.create.event.json | 50 +++++++++ test/messages/taas.role.delete.event .json | 9 ++ test/messages/taas.role.update.event.json | 48 +++++++++ 17 files changed, 309 insertions(+), 14 deletions(-) create mode 100644 src/services/RoleProcessorService.js create mode 100644 test/messages/taas.role.create.event.json create mode 100644 test/messages/taas.role.delete.event .json create mode 100644 test/messages/taas.role.update.event.json diff --git a/README.md b/README.md index 9136fcc..2abb251 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,9 @@ The following parameters can be set in config files or in env variables: - `topics.TAAS_INTERVIEW_REQUEST_TOPIC`: the request interview entity Kafka message topic - `topics.TAAS_INTERVIEW_UPDATE_TOPIC`: the update interview entity Kafka message topic - `topics.TAAS_INTERVIEW_BULK_UPDATE_TOPIC`: the bulk update interview entity Kafka message topic +- `topics.TAAS_ROLE_CREATE_TOPIC`: the create role entity Kafka message topic +- `topics.TAAS_ROLE_UPDATE_TOPIC`: the update role entity Kafka message topic +- `topics.TAAS_ROLE_DELETE_TOPIC`: the delete role entity Kafka message topic - `esConfig.HOST`: Elasticsearch host - `esConfig.AWS_REGION`: The Amazon region to use when using AWS Elasticsearch service - `esConfig.ELASTICCLOUD.id`: The elastic cloud id, if your elasticsearch instance is hosted on elastic cloud. DO NOT provide a value for ES_HOST if you are using this @@ -46,6 +49,7 @@ The following parameters can be set in config files or in env variables: - `esConfig.ES_INDEX_JOB`: the index name for job - `esConfig.ES_INDEX_JOB_CANDIDATE`: the index name for job candidate - `esConfig.ES_INDEX_RESOURCE_BOOKING`: the index name for resource booking +- `esConfig.ES_INDEX_ROLE`: the index name for role - `auth0.AUTH0_URL`: Auth0 URL, used to get TC M2M token - `auth0.AUTH0_AUDIENCE`: Auth0 audience, used to get TC M2M token diff --git a/VERIFICATION.md b/VERIFICATION.md index ef9046d..d6696f1 100644 --- a/VERIFICATION.md +++ b/VERIFICATION.md @@ -2,7 +2,7 @@ ## Create documents in ES -- Run the following commands to create `Job`, `JobCandidate`, `Interview`, `ResourceBooking`, `WorkPeriod`, `WorkPeriodPayment` documents in ES. +- Run the following commands to create `Job`, `JobCandidate`, `Interview`, `ResourceBooking`, `WorkPeriod`, `WorkPeriodPayment`, `Role` documents in ES. ``` bash # for Job @@ -17,12 +17,14 @@ docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.workperiod.create < test/messages/taas.workperiod.create.event.json # for WorkPeriodPayment docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.workperiodpayment.create < test/messages/taas.workperiodpayment.create.event.json + # for Role + docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.role.requested < test/messages/taas.role.create.event.json ``` - Run `npm run view-data ` to see if documents were created. ## Update documents in ES -- Run the following commands to update `Job`, `JobCandidate`, `Interview`, `ResourceBooking`, `WorkPeriod`, `WorkPeriodPayment` documents in ES. +- Run the following commands to update `Job`, `JobCandidate`, `Interview`, `ResourceBooking`, `WorkPeriod`, `WorkPeriodPayment`, `Role` documents in ES. ``` bash # for Job @@ -37,12 +39,14 @@ docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.workperiod.update < test/messages/taas.workperiod.update.event.json # for WorkPeriodPayment docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.workperiodpayment.update < test/messages/taas.workperiodpayment.update.event.json + # for Role + docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.role.update < test/messages/taas.role.update.event.json ``` - Run `npm run view-data ` to see if documents were updated. ## Delete documents in ES -- Run the following commands to delete `Job`, `JobCandidate`, `ResourceBooking`, `WorkPeriod` documents in ES. +- Run the following commands to delete `Job`, `JobCandidate`, `ResourceBooking`, `WorkPeriod`, `Role` documents in ES. ``` bash # for Job @@ -53,6 +57,8 @@ docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.resourcebooking.delete < test/messages/taas.resourcebooking.delete.event.json # for WorkPeriod docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.workperiod.delete < test/messages/taas.workperiod.delete.event.json + # for Role + docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.role.delete < test/messages/taas.role.delete.event.json ``` - Run `npm run view-data ` to see if documents were deleted. diff --git a/config/default.js b/config/default.js index 1ea7851..ffb2c16 100644 --- a/config/default.js +++ b/config/default.js @@ -38,7 +38,11 @@ module.exports = { // topics for interview service TAAS_INTERVIEW_REQUEST_TOPIC: process.env.TAAS_INTERVIEW_REQUEST_TOPIC || 'taas.interview.requested', TAAS_INTERVIEW_UPDATE_TOPIC: process.env.TAAS_INTERVIEW_UPDATE_TOPIC || 'taas.interview.update', - TAAS_INTERVIEW_BULK_UPDATE_TOPIC: process.env.TAAS_INTERVIEW_BULK_UPDATE_TOPIC || 'taas.interview.bulkUpdate' + TAAS_INTERVIEW_BULK_UPDATE_TOPIC: process.env.TAAS_INTERVIEW_BULK_UPDATE_TOPIC || 'taas.interview.bulkUpdate', + // topics for role service + TAAS_ROLE_CREATE_TOPIC: process.env.TAAS_ROLE_CREATE_TOPIC || 'taas.role.requested', + TAAS_ROLE_UPDATE_TOPIC: process.env.TAAS_ROLE_UPDATE_TOPIC || 'taas.role.update', + TAAS_ROLE_DELETE_TOPIC: process.env.TAAS_ROLE_DELETE_TOPIC || 'taas.role.delete' }, esConfig: { @@ -54,7 +58,8 @@ module.exports = { ES_INDEX_JOB: process.env.ES_INDEX_JOB || 'job', ES_INDEX_JOB_CANDIDATE: process.env.ES_INDEX_JOB_CANDIDATE || 'job_candidate', - ES_INDEX_RESOURCE_BOOKING: process.env.ES_INDEX_RESOURCE_BOOKING || 'resource_booking' + ES_INDEX_RESOURCE_BOOKING: process.env.ES_INDEX_RESOURCE_BOOKING || 'resource_booking', + ES_INDEX_ROLE: process.env.ES_INDEX_ROLE || 'role' }, auth0: { diff --git a/package-lock.json b/package-lock.json index 8db26cb..b198b5c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3967,9 +3967,9 @@ "dev": true }, "normalize-url": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.0.tgz", - "integrity": "sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ==", + "version": "4.5.1", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.1.tgz", + "integrity": "sha512-9UZCFRHQdNrfTpGg8+1INIg93B6zE0aXMVFkw1WFwvO4SlZywU6aLg5Of0Ap/PgcbSw4LNxvMWXMeugwMCX0AA==", "dev": true }, "npm-run-path": { diff --git a/src/app.js b/src/app.js index eb87bc9..5b403b0 100644 --- a/src/app.js +++ b/src/app.js @@ -15,6 +15,7 @@ const ResourceBookingProcessorService = require('./services/ResourceBookingProce const WorkPeriodProcessorService = require('./services/WorkPeriodProcessorService') const InterviewProcessorService = require('./services/InterviewProcessorService') const WorkPeriodPaymentProcessorService = require('./services/WorkPeriodPaymentProcessorService') +const RoleProcessorService = require('./services/RoleProcessorService') const Mutex = require('async-mutex').Mutex const events = require('events') @@ -52,7 +53,11 @@ const topicServiceMapping = { // interview [config.topics.TAAS_INTERVIEW_REQUEST_TOPIC]: InterviewProcessorService.processRequestInterview, [config.topics.TAAS_INTERVIEW_UPDATE_TOPIC]: InterviewProcessorService.processUpdateInterview, - [config.topics.TAAS_INTERVIEW_BULK_UPDATE_TOPIC]: InterviewProcessorService.processBulkUpdateInterviews + [config.topics.TAAS_INTERVIEW_BULK_UPDATE_TOPIC]: InterviewProcessorService.processBulkUpdateInterviews, + // role + [config.topics.TAAS_ROLE_CREATE_TOPIC]: RoleProcessorService.processCreate, + [config.topics.TAAS_ROLE_UPDATE_TOPIC]: RoleProcessorService.processUpdate, + [config.topics.TAAS_ROLE_DELETE_TOPIC]: RoleProcessorService.processDelete } // Start kafka consumer diff --git a/src/bootstrap.js b/src/bootstrap.js index c1df698..1e5a9bd 100644 --- a/src/bootstrap.js +++ b/src/bootstrap.js @@ -23,6 +23,7 @@ Joi.workPeriodPaymentStatus = () => Joi.string().valid('completed', 'cancelled') // See https://joi.dev/api/?v=17.3.0#string fro details why it's like this. // In many cases we would like to allow empty string to make it easier to create UI for editing data. Joi.stringAllowEmpty = () => Joi.string().allow('') +Joi.smallint = () => Joi.number().min(-32768).max(32767) const zapierSwitch = Joi.string().label('ZAPIER_SWITCH').valid(...Object.values(constants.Zapier.Switch)) diff --git a/src/scripts/createIndex.js b/src/scripts/createIndex.js index aae5c10..9472f3c 100644 --- a/src/scripts/createIndex.js +++ b/src/scripts/createIndex.js @@ -28,6 +28,7 @@ async function createIndex () { skills: { type: 'keyword' }, status: { type: 'keyword' }, isApplicationPageActive: { type: 'boolean' }, + roleIds: { type: 'keyword' }, createdAt: { type: 'date' }, createdBy: { type: 'keyword' }, updatedAt: { type: 'date' }, @@ -140,6 +141,39 @@ async function createIndex () { } } } + }, + { index: config.get('esConfig.ES_INDEX_ROLE'), + body: { + mappings: { + properties: { + name: { type: 'keyword' }, + description: { type: 'keyword' }, + listOfSkills: { type: 'keyword' }, + rates: { + properties: { + global: { type: 'integer' }, + inCountry: { type: 'integer' }, + offShore: { type: 'integer' }, + rate30Global: { type: 'integer' }, + rate30InCountry: { type: 'integer' }, + rate30OffShore: { type: 'integer' }, + rate20Global: { type: 'integer' }, + rate20InCountry: { type: 'integer' }, + rate20OffShore: { type: 'integer' } + } + }, + numberOfMembers: { type: 'integer' }, + numberOfMembersAvailable: { type: 'integer' }, + imageUrl: { type: 'keyword' }, + timeToCandidate: { type: 'integer' }, + timeToInterview: { type: 'integer' }, + createdAt: { type: 'date' }, + createdBy: { type: 'keyword' }, + updatedAt: { type: 'date' }, + updatedBy: { type: 'keyword' } + } + } + } } ] diff --git a/src/scripts/deleteIndex.js b/src/scripts/deleteIndex.js index 9c3a1bb..84e15bc 100644 --- a/src/scripts/deleteIndex.js +++ b/src/scripts/deleteIndex.js @@ -11,7 +11,8 @@ async function deleteIndex () { const esClient = helper.getESClient() const indices = [config.get('esConfig.ES_INDEX_JOB'), config.get('esConfig.ES_INDEX_JOB_CANDIDATE'), - config.get('esConfig.ES_INDEX_RESOURCE_BOOKING')] + config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), + config.get('esConfig.ES_INDEX_ROLE')] for (const index of indices) { await esClient.indices.delete({ index diff --git a/src/scripts/view-data.js b/src/scripts/view-data.js index 1db36d5..c422c0c 100644 --- a/src/scripts/view-data.js +++ b/src/scripts/view-data.js @@ -11,7 +11,8 @@ const esClient = helper.getESClient() const modelIndexMapping = { Job: 'ES_INDEX_JOB', JobCandidate: 'ES_INDEX_JOB_CANDIDATE', - ResourceBooking: 'ES_INDEX_RESOURCE_BOOKING' + ResourceBooking: 'ES_INDEX_RESOURCE_BOOKING', + Role: 'ES_INDEX_ROLE' } async function showESData () { diff --git a/src/services/JobProcessorService.js b/src/services/JobProcessorService.js index 678a3c3..adc47e5 100644 --- a/src/services/JobProcessorService.js +++ b/src/services/JobProcessorService.js @@ -83,7 +83,8 @@ processCreate.schema = { updatedAt: Joi.date().allow(null), updatedBy: Joi.string().uuid().allow(null), status: Joi.jobStatus().required(), - isApplicationPageActive: Joi.boolean().required() + isApplicationPageActive: Joi.boolean().required(), + roleIds: Joi.array().items(Joi.string().uuid().required()).allow(null) }).required() }).required(), transactionId: Joi.string().required() diff --git a/src/services/RoleProcessorService.js b/src/services/RoleProcessorService.js new file mode 100644 index 0000000..cf1386f --- /dev/null +++ b/src/services/RoleProcessorService.js @@ -0,0 +1,119 @@ +/** + * Role Processor Service + */ + +const Joi = require('@hapi/joi') +const logger = require('../common/logger') +const helper = require('../common/helper') +const constants = require('../common/constants') +const config = require('config') + +const esClient = helper.getESClient() + +/** + * Process create entity message + * @param {Object} message the kafka message + * @param {String} transactionId + */ +async function processCreate (message, transactionId) { + const role = message.payload + await esClient.createExtra({ + index: config.get('esConfig.ES_INDEX_ROLE'), + id: role.id, + transactionId, + body: role, + refresh: constants.esRefreshOption + }) +} + +processCreate.schema = { + message: Joi.object().keys({ + topic: Joi.string().required(), + originator: Joi.string().required(), + timestamp: Joi.date().required(), + 'mime-type': Joi.string().required(), + payload: Joi.object().keys({ + id: Joi.string().uuid().required(), + name: Joi.string().max(50).required(), + description: Joi.string().max(1000).allow(null), + listOfSkills: Joi.array().items(Joi.string().max(50).required()).allow(null), + rates: Joi.array().items(Joi.object().keys({ + global: Joi.smallint().required(), + inCountry: Joi.smallint().required(), + offShore: Joi.smallint().required(), + rate30Global: Joi.smallint().allow(null), + rate30InCountry: Joi.smallint().allow(null), + rate30OffShore: Joi.smallint().allow(null), + rate20Global: Joi.smallint().allow(null), + rate20InCountry: Joi.smallint().allow(null), + rate20OffShore: Joi.smallint().allow(null) + }).required()).required(), + numberOfMembers: Joi.number().allow(null), + numberOfMembersAvailable: Joi.smallint().allow(null), + imageUrl: Joi.string().uri().max(255).allow(null), + timeToCandidate: Joi.smallint().allow(null), + timeToInterview: Joi.smallint().allow(null), + createdAt: Joi.date().required(), + createdBy: Joi.string().uuid().required(), + updatedAt: Joi.date().allow(null), + updatedBy: Joi.string().uuid().allow(null) + }).required() + }).required(), + transactionId: Joi.string().required() +} + +/** + * Process update entity message + * @param {Object} message the kafka message + * @param {String} transactionId + */ +async function processUpdate (message, transactionId) { + const data = message.payload + await esClient.updateExtra({ + index: config.get('esConfig.ES_INDEX_ROLE'), + id: data.id, + transactionId, + body: { + doc: data + }, + refresh: constants.esRefreshOption + }) +} + +processUpdate.schema = processCreate.schema + +/** + * Process delete entity message + * @param {Object} message the kafka message + * @param {String} transactionId + */ +async function processDelete (message, transactionId) { + const id = message.payload.id + await esClient.deleteExtra({ + index: config.get('esConfig.ES_INDEX_ROLE'), + id, + transactionId, + refresh: constants.esRefreshOption + }) +} + +processDelete.schema = { + message: Joi.object().keys({ + topic: Joi.string().required(), + originator: Joi.string().required(), + timestamp: Joi.date().required(), + 'mime-type': Joi.string().required(), + payload: Joi.object().keys({ + id: Joi.string().uuid().required() + }).required() + }).required(), + transactionId: Joi.string().required() +} + +module.exports = { + processCreate, + processUpdate, + processDelete +} + +logger.buildService(module.exports, 'RoleProcessorService') diff --git a/test/common/testData.js b/test/common/testData.js index e583618..47a0dd1 100644 --- a/test/common/testData.js +++ b/test/common/testData.js @@ -26,6 +26,11 @@ const messages = { create: { topic: 'taas.workperiodpayment.create', message: require('../messages/taas.workperiodpayment.create.event.json') }, update: { topic: 'taas.workperiodpayment.update', message: require('../messages/taas.workperiodpayment.update.event.json') } }, + Role: { + create: { topic: 'taas.role.requested', message: require('../messages/taas.role.create.event.json') }, + update: { topic: 'taas.role.update', message: require('../messages/taas.role.update.event.json') }, + delete: { topic: 'taas.role.delete', message: require('../messages/taas.role.delete.event .json') } + }, messageInvalid: '{ "topic": "taas.job.create", }' } diff --git a/test/messages/taas.job.create.event.json b/test/messages/taas.job.create.event.json index ebc5d15..c176048 100644 --- a/test/messages/taas.job.create.event.json +++ b/test/messages/taas.job.create.event.json @@ -24,6 +24,9 @@ "createdAt": "2020-11-05T19:00:16.268Z", "createdBy": "a55fe1bc-1754-45fa-9adc-cf3d6d7c377a", "status": "sourcing", - "isApplicationPageActive": false + "isApplicationPageActive": false, + "roleIds": [ + "e7b7e818-40d4-4102-b486-09bdd21400b8" + ] } } \ No newline at end of file diff --git a/test/messages/taas.job.update.event.json b/test/messages/taas.job.update.event.json index 3be8597..8371274 100644 --- a/test/messages/taas.job.update.event.json +++ b/test/messages/taas.job.update.event.json @@ -23,6 +23,9 @@ "updatedBy": "a55fe1bc-1754-45fa-9adc-cf3d6d7c377a", "createdAt": "2020-11-05T19:00:16.268Z", "createdBy": "a55fe1bc-1754-45fa-9adc-cf3d6d7c377a", - "isApplicationPageActive": false + "isApplicationPageActive": false, + "roleIds": [ + "e7b7e818-40d4-4102-b486-09bdd21400b8" + ] } } \ No newline at end of file diff --git a/test/messages/taas.role.create.event.json b/test/messages/taas.role.create.event.json new file mode 100644 index 0000000..a45b6a8 --- /dev/null +++ b/test/messages/taas.role.create.event.json @@ -0,0 +1,50 @@ +{ + "topic": "taas.role.requested", + "originator": "taas-api", + "timestamp": "2021-05-27T21:43:09.388Z", + "mime-type": "application/json", + "payload": { + "name": "Salesforce Developer", + "description": "A Salesforce developer is a programmer who builds Salesforce applications across various PaaS (Platform as a Service) platforms.", + "listOfSkills": [ + "Docker", + ".NET", + "appcelerator", + "Flux" + ], + "rates": [ + { + "global": 50, + "inCountry": 20, + "offShore": 10, + "rate30Global": 20, + "rate30InCountry": 15, + "rate30OffShore": 35, + "rate20Global": 20, + "rate20InCountry": 15, + "rate20OffShore": 35 + }, + { + "global": 25, + "inCountry": 15, + "offShore": 5, + "rate30Global": 20, + "rate30InCountry": 15, + "rate30OffShore": 35, + "rate20Global": 20, + "rate20InCountry": 15, + "rate20OffShore": 35 + } + ], + "numberOfMembers": 10, + "numberOfMembersAvailable": 6, + "imageUrl": "http: //images.topcoder.com/member", + "timeToCandidate": 105, + "timeToInterview": 100, + "id": "e7b7e818-40d4-4102-b486-09bdd21400b8", + "createdBy": "00000000-0000-0000-0000-000000000000", + "updatedAt": "2021-05-27T21:43:09.342Z", + "createdAt": "2021-05-27T21:43:09.342Z", + "updatedBy": null + } +} \ No newline at end of file diff --git a/test/messages/taas.role.delete.event .json b/test/messages/taas.role.delete.event .json new file mode 100644 index 0000000..3bd7c32 --- /dev/null +++ b/test/messages/taas.role.delete.event .json @@ -0,0 +1,9 @@ +{ + "topic": "taas.role.delete", + "originator": "taas-api", + "timestamp": "2021-05-27T21:45:09.388Z", + "mime-type": "application/json", + "payload": { + "id": "e7b7e818-40d4-4102-b486-09bdd21400b8" + } +} \ No newline at end of file diff --git a/test/messages/taas.role.update.event.json b/test/messages/taas.role.update.event.json new file mode 100644 index 0000000..5c2a482 --- /dev/null +++ b/test/messages/taas.role.update.event.json @@ -0,0 +1,48 @@ +{ + "topic": "taas.role.update", + "originator": "taas-api", + "timestamp": "2021-05-27T21:44:09.388Z", + "mime-type": "application/json", + "payload": { + "name": "Salesforce Developer", + "description": "A Salesforce developer is a programmer who builds Salesforce applications across various PaaS (Platform as a Service) platforms.", + "listOfSkills": [ + "Docker", + ".NET" + ], + "rates": [ + { + "global": 50, + "inCountry": 20, + "offShore": 10, + "rate30Global": 20, + "rate30InCountry": 15, + "rate30OffShore": 35, + "rate20Global": 20, + "rate20InCountry": 15, + "rate20OffShore": 35 + }, + { + "global": 25, + "inCountry": 15, + "offShore": 5, + "rate30Global": 20, + "rate30InCountry": 15, + "rate30OffShore": 35, + "rate20Global": 20, + "rate20InCountry": 15, + "rate20OffShore": 35 + } + ], + "numberOfMembers": 10, + "numberOfMembersAvailable": 6, + "imageUrl": "http: //images.topcoder.com/member", + "timeToCandidate": 105, + "timeToInterview": 100, + "id": "e7b7e818-40d4-4102-b486-09bdd21400b8", + "createdBy": "00000000-0000-0000-0000-000000000000", + "updatedAt": "2021-05-27T21:43:09.342Z", + "createdAt": "2021-05-27T21:43:09.342Z", + "updatedBy": "00000000-0000-0000-0000-000000000000" + } +} \ No newline at end of file From 5a80f8a668c3a584a95d66dc4e599ef39da5896a Mon Sep 17 00:00:00 2001 From: Sushil Shinde Date: Tue, 1 Jun 2021 15:19:18 +0530 Subject: [PATCH 39/55] fix: allow null for new fields --- src/services/JobProcessorService.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/services/JobProcessorService.js b/src/services/JobProcessorService.js index 47f80d6..08f5525 100644 --- a/src/services/JobProcessorService.js +++ b/src/services/JobProcessorService.js @@ -85,12 +85,12 @@ processCreate.schema = { updatedBy: Joi.string().uuid().allow(null), status: Joi.jobStatus().required(), isApplicationPageActive: Joi.boolean().required(), - minSalary: Joi.number().integer().required(), - maxSalary: Joi.number().integer().required(), - hoursPerWeek: Joi.number().integer().required(), - jobLocation: Joi.string().required(), - jobTimezone: Joi.string().required(), - currency: Joi.string().required(), + minSalary: Joi.number().integer().allow(null), + maxSalary: Joi.number().integer().allow(null), + hoursPerWeek: Joi.number().integer().allow(null), + jobLocation: Joi.string().allow(null), + jobTimezone: Joi.string().allow(null), + currency: Joi.string().allow(null), roleIds: Joi.array().items(Joi.string().uuid().required()).allow(null) }).required() }).required(), From fc7267f8b52a471ea85d3d347b01dc891767a1d1 Mon Sep 17 00:00:00 2001 From: eisbilir Date: Thu, 3 Jun 2021 18:59:36 +0300 Subject: [PATCH 40/55] fix: role index use normalizer --- src/scripts/createIndex.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/scripts/createIndex.js b/src/scripts/createIndex.js index c643a42..65289af 100644 --- a/src/scripts/createIndex.js +++ b/src/scripts/createIndex.js @@ -155,9 +155,11 @@ async function createIndex () { body: { mappings: { properties: { - name: { type: 'keyword' }, + name: { type: 'keyword', + normalizer: 'lowercaseNormalizer' }, description: { type: 'keyword' }, - listOfSkills: { type: 'keyword' }, + listOfSkills: { type: 'keyword', + normalizer: 'lowercaseNormalizer' }, rates: { properties: { global: { type: 'integer' }, From 3a7e4a70e82e4806123f066cf79b6690d7b3d0aa Mon Sep 17 00:00:00 2001 From: eisbilir Date: Thu, 3 Jun 2021 23:19:21 +0300 Subject: [PATCH 41/55] action topic and service added --- README.md | 7 ++ config/default.js | 20 +++- package-lock.json | 133 +++++++++++++++++++-- package.json | 2 + src/app.js | 9 +- src/common/errors.js | 40 +++++++ src/common/helper.js | 71 ++++++++++- src/services/ActionProcessorService.js | 83 +++++++++++++ src/services/WorkPeriodProcessorService.js | 24 +++- test/unit/test.js | 12 +- 10 files changed, 371 insertions(+), 30 deletions(-) create mode 100644 src/common/errors.js create mode 100644 src/services/ActionProcessorService.js diff --git a/README.md b/README.md index 2abb251..257b38c 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,9 @@ The following parameters can be set in config files or in env variables: - `KAFKA_CLIENT_CERT_KEY`: Kafka connection private key, optional; if not provided, then SSL connection is not used, direct insecure connection is used; if provided, it can be either path to private key file or private key content +- `KAFKA_MESSAGE_ORIGINATOR`: The originator value for the kafka messages - `KAFKA_GROUP_ID`: the Kafka group id +- `topics.KAFKA_ERROR_TOPIC`: the error topic at which bus api will publish any errors - `topics.TAAS_JOB_CREATE_TOPIC`: the create job entity Kafka message topic - `topics.TAAS_JOB_UPDATE_TOPIC`: the update job entity Kafka message topic - `topics.TAAS_JOB_DELETE_TOPIC`: the delete job entity Kafka message topic @@ -41,6 +43,10 @@ The following parameters can be set in config files or in env variables: - `topics.TAAS_ROLE_CREATE_TOPIC`: the create role entity Kafka message topic - `topics.TAAS_ROLE_UPDATE_TOPIC`: the update role entity Kafka message topic - `topics.TAAS_ROLE_DELETE_TOPIC`: the delete role entity Kafka message topic +- `topics.TAAS_ACTION_RETRY_TOPIC`: the retry process Kafka message topic +- `MAX_RETRY`: maximum allowed retry count for failed operations for sending `taas.action.retry` message +- `BASE_RETRY_DELAY`: base amount of retry delay (ms) for failed operations +- `BUSAPI_URL`: Topcoder Bus API URL - `esConfig.HOST`: Elasticsearch host - `esConfig.AWS_REGION`: The Amazon region to use when using AWS Elasticsearch service - `esConfig.ELASTICCLOUD.id`: The elastic cloud id, if your elasticsearch instance is hosted on elastic cloud. DO NOT provide a value for ES_HOST if you are using this @@ -56,6 +62,7 @@ The following parameters can be set in config files or in env variables: - `auth0.AUTH0_CLIENT_ID`: Auth0 client id, used to get TC M2M token - `auth0.AUTH0_CLIENT_SECRET`: Auth0 client secret, used to get TC M2M token - `auth0.AUTH0_PROXY_SERVER_URL`: Proxy Auth0 URL, used to get TC M2M token +- `auth0.TOKEN_CACHE_TIME`: Auth0 token cache time, used to get TC M2M token - `zapier.ZAPIER_COMPANYID_SLUG`: your company id in zapier; numeric value - `zapier.ZAPIER_CONTACTID_SLUG`: your contact id in zapier; numeric value diff --git a/config/default.js b/config/default.js index ffb2c16..059ecdc 100644 --- a/config/default.js +++ b/config/default.js @@ -1,7 +1,7 @@ /** * The default configuration file. */ - +require('dotenv').config() module.exports = { PORT: process.env.PORT || 3001, LOG_LEVEL: process.env.LOG_LEVEL || 'debug', @@ -14,8 +14,12 @@ module.exports = { // Kafka group id KAFKA_GROUP_ID: process.env.KAFKA_GROUP_ID || 'taas-es-processor', + // The originator value for the kafka messages + KAFKA_MESSAGE_ORIGINATOR: process.env.KAFKA_MESSAGE_ORIGINATOR || 'taas-es-processor', topics: { + // The error topic at which bus api will publish any errors + KAFKA_ERROR_TOPIC: process.env.KAFKA_ERROR_TOPIC || 'common.error.reporting', // topics for job service TAAS_JOB_CREATE_TOPIC: process.env.TAAS_JOB_CREATE_TOPIC || 'taas.job.create', TAAS_JOB_UPDATE_TOPIC: process.env.TAAS_JOB_UPDATE_TOPIC || 'taas.job.update', @@ -42,8 +46,17 @@ module.exports = { // topics for role service TAAS_ROLE_CREATE_TOPIC: process.env.TAAS_ROLE_CREATE_TOPIC || 'taas.role.requested', TAAS_ROLE_UPDATE_TOPIC: process.env.TAAS_ROLE_UPDATE_TOPIC || 'taas.role.update', - TAAS_ROLE_DELETE_TOPIC: process.env.TAAS_ROLE_DELETE_TOPIC || 'taas.role.delete' + TAAS_ROLE_DELETE_TOPIC: process.env.TAAS_ROLE_DELETE_TOPIC || 'taas.role.delete', + // special kafka topics + TAAS_ACTION_RETRY_TOPIC: process.env.TAAS_ACTION_RETRY_TOPIC || 'taas.action.retry' + }, + // maximum allowed retry count for failed operations for sending `action.retry` message + MAX_RETRY: process.env.MAX_RETRY || 3, + // base amount of retry delay for failed operations + BASE_RETRY_DELAY: process.env.BASE_RETRY_DELAY || 500, + // Topcoder Bus API URL + BUSAPI_URL: process.env.BUSAPI_URL || 'https://api.topcoder-dev.com/v5', esConfig: { HOST: process.env.ES_HOST || 'http://localhost:9200', @@ -67,7 +80,8 @@ module.exports = { AUTH0_AUDIENCE: process.env.AUTH0_AUDIENCE, AUTH0_CLIENT_ID: process.env.AUTH0_CLIENT_ID, AUTH0_CLIENT_SECRET: process.env.AUTH0_CLIENT_SECRET, - AUTH0_PROXY_SERVER_URL: process.env.AUTH0_PROXY_SERVER_URL + AUTH0_PROXY_SERVER_URL: process.env.AUTH0_PROXY_SERVER_URL, + TOKEN_CACHE_TIME: process.env.TOKEN_CACHE_TIME }, zapier: { diff --git a/package-lock.json b/package-lock.json index b198b5c..fec285f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -271,6 +271,79 @@ "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==" }, + "@topcoder-platform/topcoder-bus-api-wrapper": { + "version": "github:topcoder-platform/tc-bus-api-wrapper#f8cbd335a0e0b4d6edd7cae859473593271fd97f", + "from": "github:topcoder-platform/tc-bus-api-wrapper", + "requires": { + "joi": "^13.4.0", + "lodash": "^4.17.15", + "superagent": "^3.8.3", + "tc-core-library-js": "github:appirio-tech/tc-core-library-js#v2.6.4" + }, + "dependencies": { + "debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "requires": { + "ms": "^2.1.1" + } + }, + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + }, + "superagent": { + "version": "3.8.3", + "resolved": "https://registry.npmjs.org/superagent/-/superagent-3.8.3.tgz", + "integrity": "sha512-GLQtLMCoEIK4eDv6OGtkOoSMt3D+oq0y3dsxMuYuDvaNUvuT8eFBuLmfR0iYYzHC1e8hpzC6ZsxbuP6DIalMFA==", + "requires": { + "component-emitter": "^1.2.0", + "cookiejar": "^2.1.0", + "debug": "^3.1.0", + "extend": "^3.0.0", + "form-data": "^2.3.1", + "formidable": "^1.2.0", + "methods": "^1.1.1", + "mime": "^1.4.1", + "qs": "^6.5.1", + "readable-stream": "^2.3.5" + } + }, + "tc-core-library-js": { + "version": "github:appirio-tech/tc-core-library-js#df0b36c51cf80918194cbff777214b3c0cf5a151", + "from": "github:appirio-tech/tc-core-library-js#v2.6.4", + "requires": { + "axios": "^0.19.0", + "bunyan": "^1.8.12", + "jsonwebtoken": "^8.5.1", + "jwks-rsa": "^1.6.0", + "lodash": "^4.17.15", + "millisecond": "^0.1.2", + "r7insight_node": "^1.8.4", + "request": "^2.88.0" + } + } + } + }, "@types/bluebird": { "version": "3.5.0", "resolved": "https://registry.npm.taobao.org/@types/bluebird/download/@types/bluebird-3.5.0.tgz", @@ -1133,8 +1206,7 @@ "component-emitter": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz", - "integrity": "sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==", - "dev": true + "integrity": "sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==" }, "concat-map": { "version": "0.0.1", @@ -1238,8 +1310,7 @@ "cookiejar": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.1.2.tgz", - "integrity": "sha512-Mw+adcfzPxcPeI+0WlvRrr/3lGVO0bD75SxX6811cxSh1Wbxx7xZBGK1eVtDf6si8rg2lhnUjsVLMFMfbRIuwA==", - "dev": true + "integrity": "sha512-Mw+adcfzPxcPeI+0WlvRrr/3lGVO0bD75SxX6811cxSh1Wbxx7xZBGK1eVtDf6si8rg2lhnUjsVLMFMfbRIuwA==" }, "core-js": { "version": "2.6.12", @@ -1485,6 +1556,11 @@ "is-obj": "^2.0.0" } }, + "dotenv": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-10.0.0.tgz", + "integrity": "sha512-rlBi9d8jpv9Sf1klPjNfFAuWDjKLwTIJJ/VxtoTwIR6hnZxcEOQCZg2oIL3MWBYw5GpUDKOEnND7LXTbIpQ03Q==" + }, "dtrace-provider": { "version": "0.8.8", "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.8.8.tgz", @@ -2310,7 +2386,6 @@ "version": "2.5.0", "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.0.tgz", "integrity": "sha512-WXieX3G/8side6VIqx44ablyULoGruSde5PNTxoUyo5CeyAMX6nVWUd0rgist/EuX655cjhUhTo1Fo3tRYqbcA==", - "dev": true, "requires": { "asynckit": "^0.4.0", "combined-stream": "^1.0.6", @@ -2320,8 +2395,7 @@ "formidable": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/formidable/-/formidable-1.2.1.tgz", - "integrity": "sha512-Fs9VRguL0gqGHkXS5GQiMCr1VhZBxz0JnJs4JmMp/2jL18Fmbzvv7vOFRU+U8TBkHEE/CX1qDXzJplVULgsLeg==", - "dev": true + "integrity": "sha512-Fs9VRguL0gqGHkXS5GQiMCr1VhZBxz0JnJs4JmMp/2jL18Fmbzvv7vOFRU+U8TBkHEE/CX1qDXzJplVULgsLeg==" }, "forwarded": { "version": "0.1.2", @@ -2570,6 +2644,11 @@ "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", "dev": true }, + "hoek": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/hoek/-/hoek-5.0.4.tgz", + "integrity": "sha512-Alr4ZQgoMlnere5FZJsIyfIjORBqZll5POhDsF4q64dPuJR6rNxXdDxtHSQq8OXRurhmx+PWYEE8bXRROY8h0w==" + }, "hosted-git-info": { "version": "2.7.1", "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.7.1.tgz", @@ -2931,6 +3010,21 @@ "resolved": "https://registry.npm.taobao.org/isarray/download/isarray-1.0.0.tgz", "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, + "isemail": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/isemail/-/isemail-3.2.0.tgz", + "integrity": "sha512-zKqkK+O+dGqevc93KNsbZ/TqTUFd46MwWjYOoMrjIMZ51eU7DtQG3Wmd9SQQT7i7RVnuTPEiYEWHU3MSbxC1Tg==", + "requires": { + "punycode": "2.x.x" + }, + "dependencies": { + "punycode": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" + } + } + }, "isexe": { "version": "2.0.0", "resolved": "https://registry.npm.taobao.org/isexe/download/isexe-2.0.0.tgz", @@ -3046,6 +3140,16 @@ "resolved": "https://registry.npm.taobao.org/jmespath/download/jmespath-0.15.0.tgz", "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" }, + "joi": { + "version": "13.7.0", + "resolved": "https://registry.npmjs.org/joi/-/joi-13.7.0.tgz", + "integrity": "sha512-xuY5VkHfeOYK3Hdi91ulocfuFopwgbSORmIwzcwHKESQhC7w1kD5jaVSPnqDxS2I8t3RZ9omCKAxNwXN5zG1/Q==", + "requires": { + "hoek": "5.x.x", + "isemail": "3.x.x", + "topo": "3.x.x" + } + }, "js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -5697,6 +5801,21 @@ "express": "^4.16.3" } }, + "topo": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/topo/-/topo-3.0.3.tgz", + "integrity": "sha512-IgpPtvD4kjrJ7CRA3ov2FhWQADwv+Tdqbsf1ZnPUSAtCJ9e1Z44MmoSGDXGk4IppoZA7jd/QRkNddlLJWlUZsQ==", + "requires": { + "hoek": "6.x.x" + }, + "dependencies": { + "hoek": { + "version": "6.1.3", + "resolved": "https://registry.npmjs.org/hoek/-/hoek-6.1.3.tgz", + "integrity": "sha512-YXXAAhmF9zpQbC7LEcREFtXfGq5K1fmd+4PHkBq8NUqmzW3G+Dq10bI/i0KucLRwss3YYFQ0fSfoxBZYiGUqtQ==" + } + } + }, "touch": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/touch/-/touch-3.1.0.tgz", diff --git a/package.json b/package.json index 7bd0c50..77228e9 100644 --- a/package.json +++ b/package.json @@ -34,10 +34,12 @@ "dependencies": { "@elastic/elasticsearch": "^7.9.1", "@hapi/joi": "^15.1.0", + "@topcoder-platform/topcoder-bus-api-wrapper": "github:topcoder-platform/tc-bus-api-wrapper", "async-mutex": "^0.2.4", "aws-sdk": "^2.476.0", "bluebird": "^3.5.5", "config": "^3.1.0", + "dotenv": "^10.0.0", "get-parameter-names": "^0.3.0", "lodash": "^4.17.20", "no-kafka": "^3.4.3", diff --git a/src/app.js b/src/app.js index 5b403b0..bc6bb2a 100644 --- a/src/app.js +++ b/src/app.js @@ -16,6 +16,7 @@ const WorkPeriodProcessorService = require('./services/WorkPeriodProcessorServic const InterviewProcessorService = require('./services/InterviewProcessorService') const WorkPeriodPaymentProcessorService = require('./services/WorkPeriodPaymentProcessorService') const RoleProcessorService = require('./services/RoleProcessorService') +const ActionProcessorService = require('./services/ActionProcessorService') const Mutex = require('async-mutex').Mutex const events = require('events') @@ -23,7 +24,6 @@ const eventEmitter = new events.EventEmitter() // healthcheck listening port process.env.PORT = config.PORT - const localLogger = { info: (message) => logger.info({ component: 'app', message }), debug: (message) => logger.debug({ component: 'app', message }), @@ -57,7 +57,9 @@ const topicServiceMapping = { // role [config.topics.TAAS_ROLE_CREATE_TOPIC]: RoleProcessorService.processCreate, [config.topics.TAAS_ROLE_UPDATE_TOPIC]: RoleProcessorService.processUpdate, - [config.topics.TAAS_ROLE_DELETE_TOPIC]: RoleProcessorService.processDelete + [config.topics.TAAS_ROLE_DELETE_TOPIC]: RoleProcessorService.processDelete, + // action + [config.topics.TAAS_ACTION_RETRY_TOPIC]: ActionProcessorService.processRetry } // Start kafka consumer @@ -179,5 +181,6 @@ if (!module.parent) { module.exports = { initConsumer, - eventEmitter + eventEmitter, + topicServiceMapping } diff --git a/src/common/errors.js b/src/common/errors.js new file mode 100644 index 0000000..1fee2b6 --- /dev/null +++ b/src/common/errors.js @@ -0,0 +1,40 @@ +/** + * This file defines application errors + */ +const util = require('util') + +/** + * Helper function to create generic error object with http status code + * @param {String} name the error name + * @param {Number} statusCode the http status code + * @returns {Function} the error constructor + * @private + */ +function createError (name, statusCode) { + /** + * The error constructor + * @param {String} message the error message + * @param {String} [cause] the error cause + * @constructor + */ + function ErrorCtor (message, cause) { + Error.call(this) + Error.captureStackTrace(this) + this.message = message || name + this.cause = cause + this.httpStatus = statusCode + } + + util.inherits(ErrorCtor, Error) + ErrorCtor.prototype.name = name + return ErrorCtor +} + +module.exports = { + BadRequestError: createError('BadRequestError', 400), + UnauthorizedError: createError('UnauthorizedError', 401), + ForbiddenError: createError('ForbiddenError', 403), + NotFoundError: createError('NotFoundError', 404), + ConflictError: createError('ConflictError', 409), + InternalServerError: createError('InternalServerError', 500) +} diff --git a/src/common/helper.js b/src/common/helper.js index 281b335..9530d99 100644 --- a/src/common/helper.js +++ b/src/common/helper.js @@ -6,10 +6,13 @@ const AWS = require('aws-sdk') const config = require('config') const request = require('superagent') const logger = require('./logger') +const errors = require('./errors') const elasticsearch = require('@elastic/elasticsearch') const _ = require('lodash') const { Mutex } = require('async-mutex') const m2mAuth = require('tc-core-library-js').auth.m2m +const busApi = require('@topcoder-platform/topcoder-bus-api-wrapper') +const ActionProcessorService = require('../services/ActionProcessorService') AWS.config.region = config.esConfig.AWS_REGION @@ -91,7 +94,7 @@ function getESClient () { await esClient.create(data) } catch (err) { if (err.statusCode === 409) { - throw new Error(`id: ${data.id} "${data.index}" already exists`) + throw new errors.ConflictError(`id: ${data.id} "${data.index}" already exists`) } throw err } @@ -103,7 +106,7 @@ function getESClient () { await esClient.update(data) } catch (err) { if (err.statusCode === 404) { - throw new Error(`id: ${data.id} "${data.index}" not found`) + throw new errors.NotFoundError(`id: ${data.id} "${data.index}" not found`) } throw err } @@ -117,7 +120,7 @@ function getESClient () { doc = await esClient.getSource(data) } catch (err) { if (err.statusCode === 404) { - throw new Error(`id: ${data.id} "${data.index}" not found`) + throw new errors.NotFoundError(`id: ${data.id} "${data.index}" not found`) } throw err } @@ -131,7 +134,7 @@ function getESClient () { await esClient.delete(data) } catch (err) { if (err.statusCode === 404) { - throw new Error(`id: ${data.id} "${data.index}" not found`) + throw new errors.NotFoundError(`id: ${data.id} "${data.index}" not found`) } throw err } @@ -178,10 +181,68 @@ async function postMessageViaWebhook (webhook, message) { await request.post(webhook).send(message) } +/** + * Calls ActionProcessorService to attempt to retry failed process + * @param {String} topic the failed topic name + * @param {Object} payload the payload + * @param {String} id the id that was the subject of the operation failed + */ +async function retryFailedProcess (topic, payload, id) { + await ActionProcessorService.processCreate(topic, payload, id) +} + +let busApiClient + +/** + * Get bus api client. + * + * @returns {Object} the bus api client + */ +function getBusApiClient () { + if (busApiClient) { + return busApiClient + } + busApiClient = busApi( + _.assign(_.pick(config.auth0, [ + 'AUTH0_URL', + 'AUTH0_AUDIENCE', + 'TOKEN_CACHE_TIME', + 'AUTH0_CLIENT_ID', + 'AUTH0_CLIENT_SECRET', + 'AUTH0_PROXY_SERVER_URL' + ]), _.pick(config, 'BUSAPI_URL'), + _.pick(config.topics, 'KAFKA_ERROR_TOPIC')) + + ) + return busApiClient +} + +/** + * Send Kafka event message + * @param {String} topic the topic name + * @param {Object} payload the payload + */ +async function postEvent (topic, payload) { + logger.debug({ component: 'helper', context: 'postEvent', message: `Posting event to Kafka topic ${topic}, ${JSON.stringify(payload)}` }) + + const client = getBusApiClient() + const message = { + topic, + originator: config.KAFKA_MESSAGE_ORIGINATOR, + timestamp: new Date().toISOString(), + 'mime-type': 'application/json', + payload + } + await client.postEvent(message) +} + module.exports = { getKafkaOptions, getESClient, checkEsMutexRelease, getM2MToken, - postMessageViaWebhook + postMessageViaWebhook, + retryFailedProcess, + getBusApiClient, + postEvent } diff --git a/src/services/ActionProcessorService.js b/src/services/ActionProcessorService.js new file mode 100644 index 0000000..3faac7c --- /dev/null +++ b/src/services/ActionProcessorService.js @@ -0,0 +1,83 @@ +/** + * Action Processor Service + */ + +const Joi = require('@hapi/joi') +const logger = require('../common/logger') +const helper = require('../common/helper') +const config = require('config') +const _ = require('lodash') + +const localLogger = { + debug: ({ context, message }) => logger.debug({ component: 'ActionProcessorService', context, message }) +} + +const retryMap = {} + +/** + * Process retry operation message + * @param {Object} message the kafka message + * @param {String} transactionId + */ +async function processRetry (message, transactionId) { + if (message.originator !== config.KAFKA_MESSAGE_ORIGINATOR) { + localLogger.debug({ context: 'processRetry', message: `originator: ${message.originator} does not match with ${config.KAFKA_MESSAGE_ORIGINATOR} - ignored` }) + return + } + const { topicServiceMapping } = require('../app') + message.topic = message.payload.originalTopic + message.payload = message.payload.originalPayload + await topicServiceMapping[message.topic](message, transactionId) +} + +processRetry.schema = { + message: Joi.object().keys({ + topic: Joi.string().required(), + originator: Joi.string().required(), + timestamp: Joi.date().required(), + 'mime-type': Joi.string().required(), + payload: Joi.object().keys({ + originalTopic: Joi.string().required(), + originalPayload: Joi.object().required(), + retry: Joi.number().integer().min(1).required() + }).required() + }).required(), + transactionId: Joi.string().required() +} + +/** + * Analyzes the failed process and sends it to bus api to be received again. + * @param {String} originalTopic the failed topic name + * @param {Object} originalPayload the payload + * @param {String} id the id that was the subject of the operation failed + */ +async function processCreate (originalTopic, originalPayload, id) { + const retry = _.defaultTo(retryMap[id], 0) + 1 + if (retry > config.MAX_RETRY) { + localLogger.debug({ context: 'processCreate', message: `retry: ${retry} for ${id} exceeds the max retry: ${config.MAX_RETRY} - ignored` }) + return + } + localLogger.debug({ context: 'processCreate', message: `retry: ${retry} for ${id}` }) + retryMap[id] = retry + const payload = { + originalTopic, + originalPayload, + retry + } + setTimeout(async function () { + await helper.postEvent(config.topics.TAAS_ACTION_RETRY_TOPIC, payload) + }, 2 ** retry * config.BASE_RETRY_DELAY) +} + +processCreate.schema = { + originalTopic: Joi.string().required(), + originalPayload: Joi.object().required(), + id: Joi.string().uuid().required() +} + +module.exports = { + processRetry, + processCreate +} + +logger.buildService(module.exports, 'ActionProcessorService') diff --git a/src/services/WorkPeriodProcessorService.js b/src/services/WorkPeriodProcessorService.js index 770f774..3b91a91 100644 --- a/src/services/WorkPeriodProcessorService.js +++ b/src/services/WorkPeriodProcessorService.js @@ -18,11 +18,25 @@ const esClient = helper.getESClient() async function processCreate (message, transactionId) { const workPeriod = message.payload // Find related resourceBooking - const resourceBooking = await esClient.getExtra({ - index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), - transactionId, - id: workPeriod.resourceBookingId - }) + let resourceBooking + try { + resourceBooking = await esClient.getExtra({ + index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), + transactionId, + id: workPeriod.resourceBookingId + }) + } catch (err) { + // if resource booking was not found, it may be because + // it has not yet been created. We should send a retry request. + if (err.httpStatus === 404) { + logger.logFullError(err, { component: 'WorkPeriodProcessorService', context: 'processCreate' }) + await helper.retryFailedProcess(message.topic, workPeriod, workPeriod.resourceBookingId) + return + } else { + throw err + } + } + console.log(`[RB value-999] before update: ${JSON.stringify(resourceBooking)}`) // Get ResourceBooking's existing workPeriods const workPeriods = _.isArray(resourceBooking.body.workPeriods) ? resourceBooking.body.workPeriods : [] diff --git a/test/unit/test.js b/test/unit/test.js index d2d3f66..b89a18c 100644 --- a/test/unit/test.js +++ b/test/unit/test.js @@ -17,7 +17,8 @@ const services = { JobCandidateProcessorService: require('../../src/services/JobCandidateProcessorService'), ResourceBookingProcessorService: require('../../src/services/ResourceBookingProcessorService'), WorkPeriodProcessorService: require('../../src/services/WorkPeriodProcessorService'), - WorkPeriodPaymentProcessorService: require('../../src/services/WorkPeriodPaymentProcessorService') + WorkPeriodPaymentProcessorService: require('../../src/services/WorkPeriodPaymentProcessorService'), + ActionProcessorService: require('../../src/services/ActionProcessorService') } // random transaction id here @@ -172,12 +173,9 @@ describe('General Logic Tests', () => { ) }) it(`Failure - processCreate - ${modelInSpaceCase} not found`, async () => { - try { - await services[`${model}ProcessorService`].processCreate(testData.messages[model].create.message, transactionId) - throw new Error() - } catch (err) { - should.equal(err.message, `id: ${testData.messages[parentModel].create.message.payload.id} "${index}" not found`) - } + const processCreateStub = sandbox.stub(services.ActionProcessorService, 'processCreate').callsFake(() => {}) + await services[`${model}ProcessorService`].processCreate(testData.messages[model].create.message, transactionId) + should.equal(processCreateStub.getCall(0).args[0], testData.messages[model].create.topic) }) it(`Failure - processUpdate - ${modelInSpaceCase} not found`, async () => { From 59ec92decb8838af019c10fe69f6d46fd4e0c97e Mon Sep 17 00:00:00 2001 From: eisbilir Date: Mon, 7 Jun 2021 23:50:09 +0300 Subject: [PATCH 42/55] use retry value from payload --- src/common/helper.js | 8 ++++---- src/services/ActionProcessorService.js | 21 +++++++++------------ src/services/WorkPeriodProcessorService.js | 12 +++++++++--- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/src/common/helper.js b/src/common/helper.js index 9530d99..ae07432 100644 --- a/src/common/helper.js +++ b/src/common/helper.js @@ -12,7 +12,6 @@ const _ = require('lodash') const { Mutex } = require('async-mutex') const m2mAuth = require('tc-core-library-js').auth.m2m const busApi = require('@topcoder-platform/topcoder-bus-api-wrapper') -const ActionProcessorService = require('../services/ActionProcessorService') AWS.config.region = config.esConfig.AWS_REGION @@ -185,10 +184,11 @@ async function postMessageViaWebhook (webhook, message) { * Calls ActionProcessorService to attempt to retry failed process * @param {String} topic the failed topic name * @param {Object} payload the payload - * @param {String} id the id that was the subject of the operation failed + * @param {String} retry how many times has it been retried */ -async function retryFailedProcess (topic, payload, id) { - await ActionProcessorService.processCreate(topic, payload, id) +async function retryFailedProcess (topic, payload, retry) { + const ActionProcessorService = require('../services/ActionProcessorService') + await ActionProcessorService.processCreate(topic, payload, retry) } let busApiClient diff --git a/src/services/ActionProcessorService.js b/src/services/ActionProcessorService.js index 3faac7c..91fa310 100644 --- a/src/services/ActionProcessorService.js +++ b/src/services/ActionProcessorService.js @@ -6,14 +6,11 @@ const Joi = require('@hapi/joi') const logger = require('../common/logger') const helper = require('../common/helper') const config = require('config') -const _ = require('lodash') const localLogger = { debug: ({ context, message }) => logger.debug({ component: 'ActionProcessorService', context, message }) } -const retryMap = {} - /** * Process retry operation message * @param {Object} message the kafka message @@ -25,9 +22,10 @@ async function processRetry (message, transactionId) { return } const { topicServiceMapping } = require('../app') + const retry = message.payload.retry message.topic = message.payload.originalTopic message.payload = message.payload.originalPayload - await topicServiceMapping[message.topic](message, transactionId) + await topicServiceMapping[message.topic](message, transactionId, { retry }) } processRetry.schema = { @@ -49,22 +47,21 @@ processRetry.schema = { * Analyzes the failed process and sends it to bus api to be received again. * @param {String} originalTopic the failed topic name * @param {Object} originalPayload the payload - * @param {String} id the id that was the subject of the operation failed + * @param {Number} retry how many times has it been retried */ -async function processCreate (originalTopic, originalPayload, id) { - const retry = _.defaultTo(retryMap[id], 0) + 1 +async function processCreate (originalTopic, originalPayload, retry) { + retry = retry + 1 if (retry > config.MAX_RETRY) { - localLogger.debug({ context: 'processCreate', message: `retry: ${retry} for ${id} exceeds the max retry: ${config.MAX_RETRY} - ignored` }) + localLogger.debug({ context: 'processCreate', message: `retry: ${retry} for ${originalPayload.id} exceeds the max retry: ${config.MAX_RETRY} - ignored` }) return } - localLogger.debug({ context: 'processCreate', message: `retry: ${retry} for ${id}` }) - retryMap[id] = retry + localLogger.debug({ context: 'processCreate', message: `retry: ${retry} for ${originalPayload.id}` }) const payload = { originalTopic, originalPayload, retry } - setTimeout(async function () { + setTimeout(async () => { await helper.postEvent(config.topics.TAAS_ACTION_RETRY_TOPIC, payload) }, 2 ** retry * config.BASE_RETRY_DELAY) } @@ -72,7 +69,7 @@ async function processCreate (originalTopic, originalPayload, id) { processCreate.schema = { originalTopic: Joi.string().required(), originalPayload: Joi.object().required(), - id: Joi.string().uuid().required() + retry: Joi.number().integer().min(0).required() } module.exports = { diff --git a/src/services/WorkPeriodProcessorService.js b/src/services/WorkPeriodProcessorService.js index 3b91a91..e86b21d 100644 --- a/src/services/WorkPeriodProcessorService.js +++ b/src/services/WorkPeriodProcessorService.js @@ -14,8 +14,9 @@ const esClient = helper.getESClient() * Process create entity message * @param {Object} message the kafka message * @param {String} transactionId + * @param {Object} options */ -async function processCreate (message, transactionId) { +async function processCreate (message, transactionId, options) { const workPeriod = message.payload // Find related resourceBooking let resourceBooking @@ -30,7 +31,7 @@ async function processCreate (message, transactionId) { // it has not yet been created. We should send a retry request. if (err.httpStatus === 404) { logger.logFullError(err, { component: 'WorkPeriodProcessorService', context: 'processCreate' }) - await helper.retryFailedProcess(message.topic, workPeriod, workPeriod.resourceBookingId) + await helper.retryFailedProcess(message.topic, workPeriod, options.retry) return } else { throw err @@ -78,7 +79,12 @@ processCreate.schema = { updatedBy: Joi.string().uuid().allow(null) }).required() }).required(), - transactionId: Joi.string().required() + transactionId: Joi.string().required(), + options: Joi.object().keys({ + retry: Joi.number().integer().min(0).default(0) + }).default({ + retry: 0 + }) } /** From 3716995d1631c5e52135bb486ae58a785c43f05d Mon Sep 17 00:00:00 2001 From: eisbilir Date: Tue, 8 Jun 2021 21:11:36 +0300 Subject: [PATCH 43/55] avoid circular dependency --- src/common/helper.js | 12 ------------ src/services/WorkPeriodProcessorService.js | 3 ++- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/src/common/helper.js b/src/common/helper.js index ae07432..6725df1 100644 --- a/src/common/helper.js +++ b/src/common/helper.js @@ -180,17 +180,6 @@ async function postMessageViaWebhook (webhook, message) { await request.post(webhook).send(message) } -/** - * Calls ActionProcessorService to attempt to retry failed process - * @param {String} topic the failed topic name - * @param {Object} payload the payload - * @param {String} retry how many times has it been retried - */ -async function retryFailedProcess (topic, payload, retry) { - const ActionProcessorService = require('../services/ActionProcessorService') - await ActionProcessorService.processCreate(topic, payload, retry) -} - let busApiClient /** @@ -242,7 +231,6 @@ module.exports = { checkEsMutexRelease, getM2MToken, postMessageViaWebhook, - retryFailedProcess, getBusApiClient, postEvent } diff --git a/src/services/WorkPeriodProcessorService.js b/src/services/WorkPeriodProcessorService.js index e86b21d..ed24304 100644 --- a/src/services/WorkPeriodProcessorService.js +++ b/src/services/WorkPeriodProcessorService.js @@ -9,6 +9,7 @@ const constants = require('../common/constants') const config = require('config') const _ = require('lodash') const esClient = helper.getESClient() +const ActionProcessorService = require('../services/ActionProcessorService') /** * Process create entity message @@ -31,7 +32,7 @@ async function processCreate (message, transactionId, options) { // it has not yet been created. We should send a retry request. if (err.httpStatus === 404) { logger.logFullError(err, { component: 'WorkPeriodProcessorService', context: 'processCreate' }) - await helper.retryFailedProcess(message.topic, workPeriod, options.retry) + await ActionProcessorService.processCreate(message.topic, workPeriod, options.retry) return } else { throw err From b349be1c5ec0a5a50bb967ab157c41f214f62572 Mon Sep 17 00:00:00 2001 From: Maksym Mykhailenko Date: Wed, 9 Jun 2021 14:52:23 +0300 Subject: [PATCH 44/55] refactor: improve retry logic --- config/test.js | 4 ++- src/common/helper.js | 11 ++++++++ src/common/logger.js | 18 ++++++++++++++ src/services/ActionProcessorService.js | 29 ++++++++++++---------- src/services/WorkPeriodProcessorService.js | 10 ++++++-- 5 files changed, 56 insertions(+), 16 deletions(-) diff --git a/config/test.js b/config/test.js index c462fb0..3ca4ffe 100644 --- a/config/test.js +++ b/config/test.js @@ -6,5 +6,7 @@ module.exports = { zapier: { ZAPIER_SWITCH: process.env.ZAPIER_SWITCH || 'ON', ZAPIER_JOB_CANDIDATE_SWITCH: process.env.ZAPIER_JOB_CANDIDATE_SWITCH || 'ON' - } + }, + // don't retry actions during tests because tests for now don't expect it and should be updated first + MAX_RETRY: 0, } diff --git a/src/common/helper.js b/src/common/helper.js index 6725df1..78f78a0 100644 --- a/src/common/helper.js +++ b/src/common/helper.js @@ -225,7 +225,18 @@ async function postEvent (topic, payload) { await client.postEvent(message) } +/** + * Sleep for a given number of milliseconds. + * + * @param {Number} milliseconds the sleep time + * @returns {undefined} + */ +async function sleep (milliseconds) { + return new Promise((resolve) => setTimeout(resolve, milliseconds)) +} + module.exports = { + sleep, getKafkaOptions, getESClient, checkEsMutexRelease, diff --git a/src/common/logger.js b/src/common/logger.js index b533070..2204596 100644 --- a/src/common/logger.js +++ b/src/common/logger.js @@ -56,6 +56,24 @@ logger.logFullError = (err, context = {}) => { err.logged = true } +/** + * Log warning details + * @param {Object} err the error + * @param {Object} context contains extra info about errors + */ +logger.logFullWarning = (err, context = {}) => { + if (!err) { + return + } + if (err.logged) { + return + } + const signature = context.signature ? `${context.signature} : ` : '' + const errMessage = err.message || util.inspect(err).split('\n')[0] + logger.warn({ ..._.pick(context, ['component', 'context']), message: `${signature}${errMessage}` }) + err.logged = true +} + /** * Remove invalid properties from the object and hide long arrays * @param {Object} obj the object diff --git a/src/services/ActionProcessorService.js b/src/services/ActionProcessorService.js index 91fa310..662b0c7 100644 --- a/src/services/ActionProcessorService.js +++ b/src/services/ActionProcessorService.js @@ -48,33 +48,36 @@ processRetry.schema = { * @param {String} originalTopic the failed topic name * @param {Object} originalPayload the payload * @param {Number} retry how many times has it been retried + * + * @returns {Promise|null} returns Promise which would be resolved when retry event sent to Kafka, + * or `null` if it would not be scheduled */ -async function processCreate (originalTopic, originalPayload, retry) { +function scheduleRetry (originalTopic, originalPayload, retry) { retry = retry + 1 if (retry > config.MAX_RETRY) { - localLogger.debug({ context: 'processCreate', message: `retry: ${retry} for ${originalPayload.id} exceeds the max retry: ${config.MAX_RETRY} - ignored` }) + localLogger.debug({ context: 'scheduleRetry', message: `retry: ${retry} for topic: ${originalTopic} id: ${originalPayload.id} exceeds the max retry: ${config.MAX_RETRY} - ignored` }) return } - localLogger.debug({ context: 'processCreate', message: `retry: ${retry} for ${originalPayload.id}` }) + + localLogger.debug({ context: 'scheduleRetry', message: `retry: ${retry} for topic: ${originalTopic} id: ${originalPayload.id}` }) + const payload = { originalTopic, originalPayload, retry } - setTimeout(async () => { - await helper.postEvent(config.topics.TAAS_ACTION_RETRY_TOPIC, payload) - }, 2 ** retry * config.BASE_RETRY_DELAY) -} -processCreate.schema = { - originalTopic: Joi.string().required(), - originalPayload: Joi.object().required(), - retry: Joi.number().integer().min(0).required() + return helper.sleep(2 ** retry * config.BASE_RETRY_DELAY).then(() => + helper.postEvent(config.topics.TAAS_ACTION_RETRY_TOPIC, payload) + ) } module.exports = { - processRetry, - processCreate + processRetry } logger.buildService(module.exports, 'ActionProcessorService') + +// we don't want to wrap this method into service wrappers +// because it would transform this method to `async` while we want to keep it sync +module.exports.scheduleRetry = scheduleRetry diff --git a/src/services/WorkPeriodProcessorService.js b/src/services/WorkPeriodProcessorService.js index ed24304..c40f473 100644 --- a/src/services/WorkPeriodProcessorService.js +++ b/src/services/WorkPeriodProcessorService.js @@ -31,8 +31,14 @@ async function processCreate (message, transactionId, options) { // if resource booking was not found, it may be because // it has not yet been created. We should send a retry request. if (err.httpStatus === 404) { - logger.logFullError(err, { component: 'WorkPeriodProcessorService', context: 'processCreate' }) - await ActionProcessorService.processCreate(message.topic, workPeriod, options.retry) + const schedulePromise = ActionProcessorService.scheduleRetry(message.topic, workPeriod, options.retry) + if (schedulePromise) { + // as retry was scheduled, log this error as warning + logger.logFullWarning(err, { component: 'WorkPeriodProcessorService', context: 'processCreate' }) + } else { + // as retry was not scheduled, then log this error as error + logger.logFullError(err, { component: 'WorkPeriodProcessorService', context: 'processCreate' }) + } return } else { throw err From b570ba3da051612eba153b17aead2c260fb3c053 Mon Sep 17 00:00:00 2001 From: Maksym Mykhailenko Date: Wed, 9 Jun 2021 14:56:12 +0300 Subject: [PATCH 45/55] docs: update README --- README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.md b/README.md index 257b38c..206e352 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,22 @@ The following parameters can be set in config files or in env variables: npm run lint:fix ``` +4. Local config + + In the `taas-es-processor` root directory create `.env` file with the next environment variables. Values for **Auth0 config** should be shared with you on the forum.
+ + ```bash + # Auth0 config + AUTH0_URL= + AUTH0_AUDIENCE= + AUTH0_AUDIENCE_UBAHN= + AUTH0_CLIENT_ID= + AUTH0_CLIENT_SECRET= + ``` + + - Values from this file would be automatically used by many `npm` commands. + - ⚠️ Never commit this file or its copy to the repository! + 5. Start the processor and health check dropin ```bash From 3bcfe88e84fc823f43279308b9efeb64e8a9af7e Mon Sep 17 00:00:00 2001 From: nkumar-topcoder <33625707+nkumar-topcoder@users.noreply.github.com> Date: Wed, 9 Jun 2021 17:30:59 +0530 Subject: [PATCH 46/55] Update WorkPeriodProcessorService.js --- src/services/WorkPeriodProcessorService.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/services/WorkPeriodProcessorService.js b/src/services/WorkPeriodProcessorService.js index 770f774..8350056 100644 --- a/src/services/WorkPeriodProcessorService.js +++ b/src/services/WorkPeriodProcessorService.js @@ -47,6 +47,7 @@ processCreate.schema = { originator: Joi.string().required(), timestamp: Joi.date().required(), 'mime-type': Joi.string().required(), + key: Joi.string().allow(null), payload: Joi.object().keys({ id: Joi.string().uuid().required(), resourceBookingId: Joi.string().uuid().required(), From 9c43b15a83b7c6f78fa1bcc4753ce7d135c8bbdc Mon Sep 17 00:00:00 2001 From: Maksym Mykhailenko Date: Wed, 9 Jun 2021 15:30:29 +0300 Subject: [PATCH 47/55] docs: fix README --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 206e352..791cd07 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,6 @@ The following parameters can be set in config files or in env variables: # Auth0 config AUTH0_URL= AUTH0_AUDIENCE= - AUTH0_AUDIENCE_UBAHN= AUTH0_CLIENT_ID= AUTH0_CLIENT_SECRET= ``` From 0e55d65e01e600f3a58e98b0cedd0fb36bdb4f06 Mon Sep 17 00:00:00 2001 From: xxcxy Date: Fri, 11 Jun 2021 16:43:44 +0800 Subject: [PATCH 48/55] Include addition param key to Postevent method #329 --- src/services/InterviewProcessorService.js | 2 ++ src/services/JobCandidateProcessorService.js | 2 ++ src/services/JobProcessorService.js | 2 ++ src/services/ResourceBookingProcessorService.js | 2 ++ src/services/RoleProcessorService.js | 2 ++ src/services/WorkPeriodPaymentProcessorService.js | 1 + src/services/WorkPeriodProcessorService.js | 1 + 7 files changed, 12 insertions(+) diff --git a/src/services/InterviewProcessorService.js b/src/services/InterviewProcessorService.js index b7d2e2e..3af01a8 100644 --- a/src/services/InterviewProcessorService.js +++ b/src/services/InterviewProcessorService.js @@ -56,6 +56,7 @@ processRequestInterview.schema = { originator: Joi.string().required(), timestamp: Joi.date().required(), 'mime-type': Joi.string().required(), + key: Joi.string().allow(null), payload: Joi.object().keys({ id: Joi.string().uuid().required(), xaiId: Joi.string().allow(null), @@ -176,6 +177,7 @@ processBulkUpdateInterviews.schema = { originator: Joi.string().required(), timestamp: Joi.date().required(), 'mime-type': Joi.string().required(), + key: Joi.string().allow(null), payload: Joi.object().pattern( Joi.string().uuid(), // key - jobCandidateId Joi.object().pattern( diff --git a/src/services/JobCandidateProcessorService.js b/src/services/JobCandidateProcessorService.js index bde7e88..87e66bf 100644 --- a/src/services/JobCandidateProcessorService.js +++ b/src/services/JobCandidateProcessorService.js @@ -91,6 +91,7 @@ processCreate.schema = { originator: Joi.string().required(), timestamp: Joi.date().required(), 'mime-type': Joi.string().required(), + key: Joi.string().allow(null), payload: Joi.object().keys({ id: Joi.string().uuid().required(), jobId: Joi.string().uuid().required(), @@ -160,6 +161,7 @@ processDelete.schema = { originator: Joi.string().required(), timestamp: Joi.date().required(), 'mime-type': Joi.string().required(), + key: Joi.string().allow(null), payload: Joi.object().keys({ id: Joi.string().uuid().required() }).required() diff --git a/src/services/JobProcessorService.js b/src/services/JobProcessorService.js index 08f5525..1fdb954 100644 --- a/src/services/JobProcessorService.js +++ b/src/services/JobProcessorService.js @@ -65,6 +65,7 @@ processCreate.schema = { originator: Joi.string().required(), timestamp: Joi.date().required(), 'mime-type': Joi.string().required(), + key: Joi.string().allow(null), payload: Joi.object().keys({ id: Joi.string().uuid().required(), projectId: Joi.number().integer().required(), @@ -142,6 +143,7 @@ processDelete.schema = { originator: Joi.string().required(), timestamp: Joi.date().required(), 'mime-type': Joi.string().required(), + key: Joi.string().allow(null), payload: Joi.object().keys({ id: Joi.string().uuid().required() }).required() diff --git a/src/services/ResourceBookingProcessorService.js b/src/services/ResourceBookingProcessorService.js index f407b2b..836e3e1 100644 --- a/src/services/ResourceBookingProcessorService.js +++ b/src/services/ResourceBookingProcessorService.js @@ -32,6 +32,7 @@ processCreate.schema = { originator: Joi.string().required(), timestamp: Joi.date().required(), 'mime-type': Joi.string().required(), + key: Joi.string().allow(null), payload: Joi.object().keys({ id: Joi.string().uuid().required(), projectId: Joi.number().integer().required(), @@ -94,6 +95,7 @@ processDelete.schema = { originator: Joi.string().required(), timestamp: Joi.date().required(), 'mime-type': Joi.string().required(), + key: Joi.string().allow(null), payload: Joi.object().keys({ id: Joi.string().uuid().required() }).required() diff --git a/src/services/RoleProcessorService.js b/src/services/RoleProcessorService.js index cf1386f..b183577 100644 --- a/src/services/RoleProcessorService.js +++ b/src/services/RoleProcessorService.js @@ -32,6 +32,7 @@ processCreate.schema = { originator: Joi.string().required(), timestamp: Joi.date().required(), 'mime-type': Joi.string().required(), + key: Joi.string().allow(null), payload: Joi.object().keys({ id: Joi.string().uuid().required(), name: Joi.string().max(50).required(), @@ -103,6 +104,7 @@ processDelete.schema = { originator: Joi.string().required(), timestamp: Joi.date().required(), 'mime-type': Joi.string().required(), + key: Joi.string().allow(null), payload: Joi.object().keys({ id: Joi.string().uuid().required() }).required() diff --git a/src/services/WorkPeriodPaymentProcessorService.js b/src/services/WorkPeriodPaymentProcessorService.js index 736cacb..e1bc583 100644 --- a/src/services/WorkPeriodPaymentProcessorService.js +++ b/src/services/WorkPeriodPaymentProcessorService.js @@ -62,6 +62,7 @@ processCreate.schema = { originator: Joi.string().required(), timestamp: Joi.date().required(), 'mime-type': Joi.string().required(), + key: Joi.string().allow(null), payload: Joi.object().keys({ id: Joi.string().uuid().required(), workPeriodId: Joi.string().uuid().required(), diff --git a/src/services/WorkPeriodProcessorService.js b/src/services/WorkPeriodProcessorService.js index 8350056..eaff8a9 100644 --- a/src/services/WorkPeriodProcessorService.js +++ b/src/services/WorkPeriodProcessorService.js @@ -201,6 +201,7 @@ processDelete.schema = { originator: Joi.string().required(), timestamp: Joi.date().required(), 'mime-type': Joi.string().required(), + key: Joi.string().allow(null), payload: Joi.object().keys({ id: Joi.string().uuid().required() }).required() From 0f0dabb36b5884d4fe45e5bed5b3be13a842b7b0 Mon Sep 17 00:00:00 2001 From: Sushil Shinde Date: Fri, 11 Jun 2021 14:14:32 +0530 Subject: [PATCH 49/55] fix: 1. allow empty strings, 2. added new ratetype enum 'annual' --- src/bootstrap.js | 2 +- src/services/JobCandidateProcessorService.js | 46 +++++++++++--------- src/services/JobProcessorService.js | 6 +-- 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/src/bootstrap.js b/src/bootstrap.js index ab512e6..0d773b9 100644 --- a/src/bootstrap.js +++ b/src/bootstrap.js @@ -9,7 +9,7 @@ const allowedInterviewStatuses = _.values(Interview.Status) global.Promise = require('bluebird') -Joi.rateType = () => Joi.string().valid('hourly', 'daily', 'weekly', 'monthly') +Joi.rateType = () => Joi.string().valid('hourly', 'daily', 'weekly', 'monthly','annual') Joi.jobStatus = () => Joi.string().valid('sourcing', 'in-review', 'assigned', 'closed', 'cancelled') Joi.resourceBookingStatus = () => Joi.string().valid('placed', 'closed', 'cancelled') Joi.jobCandidateStatus = () => Joi.string().valid('open', 'placed', 'selected', 'client rejected - screening', 'client rejected - interview', 'rejected - other', 'cancelled', 'interview', 'topcoder-rejected', 'applied', 'rejected-pre-screen', 'skills-test', 'skills-test', 'phone-screen', 'job-closed', 'offered') diff --git a/src/services/JobCandidateProcessorService.js b/src/services/JobCandidateProcessorService.js index bde7e88..f17b802 100644 --- a/src/services/JobCandidateProcessorService.js +++ b/src/services/JobCandidateProcessorService.js @@ -86,27 +86,31 @@ async function processCreate (message, transactionId) { } processCreate.schema = { - message: Joi.object().keys({ - topic: Joi.string().required(), - originator: Joi.string().required(), - timestamp: Joi.date().required(), - 'mime-type': Joi.string().required(), - payload: Joi.object().keys({ - id: Joi.string().uuid().required(), - jobId: Joi.string().uuid().required(), - userId: Joi.string().uuid().required(), - createdAt: Joi.date().required(), - createdBy: Joi.string().uuid().required(), - updatedAt: Joi.date().allow(null), - updatedBy: Joi.string().uuid().allow(null), - status: Joi.jobCandidateStatus().required(), - externalId: Joi.string().allow(null), - resume: Joi.string().uri().allow(null), - remark: Joi.string().allow(null) - }).required() - }).required(), - transactionId: Joi.string().required() -} + message: Joi.object() + .keys({ + topic: Joi.string().required(), + originator: Joi.string().required(), + timestamp: Joi.date().required(), + "mime-type": Joi.string().required(), + payload: Joi.object() + .keys({ + id: Joi.string().uuid().required(), + jobId: Joi.string().uuid().required(), + userId: Joi.string().uuid().required(), + createdAt: Joi.date().required(), + createdBy: Joi.string().uuid().required(), + updatedAt: Joi.date().allow(null), + updatedBy: Joi.string().uuid().allow(null), + status: Joi.jobCandidateStatus().required(), + externalId: Joi.string().allow(null), + resume: Joi.string().uri().allow(null).allow(''), + remark: Joi.string().allow(null).allow('') + }) + .required(), + }) + .required(), + transactionId: Joi.string().required(), +}; /** * Process update entity message diff --git a/src/services/JobProcessorService.js b/src/services/JobProcessorService.js index 08f5525..19244cc 100644 --- a/src/services/JobProcessorService.js +++ b/src/services/JobProcessorService.js @@ -88,9 +88,9 @@ processCreate.schema = { minSalary: Joi.number().integer().allow(null), maxSalary: Joi.number().integer().allow(null), hoursPerWeek: Joi.number().integer().allow(null), - jobLocation: Joi.string().allow(null), - jobTimezone: Joi.string().allow(null), - currency: Joi.string().allow(null), + jobLocation: Joi.string().allow(null).allow(''), + jobTimezone: Joi.string().allow(null).allow(''), + currency: Joi.string().allow(null).allow(''), roleIds: Joi.array().items(Joi.string().uuid().required()).allow(null) }).required() }).required(), From 51f2c76ed6306459bd620794c631c82dab18d833 Mon Sep 17 00:00:00 2001 From: Sushil Shinde Date: Fri, 11 Jun 2021 14:25:24 +0530 Subject: [PATCH 50/55] ci: deploying this branch on dev env --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0222864..3dc37e3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -72,6 +72,7 @@ workflows: only: - dev - dev-circleci + - change-validatations-in-job-j # Production builds are exectuted only on tagged commits to the # master branch. From a7b8c473c1b469c32251d0d330d60c2f5ab7c92f Mon Sep 17 00:00:00 2001 From: Maksym Mykhailenko Date: Fri, 11 Jun 2021 12:32:41 +0300 Subject: [PATCH 51/55] docs: improve README --- .nvmrc | 1 + README.md | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 .nvmrc diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 0000000..48082f7 --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +12 diff --git a/README.md b/README.md index 791cd07..647f29f 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,13 @@ The following parameters can be set in config files or in env variables: ## Local deployment -1. Make sure that Kafka and Elasticsearch is running as per instructions above. +0. Make sure that Kafka and Elasticsearch is running as per instructions above. + +1. Make sure to use Node v12+ by command `node -v`. We recommend using [NVM](https://github.com/nvm-sh/nvm) to quickly switch to the right version: + + ```bash + nvm use + ``` 2. From the project root directory, run the following command to install the dependencies From 20ed3dbc2a5c0ec3796bee5438e8c303da38d22e Mon Sep 17 00:00:00 2001 From: Maksym Mykhailenko Date: Fri, 11 Jun 2021 13:35:24 +0300 Subject: [PATCH 52/55] fix: lint --- config/test.js | 2 +- src/bootstrap.js | 2 +- src/services/JobCandidateProcessorService.js | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/config/test.js b/config/test.js index 3ca4ffe..c5b51db 100644 --- a/config/test.js +++ b/config/test.js @@ -8,5 +8,5 @@ module.exports = { ZAPIER_JOB_CANDIDATE_SWITCH: process.env.ZAPIER_JOB_CANDIDATE_SWITCH || 'ON' }, // don't retry actions during tests because tests for now don't expect it and should be updated first - MAX_RETRY: 0, + MAX_RETRY: 0 } diff --git a/src/bootstrap.js b/src/bootstrap.js index 0ded4b9..8b9bf51 100644 --- a/src/bootstrap.js +++ b/src/bootstrap.js @@ -9,7 +9,7 @@ const allowedInterviewStatuses = _.values(Interview.Status) global.Promise = require('bluebird') -Joi.rateType = () => Joi.string().valid('hourly', 'daily', 'weekly', 'monthly','annual') +Joi.rateType = () => Joi.string().valid('hourly', 'daily', 'weekly', 'monthly', 'annual') Joi.jobStatus = () => Joi.string().valid('sourcing', 'in-review', 'assigned', 'closed', 'cancelled') Joi.resourceBookingStatus = () => Joi.string().valid('placed', 'closed', 'cancelled') Joi.jobCandidateStatus = () => Joi.string().valid('open', 'placed', 'selected', 'client rejected - screening', 'client rejected - interview', 'rejected - other', 'cancelled', 'interview', 'topcoder-rejected', 'applied', 'rejected-pre-screen', 'skills-test', 'skills-test', 'phone-screen', 'job-closed', 'offered') diff --git a/src/services/JobCandidateProcessorService.js b/src/services/JobCandidateProcessorService.js index dfa9b97..5e6a044 100644 --- a/src/services/JobCandidateProcessorService.js +++ b/src/services/JobCandidateProcessorService.js @@ -91,7 +91,7 @@ processCreate.schema = { topic: Joi.string().required(), originator: Joi.string().required(), timestamp: Joi.date().required(), - "mime-type": Joi.string().required(), + 'mime-type': Joi.string().required(), key: Joi.string().allow(null), payload: Joi.object() .keys({ @@ -107,10 +107,10 @@ processCreate.schema = { resume: Joi.string().uri().allow(null).allow(''), remark: Joi.string().allow(null).allow('') }) - .required(), + .required() }) .required(), - transactionId: Joi.string().required(), + transactionId: Joi.string().required() } /** From 18b386064d82ae89f6ebaa0afa7e96d3787e8632 Mon Sep 17 00:00:00 2001 From: eisbilir Date: Mon, 14 Jun 2021 03:00:10 +0300 Subject: [PATCH 53/55] wp-wpp fields update --- src/bootstrap.js | 2 +- src/scripts/createIndex.js | 7 +++++-- src/services/WorkPeriodPaymentProcessorService.js | 3 +++ src/services/WorkPeriodProcessorService.js | 6 +++--- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/bootstrap.js b/src/bootstrap.js index 8b9bf51..518bf6e 100644 --- a/src/bootstrap.js +++ b/src/bootstrap.js @@ -15,7 +15,7 @@ Joi.resourceBookingStatus = () => Joi.string().valid('placed', 'closed', 'cancel Joi.jobCandidateStatus = () => Joi.string().valid('open', 'placed', 'selected', 'client rejected - screening', 'client rejected - interview', 'rejected - other', 'cancelled', 'interview', 'topcoder-rejected', 'applied', 'rejected-pre-screen', 'skills-test', 'skills-test', 'phone-screen', 'job-closed', 'offered') Joi.workload = () => Joi.string().valid('full-time', 'fractional') Joi.title = () => Joi.string().max(128) -Joi.paymentStatus = () => Joi.string().valid('pending', 'partially-completed', 'completed', 'cancelled') +Joi.paymentStatus = () => Joi.string().valid('pending', 'in-progress', 'partially-completed', 'completed', 'failed', 'noDays') Joi.xaiTemplate = () => Joi.string().valid(...allowedXAITemplates) Joi.interviewStatus = () => Joi.string().valid(...allowedInterviewStatuses) Joi.workPeriodPaymentStatus = () => Joi.string().valid('completed', 'scheduled', 'in-progress', 'failed', 'cancelled') diff --git a/src/scripts/createIndex.js b/src/scripts/createIndex.js index f6f5c07..1cb89ce 100644 --- a/src/scripts/createIndex.js +++ b/src/scripts/createIndex.js @@ -119,8 +119,8 @@ async function createIndex () { startDate: { type: 'date', format: 'yyyy-MM-dd' }, endDate: { type: 'date', format: 'yyyy-MM-dd' }, daysWorked: { type: 'integer' }, - memberRate: { type: 'float' }, - customerRate: { type: 'float' }, + daysPaid: { type: 'integer' }, + paymentTotal: { type: 'float' }, paymentStatus: { type: 'keyword' }, payments: { type: 'nested', @@ -128,6 +128,9 @@ async function createIndex () { id: { type: 'keyword' }, workPeriodId: { type: 'keyword' }, challengeId: { type: 'keyword' }, + memberRate: { type: 'float' }, + customerRate: { type: 'float' }, + days: { type: 'integer' }, amount: { type: 'float' }, status: { type: 'keyword' }, statusDetails: { diff --git a/src/services/WorkPeriodPaymentProcessorService.js b/src/services/WorkPeriodPaymentProcessorService.js index bede8fd..3d8e176 100644 --- a/src/services/WorkPeriodPaymentProcessorService.js +++ b/src/services/WorkPeriodPaymentProcessorService.js @@ -67,6 +67,9 @@ processCreate.schema = { id: Joi.string().uuid().required(), workPeriodId: Joi.string().uuid().required(), challengeId: Joi.string().uuid().allow(null), + memberRate: Joi.number().required(), + customerRate: Joi.number().allow(null), + days: Joi.number().integer().min(1).max(5).required(), amount: Joi.number().greater(0).allow(null), status: Joi.workPeriodPaymentStatus().required(), billingAccountId: Joi.number().allow(null), diff --git a/src/services/WorkPeriodProcessorService.js b/src/services/WorkPeriodProcessorService.js index 4ea661f..bebe869 100644 --- a/src/services/WorkPeriodProcessorService.js +++ b/src/services/WorkPeriodProcessorService.js @@ -77,9 +77,9 @@ processCreate.schema = { projectId: Joi.number().integer().required(), startDate: Joi.string().required(), endDate: Joi.string().required(), - daysWorked: Joi.number().integer().min(0).allow(null), - memberRate: Joi.number().allow(null), - customerRate: Joi.number().allow(null), + daysWorked: Joi.number().integer().min(0).max(5).required(), + daysPaid: Joi.number().integer().min(0).max(5).required(), + paymentTotal: Joi.number().min(0).required(), paymentStatus: Joi.paymentStatus().required(), createdAt: Joi.date().required(), createdBy: Joi.string().uuid().required(), From a8e039a39c0ecaeb81b796a0f99860b414e423ce Mon Sep 17 00:00:00 2001 From: eisbilir Date: Mon, 14 Jun 2021 03:03:37 +0300 Subject: [PATCH 54/55] fix parallelism problem --- .../WorkPeriodPaymentProcessorService.js | 135 ++++-------------- src/services/WorkPeriodProcessorService.js | 120 +++++----------- 2 files changed, 64 insertions(+), 191 deletions(-) diff --git a/src/services/WorkPeriodPaymentProcessorService.js b/src/services/WorkPeriodPaymentProcessorService.js index 3d8e176..33d76df 100644 --- a/src/services/WorkPeriodPaymentProcessorService.js +++ b/src/services/WorkPeriodPaymentProcessorService.js @@ -4,7 +4,6 @@ const Joi = require('@hapi/joi') const config = require('config') -const _ = require('lodash') const logger = require('../common/logger') const helper = require('../common/helper') const constants = require('../common/constants') @@ -12,45 +11,39 @@ const constants = require('../common/constants') const esClient = helper.getESClient() /** - * Process create entity message - * @param {Object} message the kafka message - * @param {String} transactionId - */ + * Process create entity message + * @param {Object} message the kafka message + * @param {String} transactionId + */ async function processCreate (message, transactionId) { - const data = message.payload + const workPeriodPayment = message.payload // find related resourceBooking - const result = await esClient.search({ + const resourceBooking = await esClient.search({ index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), body: { query: { nested: { path: 'workPeriods', query: { - match: { 'workPeriods.id': data.workPeriodId } + match: { 'workPeriods.id': workPeriodPayment.workPeriodId } } } } } }) - if (!result.body.hits.total.value) { - throw new Error(`id: ${data.workPeriodId} "WorkPeriod" not found`) + if (!resourceBooking.body.hits.total.value) { + throw new Error(`id: ${workPeriodPayment.workPeriodId} "WorkPeriod" not found`) } - const resourceBooking = result.body.hits.hits[0]._source - // find related workPeriod record - const workPeriod = _.find(resourceBooking.workPeriods, ['id', data.workPeriodId]) - // Get workPeriod's existing payments - const payments = _.isArray(workPeriod.payments) ? workPeriod.payments : [] - // Append new payment - payments.push(data) - // Assign new payments array to workPeriod - workPeriod.payments = payments - // Update ResourceBooking's workPeriods property - await esClient.updateExtra({ + await esClient.update({ index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), - id: resourceBooking.id, + id: resourceBooking.body.hits.hits[0]._id, transactionId, body: { - doc: { workPeriods: resourceBooking.workPeriods } + script: { + lang: 'painless', + source: 'def wp = ctx._source.workPeriods.find(workPeriod -> workPeriod.id == params.workPeriodPayment.workPeriodId); if(!wp.containsKey("payments") || wp.payments == null){wp["payments"]=[]}wp.payments.add(params.workPeriodPayment)', + params: { workPeriodPayment } + } }, refresh: constants.esRefreshOption }) @@ -90,14 +83,14 @@ processCreate.schema = { } /** - * Process update entity message - * @param {Object} message the kafka message - * @param {String} transactionId - */ + * Process update entity message + * @param {Object} message the kafka message + * @param {String} transactionId + */ async function processUpdate (message, transactionId) { const data = message.payload // find workPeriodPayment in it's parent ResourceBooking - let result = await esClient.search({ + const resourceBooking = await esClient.search({ index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), body: { query: { @@ -110,89 +103,19 @@ async function processUpdate (message, transactionId) { } } }) - if (!result.body.hits.total.value) { + if (!resourceBooking.body.hits.total.value) { throw new Error(`id: ${data.id} "WorkPeriodPayment" not found`) } - const resourceBooking = _.cloneDeep(result.body.hits.hits[0]._source) - let workPeriod = null - let payment = null - let paymentIndex = null - // find workPeriod and workPeriodPayment records - _.forEach(resourceBooking.workPeriods, wp => { - _.forEach(wp.payments, (p, pi) => { - if (p.id === data.id) { - payment = p - paymentIndex = pi - return false - } - }) - if (payment) { - workPeriod = wp - return false - } - }) - let payments - // if WorkPeriodPayment's workPeriodId changed then it must be deleted from the old WorkPeriod - // and added to the new WorkPeriod - if (payment.workPeriodId !== data.workPeriodId) { - // remove payment from payments - payments = _.filter(workPeriod.payments, p => p.id !== data.id) - // assign payments to workPeriod record - workPeriod.payments = payments - // Update old ResourceBooking's workPeriods property - await esClient.updateExtra({ - index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), - id: resourceBooking.id, - transactionId, - body: { - doc: { workPeriods: resourceBooking.workPeriods } - }, - refresh: constants.esRefreshOption - }) - // find workPeriodPayment's new parent WorkPeriod - result = await esClient.search({ - index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), - body: { - query: { - nested: { - path: 'workPeriods', - query: { - match: { 'workPeriods.id': data.workPeriodId } - } - } - } - } - }) - const newResourceBooking = result.body.hits.hits[0]._source - // find WorkPeriod record in ResourceBooking - const newWorkPeriod = _.find(newResourceBooking.workPeriods, ['id', data.workPeriodId]) - // Get WorkPeriod's existing payments - const newPayments = _.isArray(newWorkPeriod.payments) ? newWorkPeriod.payments : [] - // Append new payment - newPayments.push(data) - // Assign new payments array to workPeriod - newWorkPeriod.payments = newPayments - // Update new ResourceBooking's workPeriods property - await esClient.updateExtra({ - index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), - id: newResourceBooking.id, - transactionId, - body: { - doc: { workPeriods: newResourceBooking.workPeriods } - }, - refresh: constants.esRefreshOption - }) - return - } - // update payment record - workPeriod.payments[paymentIndex] = data - // Update ResourceBooking's workPeriods property - await esClient.updateExtra({ + await esClient.update({ index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), - id: resourceBooking.id, + id: resourceBooking.body.hits.hits[0]._id, transactionId, body: { - doc: { workPeriods: resourceBooking.workPeriods } + script: { + lang: 'painless', + source: 'def wp = ctx._source.workPeriods.find(workPeriod -> workPeriod.id == params.data.workPeriodId); wp.payments.removeIf(payment -> payment.id == params.data.id); wp.payments.add(params.data)', + params: { data } + } }, refresh: constants.esRefreshOption }) diff --git a/src/services/WorkPeriodProcessorService.js b/src/services/WorkPeriodProcessorService.js index bebe869..bcf6fc9 100644 --- a/src/services/WorkPeriodProcessorService.js +++ b/src/services/WorkPeriodProcessorService.js @@ -7,16 +7,15 @@ const logger = require('../common/logger') const helper = require('../common/helper') const constants = require('../common/constants') const config = require('config') -const _ = require('lodash') const esClient = helper.getESClient() const ActionProcessorService = require('../services/ActionProcessorService') /** - * Process create entity message - * @param {Object} message the kafka message - * @param {String} transactionId - * @param {Object} options - */ + * Process create entity message + * @param {Object} message the kafka message + * @param {String} transactionId + * @param {Object} options + */ async function processCreate (message, transactionId, options) { const workPeriod = message.payload // Find related resourceBooking @@ -44,20 +43,16 @@ async function processCreate (message, transactionId, options) { throw err } } - - console.log(`[RB value-999] before update: ${JSON.stringify(resourceBooking)}`) - // Get ResourceBooking's existing workPeriods - const workPeriods = _.isArray(resourceBooking.body.workPeriods) ? resourceBooking.body.workPeriods : [] - // Append new workPeriod - workPeriods.push(workPeriod) - // Update ResourceBooking's workPeriods property - console.log(`[WP value-999]: ${JSON.stringify(workPeriod)}`) - await esClient.updateExtra({ + await esClient.update({ index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), - id: workPeriod.resourceBookingId, + id: resourceBooking.body.id, transactionId, body: { - doc: { workPeriods } + script: { + lang: 'painless', + source: 'if(!ctx._source.containsKey("workPeriods") || ctx._source.workPeriods == null){ctx._source["workPeriods"]=[]}ctx._source.workPeriods.add(params.workPeriod)', + params: { workPeriod } + } }, refresh: constants.esRefreshOption }) @@ -96,14 +91,14 @@ processCreate.schema = { } /** - * Process update entity message - * @param {Object} message the kafka message - * @param {String} transactionId - */ + * Process update entity message + * @param {Object} message the kafka message + * @param {String} transactionId + */ async function processUpdate (message, transactionId) { const data = message.payload // find workPeriod in it's parent ResourceBooking - let resourceBooking = await esClient.search({ + const resourceBooking = await esClient.search({ index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), transactionId, body: { @@ -120,62 +115,16 @@ async function processUpdate (message, transactionId) { if (!resourceBooking.body.hits.total.value) { throw new Error(`id: ${data.id} "WorkPeriod" not found`) } - let workPeriods - // if WorkPeriod's resourceBookingId changed then it must be deleted from the old ResourceBooking - // and added to the new ResourceBooking - if (resourceBooking.body.hits.hits[0]._source.id !== data.resourceBookingId) { - // find old workPeriod record, so we can keep it's existing nested payments field - let oldWorkPeriod = _.find(resourceBooking.body.hits.hits[0]._source.workPeriods, ['id', data.id]) - // remove workPeriod from it's old parent - workPeriods = _.filter(resourceBooking.body.hits.hits[0]._source.workPeriods, (workPeriod) => workPeriod.id !== data.id) - // Update old ResourceBooking's workPeriods property - await esClient.updateExtra({ - index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), - id: resourceBooking.body.hits.hits[0]._source.id, - transactionId, - body: { - doc: { workPeriods } - }, - refresh: constants.esRefreshOption - }) - // find workPeriod's new parent ResourceBooking - resourceBooking = await esClient.getExtra({ - index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), - transactionId, - id: data.resourceBookingId - }) - // Get ResourceBooking's existing workPeriods - workPeriods = _.isArray(resourceBooking.body.workPeriods) ? resourceBooking.body.workPeriods : [] - // Update workPeriod record - const newData = _.assign(oldWorkPeriod, data) - // Append updated workPeriod to workPeriods - workPeriods.push(newData) - // Update new ResourceBooking's workPeriods property - await esClient.updateExtra({ - index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), - id: data.resourceBookingId, - transactionId, - body: { - doc: { workPeriods } - }, - refresh: constants.esRefreshOption - }) - return - } - // Update workPeriod record - workPeriods = _.map(resourceBooking.body.hits.hits[0]._source.workPeriods, (workPeriod) => { - if (workPeriod.id === data.id) { - return _.assign(workPeriod, data) - } - return workPeriod - }) - // Update ResourceBooking's workPeriods property - await esClient.updateExtra({ + await esClient.update({ index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), - id: data.resourceBookingId, + id: resourceBooking.body.hits.hits[0]._id, transactionId, body: { - doc: { workPeriods } + script: { + lang: 'painless', + source: 'def wp = ctx._source.workPeriods.find(workPeriod -> workPeriod.id == params.data.id); ctx._source.workPeriods.removeIf(workPeriod -> workPeriod.id == params.data.id); params.data.payments = wp.payments; ctx._source.workPeriods.add(params.data)', + params: { data } + } }, refresh: constants.esRefreshOption }) @@ -184,10 +133,10 @@ async function processUpdate (message, transactionId) { processUpdate.schema = processCreate.schema /** - * Process delete entity message - * @param {Object} message the kafka message - * @param {String} transactionId - */ + * Process delete entity message + * @param {Object} message the kafka message + * @param {String} transactionId + */ async function processDelete (message, transactionId) { const data = message.payload // Find related ResourceBooking @@ -208,15 +157,16 @@ async function processDelete (message, transactionId) { if (!resourceBooking.body.hits.total.value) { throw new Error(`id: ${data.id} "WorkPeriod" not found`) } - // Remove workPeriod from workPeriods - const workPeriods = _.filter(resourceBooking.body.hits.hits[0]._source.workPeriods, (workPeriod) => workPeriod.id !== data.id) - // Update ResourceBooking's workPeriods property - await esClient.updateExtra({ + await esClient.update({ index: config.get('esConfig.ES_INDEX_RESOURCE_BOOKING'), - id: resourceBooking.body.hits.hits[0]._source.id, + id: resourceBooking.body.hits.hits[0]._id, transactionId, body: { - doc: { workPeriods } + script: { + lang: 'painless', + source: 'ctx._source.workPeriods.removeIf(workPeriod -> workPeriod.id == params.data.id)', + params: { data } + } }, refresh: constants.esRefreshOption }) From e1e127d1aec72af3da9eaa2c888dcb642ba41d9c Mon Sep 17 00:00:00 2001 From: eisbilir Date: Tue, 15 Jun 2021 00:01:28 +0300 Subject: [PATCH 55/55] paymentStatus no-days --- src/bootstrap.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap.js b/src/bootstrap.js index 518bf6e..8361092 100644 --- a/src/bootstrap.js +++ b/src/bootstrap.js @@ -15,7 +15,7 @@ Joi.resourceBookingStatus = () => Joi.string().valid('placed', 'closed', 'cancel Joi.jobCandidateStatus = () => Joi.string().valid('open', 'placed', 'selected', 'client rejected - screening', 'client rejected - interview', 'rejected - other', 'cancelled', 'interview', 'topcoder-rejected', 'applied', 'rejected-pre-screen', 'skills-test', 'skills-test', 'phone-screen', 'job-closed', 'offered') Joi.workload = () => Joi.string().valid('full-time', 'fractional') Joi.title = () => Joi.string().max(128) -Joi.paymentStatus = () => Joi.string().valid('pending', 'in-progress', 'partially-completed', 'completed', 'failed', 'noDays') +Joi.paymentStatus = () => Joi.string().valid('pending', 'in-progress', 'partially-completed', 'completed', 'failed', 'no-days') Joi.xaiTemplate = () => Joi.string().valid(...allowedXAITemplates) Joi.interviewStatus = () => Joi.string().valid(...allowedInterviewStatuses) Joi.workPeriodPaymentStatus = () => Joi.string().valid('completed', 'scheduled', 'in-progress', 'failed', 'cancelled')