From a131404723655b38fa092558dc5494497db52390 Mon Sep 17 00:00:00 2001 From: imcaizheng Date: Wed, 10 Mar 2021 06:08:26 +0800 Subject: [PATCH] Create script for migrating `isApplicationPageActive` from CSV file --- scripts/common/helper.js | 84 ++++++++++++++++ scripts/common/logger.js | 10 ++ scripts/recruit-crm-job-import/helper.js | 39 ++++---- scripts/recruit-crm-job-import/index.js | 48 +--------- scripts/recruit-crm-job-import/logger.js | 8 +- scripts/recruit-crm-job-sync/README.md | 80 ++++++++++++++++ scripts/recruit-crm-job-sync/config.js | 20 ++++ scripts/recruit-crm-job-sync/constants.js | 15 +++ scripts/recruit-crm-job-sync/example_data.csv | 8 ++ scripts/recruit-crm-job-sync/helper.js | 60 ++++++++++++ scripts/recruit-crm-job-sync/index.js | 96 +++++++++++++++++++ scripts/recruit-crm-job-sync/logger.js | 6 ++ scripts/recruit-crm-job-sync/report.js | 59 ++++++++++++ 13 files changed, 464 insertions(+), 69 deletions(-) create mode 100644 scripts/common/helper.js create mode 100644 scripts/common/logger.js create mode 100644 scripts/recruit-crm-job-sync/README.md create mode 100644 scripts/recruit-crm-job-sync/config.js create mode 100644 scripts/recruit-crm-job-sync/constants.js create mode 100644 scripts/recruit-crm-job-sync/example_data.csv create mode 100644 scripts/recruit-crm-job-sync/helper.js create mode 100644 scripts/recruit-crm-job-sync/index.js create mode 100644 scripts/recruit-crm-job-sync/logger.js create mode 100644 scripts/recruit-crm-job-sync/report.js diff --git a/scripts/common/helper.js b/scripts/common/helper.js new file mode 100644 index 00000000..fdda34c7 --- /dev/null +++ b/scripts/common/helper.js @@ -0,0 +1,84 @@ +/* + * Provide some commonly used functions for scripts. + */ +const csv = require('csv-parser') +const fs = require('fs') +const request = require('superagent') + +/** + * Load CSV data from file. + * + * @param {String} pathname the pathname for the file + * @param {Object} fieldNameMap mapping values of headers + * @returns {Array} the result jobs data + */ +async function loadCSVFromFile (pathname, fieldNameMap = {}) { + let lnum = 1 + const result = [] + return new Promise((resolve, reject) => { + fs.createReadStream(pathname) + .pipe(csv({ + mapHeaders: ({ header }) => fieldNameMap[header] || header + })) + .on('data', (data) => { + result.push({ ...data, _lnum: lnum }) + lnum += 1 + }) + .on('error', err => reject(err)) + .on('end', () => resolve(result)) + }) +} + +/** + * Get pathname from command line arguments. + * + * @returns {String} the pathname + */ +function getPathnameFromCommandline () { + if (process.argv.length < 3) { + throw new Error('pathname for the csv file is required') + } + const pathname = process.argv[2] + if (!fs.existsSync(pathname)) { + throw new Error(`pathname: ${pathname} path not exist`) + } + if (!fs.lstatSync(pathname).isFile()) { + throw new Error(`pathname: ${pathname} path is not a regular file`) + } + return pathname +} + +/** + * Sleep for a given number of milliseconds. + * + * @param {Number} milliseconds the sleep time + * @returns {undefined} + */ +async function sleep (milliseconds) { + return new Promise((resolve) => setTimeout(resolve, milliseconds)) +} + +/** + * Find taas job by external id. + * + * @param {String} token the auth token + * @param {String} taasApiUrl url for TaaS API + * @param {String} externalId the external id + * @returns {Object} the result + */ +async function getJobByExternalId (token, taasApiUrl, externalId) { + const { body: jobs } = await request.get(`${taasApiUrl}/jobs`) + .query({ externalId }) + .set('Authorization', `Bearer ${token}`) + if (!jobs.length) { + throw new Error(`externalId: ${externalId} job not found`) + } + return jobs[0] +} + +module.exports = { + loadCSVFromFile, + getPathnameFromCommandline, + sleep, + getJobByExternalId +} diff --git a/scripts/common/logger.js b/scripts/common/logger.js new file mode 100644 index 00000000..273b3841 --- /dev/null +++ b/scripts/common/logger.js @@ -0,0 +1,10 @@ +/* + * Logger for scripts. + */ + +module.exports = { + info: (message) => console.log(`INFO: ${message}`), + debug: (message) => console.log(`DEBUG: ${message}`), + warn: (message) => console.log(`WARN: ${message}`), + error: (message) => console.log(`ERROR: ${message}`) +} diff --git a/scripts/recruit-crm-job-import/helper.js b/scripts/recruit-crm-job-import/helper.js index 5cdf10ff..43591ff3 100644 --- a/scripts/recruit-crm-job-import/helper.js +++ b/scripts/recruit-crm-job-import/helper.js @@ -2,18 +2,27 @@ * Provide some commonly used functions for the RCRM import script. */ const config = require('./config') +const _ = require('lodash') const request = require('superagent') -const { getM2MToken } = require('../../src/common/helper') +const commonHelper = require('../common/helper') -/** - * Sleep for a given number of milliseconds. - * - * @param {Number} milliseconds the sleep time - * @returns {undefined} +/* + * Function to get M2M token + * @returns {Promise} */ -async function sleep (milliseconds) { - return new Promise((resolve) => setTimeout(resolve, milliseconds)) -} +const getM2MToken = (() => { + const m2mAuth = require('tc-core-library-js').auth.m2m + const m2m = m2mAuth(_.pick(config, [ + 'AUTH0_URL', + 'AUTH0_AUDIENCE', + 'AUTH0_CLIENT_ID', + 'AUTH0_CLIENT_SECRET', + 'AUTH0_PROXY_SERVER_URL' + ])) + return async () => { + return await m2m.getMachineToken(config.AUTH0_CLIENT_ID, config.AUTH0_CLIENT_SECRET) + } +})() /** * Create a new job via taas api. @@ -38,13 +47,7 @@ async function createJob (data) { */ async function getJobByExternalId (externalId) { const token = await getM2MToken() - const { body: jobs } = await request.get(`${config.TAAS_API_URL}/jobs`) - .query({ externalId }) - .set('Authorization', `Bearer ${token}`) - if (!jobs.length) { - throw new Error(`externalId: ${externalId} job not found`) - } - return jobs[0] + return commonHelper.getJobByExternalId(token, config.TAAS_API_URL, externalId) } /** @@ -131,7 +134,9 @@ async function getProjectByDirectProjectId (directProjectId) { } module.exports = { - sleep, + sleep: commonHelper.sleep, + loadCSVFromFile: commonHelper.loadCSVFromFile, + getPathnameFromCommandline: commonHelper.getPathnameFromCommandline, createJob, getJobByExternalId, updateResourceBookingStatus, diff --git a/scripts/recruit-crm-job-import/index.js b/scripts/recruit-crm-job-import/index.js index 427a64f6..c82254ab 100644 --- a/scripts/recruit-crm-job-import/index.js +++ b/scripts/recruit-crm-job-import/index.js @@ -2,8 +2,6 @@ * Script to import Jobs data from Recruit CRM to Taas API. */ -const csv = require('csv-parser') -const fs = require('fs') const Joi = require('joi') .extend(require('@joi/date')) const _ = require('lodash') @@ -38,48 +36,6 @@ function validateJob (job) { return jobSchema.validate(job) } -/** - * Load Recruit CRM jobs data from file. - * - * @param {String} pathname the pathname for the file - * @returns {Array} the result jobs data - */ -async function loadRcrmJobsFromFile (pathname) { - let lnum = 1 - const result = [] - return new Promise((resolve, reject) => { - fs.createReadStream(pathname) - .pipe(csv({ - mapHeaders: ({ header }) => constants.fieldNameMap[header] || header - })) - .on('data', (data) => { - result.push({ ...data, _lnum: lnum }) - lnum += 1 - }) - .on('error', err => reject(err)) - .on('end', () => resolve(result)) - }) -} - -/** - * Get pathname for a csv file from command line arguments. - * - * @returns {undefined} - */ -function getPathname () { - if (process.argv.length < 3) { - throw new Error('pathname for the csv file is required') - } - const pathname = process.argv[2] - if (!fs.existsSync(pathname)) { - throw new Error(`pathname: ${pathname} path not exist`) - } - if (!fs.lstatSync(pathname).isFile()) { - throw new Error(`pathname: ${pathname} path is not a regular file`) - } - return pathname -} - /** * Process single job data. The processing consists of: * - Validate the data. @@ -146,8 +102,8 @@ async function processJob (job, info = []) { * @returns {undefined} */ async function main () { - const pathname = getPathname() - const jobs = await loadRcrmJobsFromFile(pathname) + const pathname = helper.getPathnameFromCommandline() + const jobs = await helper.loadCSVFromFile(pathname, constants.fieldNameMap) const report = new Report() for (const job of jobs) { logger.debug(`processing line #${job._lnum} - ${JSON.stringify(job)}`) diff --git a/scripts/recruit-crm-job-import/logger.js b/scripts/recruit-crm-job-import/logger.js index ccb00102..dc38cab5 100644 --- a/scripts/recruit-crm-job-import/logger.js +++ b/scripts/recruit-crm-job-import/logger.js @@ -1,10 +1,6 @@ /* * Logger for the RCRM import script. */ +const logger = require('../common/logger') -module.exports = { - info: (message) => console.log(`INFO: ${message}`), - debug: (message) => console.log(`DEBUG: ${message}`), - warn: (message) => console.log(`WARN: ${message}`), - error: (message) => console.log(`ERROR: ${message}`) -} +module.exports = logger diff --git a/scripts/recruit-crm-job-sync/README.md b/scripts/recruit-crm-job-sync/README.md new file mode 100644 index 00000000..38d85877 --- /dev/null +++ b/scripts/recruit-crm-job-sync/README.md @@ -0,0 +1,80 @@ +Recruit CRM Job Data Sync Script +=== + +# Configuration +Configuration file is at `./scripts/recruit-crm-job-sync/config.js`. + + +# Usage +``` bash +node scripts/recruit-crm-job-sync +``` + +By default the script updates jobs via `TC_API`. + +# Example + +1. Follow the README for `taas-apis` to deploy Taas API locally +2. Create two jobs via `Jobs > create job with booking manager` in Postman, with external ids `51913016` and `51892637` for each of the jobs. + + **NOTE**: The external ids `51913016` and `51902826` could be found at `scripts/recruit-crm-job-sync/example_data.csv` under the Slug column. + +3. Configure env variable `RCRM_SYNC_TAAS_API_URL` so that the script could make use of the local API: + + ``` bash + export RCRM_SYNC_TAAS_API_URL=http://localhost:3000/api/v5 + ``` + +4. Run the script against the sample CSV file and pipe the output from the script to a temporary file: + + ``` bash + node scripts/recruit-crm-job-sync scripts/recruit-crm-job-sync/example_data.csv | tee /tmp/report.txt + ``` + + The output should be like this: + + ``` bash + DEBUG: processing line #1 - {"ID":"1","Name":"Data job Engineer","Description":"","Qualification":"","Specialization":"","Minimum Experience In Years":"1","Maximum Experience In Years":"3","Minimum Annual Salary":"10","Maximum Annual Salary":"20","Number Of Openings":"2","Job Status":"Closed","Company":"company 1","Contact":" ","Currency":"$","allowApply":"Yes","Collaborator":"","Locality":"","City":"","Job Code":"J123456","Createdby":"abc","Created On":"02-Jun-20","Updated By":"abc","Updated On":"17-Feb-21","Owner":"abc","Custom Column 1":"","Custom Column 2":"","Custom Column 3":"","Custom Column 4":"","Custom Column 5":"","Custom Column 6":"","Custom Column 7":"","Custom Column 8":"","Custom Column 9":"","Custom Column 10":"","Custom Column 11":"","Custom Column 12":"","Custom Column 13":"","Custom Column 14":"","Custom Column 15":"","externalId":"51892637","_lnum":1} + ERROR: #1 - [EXTERNAL_ID_NOT_FOUND] externalId: 51892637 job not found + DEBUG: processed line #1 + DEBUG: processing line #2 - {"ID":"2","Name":"JAVA coffee engineer","Description":"","Qualification":"","Specialization":"","Minimum Experience In Years":"2","Maximum Experience In Years":"5","Minimum Annual Salary":"10","Maximum Annual Salary":"20","Number Of Openings":"10","Job Status":"Closed","Company":"company 2","Contact":"abc","Currency":"$","allowApply":"Yes","Collaborator":"","Locality":"","City":"","Job Code":"J123457","Createdby":"abc","Created On":"02-Jun-20","Updated By":"abc","Updated On":"12-Nov-20","Owner":"abc","Custom Column 1":"","Custom Column 2":"","Custom Column 3":"","Custom Column 4":"","Custom Column 5":"","Custom Column 6":"","Custom Column 7":"","Custom Column 8":"","Custom Column 9":"","Custom Column 10":"","Custom Column 11":"","Custom Column 12":"","Custom Column 13":"","Custom Column 14":"","Custom Column 15":"","externalId":"51913016","_lnum":2} + DEBUG: jobId: 34cee9aa-e45f-47ed-9555-ffd3f7196fec isApplicationPageActive(current): false - isApplicationPageActive(to be synced): true + INFO: #2 - id: 34cee9aa-e45f-47ed-9555-ffd3f7196fec isApplicationPageActive: true "job" updated + DEBUG: processed line #2 + DEBUG: processing line #3 - {"ID":"3","Name":"QA Seleinium","Description":"","Qualification":"","Specialization":"","Minimum Experience In Years":"3","Maximum Experience In Years":"7","Minimum Annual Salary":"10","Maximum Annual Salary":"20","Number Of Openings":"4","Job Status":"Canceled","Company":"company 3","Contact":" ","Currency":"$","allowApply":"No","Collaborator":"","Locality":"","City":"","Job Code":"J123458","Createdby":"abc","Created On":"04-Jun-20","Updated By":"abc","Updated On":"12-Nov-20","Owner":"abc","Custom Column 1":"","Custom Column 2":"","Custom Column 3":"","Custom Column 4":"","Custom Column 5":"","Custom Column 6":"","Custom Column 7":"","Custom Column 8":"","Custom Column 9":"","Custom Column 10":"","Custom Column 11":"","Custom Column 12":"","Custom Column 13":"","Custom Column 14":"","Custom Column 15":"","externalId":"51902826","_lnum":3} + DEBUG: jobId: 4acde317-c364-4b79-aa77-295b98143c8b isApplicationPageActive(current): false - isApplicationPageActive(to be synced): false + WARN: #3 - isApplicationPageActive is already set + DEBUG: processed line #3 + DEBUG: processing line #4 - {"ID":"5","Name":"Data Engineers and Data Architects","Description":"","Qualification":"","Specialization":"","Minimum Experience In Years":"4","Maximum Experience In Years":"9","Minimum Annual Salary":"10","Maximum Annual Salary":"20","Number Of Openings":"8","Job Status":"Closed","Company":"company 4","Contact":" ","Currency":"$","allowApply":"Yes","Collaborator":"","Locality":"","City":"","Job Code":"J123459","Createdby":"abc","Created On":"09-Jun-20","Updated By":"abc","Updated On":"12-Nov-20","Owner":"abc","Custom Column 1":"","Custom Column 2":"","Custom Column 3":"","Custom Column 4":"","Custom Column 5":"","Custom Column 6":"","Custom Column 7":"","Custom Column 8":"","Custom Column 9":"","Custom Column 10":"","Custom Column 11":"","Custom Column 12":"","Custom Column 13":"","Custom Column 14":"","Custom Column 15":"","externalId":"51811161","_lnum":4} + ERROR: #4 - [EXTERNAL_ID_NOT_FOUND] externalId: 51811161 job not found + DEBUG: processed line #4 + DEBUG: processing line #5 - {"ID":"6","Name":"Docker Engineer","Description":"Java & J2EE or Python, Docker, Kubernetes, AWS or GCP","Qualification":"","Specialization":"","Minimum Experience In Years":"5","Maximum Experience In Years":"10","Minimum Annual Salary":"10","Maximum Annual Salary":"20","Number Of Openings":"5","Job Status":"Closed","Company":"company 5","Contact":" ","Currency":"$","allowApply":"No","Collaborator":"","Locality":"","City":"","Job Code":"J123460","Createdby":"abc","Created On":"12-Jun-20","Updated By":"abc","Updated On":"12-Nov-20","Owner":"abc","Custom Column 1":"","Custom Column 2":"","Custom Column 3":"","Custom Column 4":"","Custom Column 5":"","Custom Column 6":"","Custom Column 7":"","Custom Column 8":"","Custom Column 9":"","Custom Column 10":"","Custom Column 11":"","Custom Column 12":"","Custom Column 13":"","Custom Column 14":"","Custom Column 15":"","externalId":"51821342","_lnum":5} + ERROR: #5 - [EXTERNAL_ID_NOT_FOUND] externalId: 51821342 job not found + DEBUG: processed line #5 + DEBUG: processing line #6 - {"ID":"7","Name":"lambda Developers","Description":"","Qualification":"","Specialization":"","Minimum Experience In Years":"0","Maximum Experience In Years":"0","Minimum Annual Salary":"10","Maximum Annual Salary":"20","Number Of Openings":"2","Job Status":"Closed","Company":"company 6","Contact":"abc","Currency":"$","allowApply":"Yes","Collaborator":"","Locality":"","City":"","Job Code":"J123461","Createdby":"abc","Created On":"12-Jun-20","Updated By":"abc","Updated On":"12-Nov-20","Owner":"abc","Custom Column 1":"","Custom Column 2":"","Custom Column 3":"","Custom Column 4":"","Custom Column 5":"","Custom Column 6":"","Custom Column 7":"","Custom Column 8":"","Custom Column 9":"","Custom Column 10":"","Custom Column 11":"","Custom Column 12":"","Custom Column 13":"","Custom Column 14":"","Custom Column 15":"","externalId":"51831524","_lnum":6} + ERROR: #6 - [EXTERNAL_ID_NOT_FOUND] externalId: 51831524 job not found + DEBUG: processed line #6 + DEBUG: processing line #7 - {"ID":"","Name":"","Description":"","Qualification":"","Specialization":"","Minimum Experience In Years":"","Maximum Experience In Years":"","Minimum Annual Salary":"","Maximum Annual Salary":"","Number Of Openings":"","Job Status":"","Company":"","Contact":"","Currency":"","allowApply":"","Collaborator":"","Locality":"","City":"","Job Code":"","Createdby":"","Created On":"","Updated By":"","Updated On":"","Owner":"","Custom Column 1":"","Custom Column 2":"","Custom Column 3":"","Custom Column 4":"","Custom Column 5":"","Custom Column 6":"","Custom Column 7":"","Custom Column 8":"","Custom Column 9":"","Custom Column 10":"","Custom Column 11":"","Custom Column 12":"","Custom Column 13":"","Custom Column 14":"","Custom Column 15":"","externalId":"","_lnum":7} + ERROR: #7 - "allowApply" must be one of [Yes, No] + DEBUG: processed line #7 + INFO: === summary === + INFO: No. of records read = 7 + INFO: No. of records updated for field isApplicationPageActive = true = 1 + INFO: No. of records updated for field isApplicationPageActive = false = 0 + INFO: No. of records : externalId not found = 4 + INFO: No. of records failed(all) = 5 + INFO: No. of records failed(excluding "externalId not found") = 1 + INFO: No. of records skipped = 1 + INFO: done! + ``` + + The following command could be used to extract the summary from the output: + + ``` bash + cat /tmp/report.txt | grep 'No. of records' | cut -d' ' -f2- + ``` + + To list all skipped lines: + + ``` bash + cat /tmp/report.txt | grep 'WARN' -B 3 diff --git a/scripts/recruit-crm-job-sync/config.js b/scripts/recruit-crm-job-sync/config.js new file mode 100644 index 00000000..d4193862 --- /dev/null +++ b/scripts/recruit-crm-job-sync/config.js @@ -0,0 +1,20 @@ +/* + * Configuration for the RCRM sync script. + * Namespace is created to allow to configure the env variables for this script independently. + */ + +const config = require('config') + +const namespace = process.env.RCRM_SYNC_CONFIG_NAMESAPCE || 'RCRM_SYNC_' + +module.exports = { + SLEEP_TIME: process.env[`${namespace}SLEEP_TIME`] || 500, + TAAS_API_URL: process.env[`${namespace}TAAS_API_URL`] || config.TC_API, + + AUTH0_URL: process.env[`${namespace}AUTH0_URL`] || config.AUTH0_URL, + AUTH0_AUDIENCE: process.env[`${namespace}AUTH0_AUDIENCE`] || config.AUTH0_AUDIENCE, + TOKEN_CACHE_TIME: process.env[`${namespace}TOKEN_CACHE_TIME`] || config.TOKEN_CACHE_TIME, + AUTH0_CLIENT_ID: process.env[`${namespace}AUTH0_CLIENT_ID`] || config.AUTH0_CLIENT_ID, + AUTH0_CLIENT_SECRET: process.env[`${namespace}AUTH0_CLIENT_SECRET`] || config.AUTH0_CLIENT_SECRET, + AUTH0_PROXY_SERVER_URL: process.env[`${namespace}AUTH0_PROXY_SERVER_URL`] || config.AUTH0_PROXY_SERVER_URL +} diff --git a/scripts/recruit-crm-job-sync/constants.js b/scripts/recruit-crm-job-sync/constants.js new file mode 100644 index 00000000..2d06ef96 --- /dev/null +++ b/scripts/recruit-crm-job-sync/constants.js @@ -0,0 +1,15 @@ +/* + * Constants for the RCRM sync script. + */ + +module.exports = { + ProcessingStatus: { + Successful: 'successful', + Failed: 'failed', + Skipped: 'skipped' + }, + fieldNameMap: { + 'Allow Apply': 'allowApply', + Slug: 'externalId' + } +} diff --git a/scripts/recruit-crm-job-sync/example_data.csv b/scripts/recruit-crm-job-sync/example_data.csv new file mode 100644 index 00000000..44ce1c88 --- /dev/null +++ b/scripts/recruit-crm-job-sync/example_data.csv @@ -0,0 +1,8 @@ +ID,Name,Description,Qualification,Specialization,Minimum Experience In Years,Maximum Experience In Years,Minimum Annual Salary,Maximum Annual Salary,Number Of Openings,Job Status,Company,Contact,Currency,Allow Apply,Collaborator,Locality,City,Job Code,Createdby,Created On,Updated By,Updated On,Owner,Custom Column 1,Custom Column 2,Custom Column 3,Custom Column 4,Custom Column 5,Custom Column 6,Custom Column 7,Custom Column 8,Custom Column 9,Custom Column 10,Custom Column 11,Custom Column 12,Custom Column 13,Custom Column 14,Custom Column 15,Slug +1,Data job Engineer,,,,1,3,10,20,2,Closed,company 1, ,$,Yes,,,,J123456,abc,02-Jun-20,abc,17-Feb-21,abc,,,,,,,,,,,,,,,,51892637 +2,JAVA coffee engineer,,,,2,5,10,20,10,Closed,company 2,abc,$,Yes,,,,J123457,abc,02-Jun-20,abc,12-Nov-20,abc,,,,,,,,,,,,,,,,51913016 +3,QA Seleinium,,,,3,7,10,20,4,Canceled,company 3, ,$,No,,,,J123458,abc,04-Jun-20,abc,12-Nov-20,abc,,,,,,,,,,,,,,,,51902826 +5,Data Engineers and Data Architects,,,,4,9,10,20,8,Closed,company 4, ,$,Yes,,,,J123459,abc,09-Jun-20,abc,12-Nov-20,abc,,,,,,,,,,,,,,,,51811161 +6,Docker Engineer,"Java & J2EE or Python, Docker, Kubernetes, AWS or GCP",,,5,10,10,20,5,Closed,company 5, ,$,No,,,,J123460,abc,12-Jun-20,abc,12-Nov-20,abc,,,,,,,,,,,,,,,,51821342 +7,lambda Developers,,,,0,0,10,20,2,Closed,company 6,abc,$,Yes,,,,J123461,abc,12-Jun-20,abc,12-Nov-20,abc,,,,,,,,,,,,,,,,51831524 +,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, \ No newline at end of file diff --git a/scripts/recruit-crm-job-sync/helper.js b/scripts/recruit-crm-job-sync/helper.js new file mode 100644 index 00000000..e28efb17 --- /dev/null +++ b/scripts/recruit-crm-job-sync/helper.js @@ -0,0 +1,60 @@ +/* + * Provide some commonly used functions for the RCRM sync script. + */ +const config = require('./config') +const _ = require('lodash') +const commonHelper = require('../common/helper') +const request = require('superagent') + +/* + * Function to get M2M token + * @returns {Promise} + */ +const getM2MToken = (() => { + const m2mAuth = require('tc-core-library-js').auth.m2m + const m2m = m2mAuth(_.pick(config, [ + 'AUTH0_URL', + 'AUTH0_AUDIENCE', + 'AUTH0_CLIENT_ID', + 'AUTH0_CLIENT_SECRET', + 'AUTH0_PROXY_SERVER_URL' + ])) + return async () => { + return await m2m.getMachineToken(config.AUTH0_CLIENT_ID, config.AUTH0_CLIENT_SECRET) + } +})() + +/** + * Find taas job by external id. + * + * @param {String} externalId the external id + * @returns {Object} the result + */ +async function getJobByExternalId (externalId) { + const token = await getM2MToken() + return commonHelper.getJobByExternalId(token, config.TAAS_API_URL, externalId) +} + +/** + * Partially update a job. + * + * @param {String} jobId the job id + * @param {Object} data the data to be updated + * @returns {Object} the result job + */ +async function updateJob (jobId, data) { + const token = await getM2MToken() + const { body: job } = await request.patch(`${config.TAAS_API_URL}/jobs/${jobId}`) + .set('Authorization', `Bearer ${token}`) + .set('Content-Type', 'application/json') + .send(data) + return job +} + +module.exports = { + sleep: commonHelper.sleep, + loadCSVFromFile: commonHelper.loadCSVFromFile, + getPathnameFromCommandline: commonHelper.getPathnameFromCommandline, + getJobByExternalId, + updateJob +} diff --git a/scripts/recruit-crm-job-sync/index.js b/scripts/recruit-crm-job-sync/index.js new file mode 100644 index 00000000..f10cd2e9 --- /dev/null +++ b/scripts/recruit-crm-job-sync/index.js @@ -0,0 +1,96 @@ +/* + * Script to sync values of Jobs from Recruit CRM to Taas API. + */ + +const Joi = require('joi') +const Report = require('./report') +const config = require('./config') +const helper = require('./helper') +const constants = require('./constants') +const logger = require('./logger') + +const jobSchema = Joi.object({ + allowApply: Joi.string().valid('Yes', 'No').required(), + externalId: Joi.string().allow('') +}).unknown(true) + +/** + * Process single job data. The processing consists of: + * - Validate the data. + * - Skip processing if externalId is missing. + * - Search job by externalId and update its `isApplicationPageActive` property + (skip processing if `isApplicationPageActive` is already set). + * + * @param {Object} job the job data + * @param {Array} info contains processing details + * @returns {Object} + */ +async function processJob (job, info = []) { + // validate the data + const { value: data, error } = jobSchema.validate(job) + data.isApplicationPageActive = data.allowApply === 'Yes' + if (error) { + info.push({ text: error.details[0].message, tag: 'validation_error' }) + return { status: constants.ProcessingStatus.Failed, info } + } + // skip processing if externalId is missing + if (!data.externalId) { + info.push({ text: 'externalId is missing', tag: 'external_id_missing' }) + return { status: constants.ProcessingStatus.Skipped, info } + } + try { + // search job by externalId and update its `isApplicationPageActive` property + const existingJob = await helper.getJobByExternalId(data.externalId) + logger.debug(`jobId: ${existingJob.id} isApplicationPageActive(current): ${existingJob.isApplicationPageActive} - isApplicationPageActive(to be synced): ${data.isApplicationPageActive}`) + // skip processing if `isApplicationPageActive` is already set + if (existingJob.isApplicationPageActive === data.isApplicationPageActive) { + info.push({ text: 'isApplicationPageActive is already set', tag: 'is_application_page_active_already_set' }) + return { status: constants.ProcessingStatus.Skipped, info } + } + const updatedJob = await helper.updateJob(existingJob.id, { isApplicationPageActive: data.allowApply === 'Yes' }) + info.push({ text: `id: ${existingJob.id} isApplicationPageActive: ${updatedJob.isApplicationPageActive} "job" updated`, tag: 'job_is_application_page_active_updated', currentValue: updatedJob.isApplicationPageActive }) + return { status: constants.ProcessingStatus.Successful, info } + } catch (err) { + if (!(err.message && err.message.includes('job not found'))) { + throw err + } + info.push({ text: `[EXTERNAL_ID_NOT_FOUND] ${err.message}`, tag: 'external_id_not_found' }) + return { status: constants.ProcessingStatus.Failed, info } + } +} + +/** + * The entry of the script. + * + * @returns {undefined} + */ +async function main () { + const pathname = helper.getPathnameFromCommandline() + const jobs = await helper.loadCSVFromFile(pathname, constants.fieldNameMap) + const report = new Report() + for (const job of jobs) { + logger.debug(`processing line #${job._lnum} - ${JSON.stringify(job)}`) + try { + const result = await processJob(job) + report.add({ lnum: job._lnum, ...result }) + } catch (err) { + if (err.response) { + report.add({ lnum: job._lnum, status: constants.ProcessingStatus.Failed, info: [{ text: err.response.error.toString().split('\n')[0], tag: 'request_error' }] }) + } else { + report.add({ lnum: job._lnum, status: constants.ProcessingStatus.Failed, info: [{ text: err.message, tag: 'internal_error' }] }) + } + } + report.print() + logger.debug(`processed line #${job._lnum}`) + await helper.sleep(config.SLEEP_TIME) + } + report.printSummary() +} + +main().then(() => { + logger.info('done!') + process.exit() +}).catch(err => { + logger.error(err.message) + process.exit(1) +}) diff --git a/scripts/recruit-crm-job-sync/logger.js b/scripts/recruit-crm-job-sync/logger.js new file mode 100644 index 00000000..5c6afcd8 --- /dev/null +++ b/scripts/recruit-crm-job-sync/logger.js @@ -0,0 +1,6 @@ +/* + * Logger for the RCRM sync script. + */ +const logger = require('../common/logger') + +module.exports = logger diff --git a/scripts/recruit-crm-job-sync/report.js b/scripts/recruit-crm-job-sync/report.js new file mode 100644 index 00000000..c24a7a75 --- /dev/null +++ b/scripts/recruit-crm-job-sync/report.js @@ -0,0 +1,59 @@ +/* + * The Report class. + */ + +const logger = require('./logger') +const constants = require('./constants') +const _ = require('lodash') + +class Report { + constructor () { + this.messages = [] + } + + // append a message to the report + add (message) { + this.messages.push(message) + } + + // print the last message to the console + print () { + const lastMessage = this.messages[this.messages.length - 1] + const output = `#${lastMessage.lnum} - ${_.map(lastMessage.info, 'text').join('; ')}` + if (lastMessage.status === constants.ProcessingStatus.Skipped) { + logger.warn(output) + } + if (lastMessage.status === constants.ProcessingStatus.Successful) { + logger.info(output) + } + if (lastMessage.status === constants.ProcessingStatus.Failed) { + logger.error(output) + } + } + + // print a summary to the console + printSummary () { + const groups = _.groupBy(this.messages, 'status') + const groupsByTag = _.groupBy(_.flatten(_.map(this.messages, message => message.info)), 'tag') + // summarize total fails + const failure = groups[constants.ProcessingStatus.Failed] || [] + // summarize total skips + const skips = groups[constants.ProcessingStatus.Skipped] || [] + // summarize total jobs with isApplicationPageActive being set to true/false + const groupsByisApplicationPageActive = _.groupBy(groupsByTag.job_is_application_page_active_updated, 'currentValue') + const jobsWithIsApplicationPageActiveSetToTrue = groupsByisApplicationPageActive.true || [] + const jobsWithIsApplicationPageActiveSetToFalse = groupsByisApplicationPageActive.false || [] + // summarize total records with externalId not found in Taas API + const recordsWithExternalIdNotFound = groupsByTag.external_id_not_found || [] + logger.info('=== summary ===') + logger.info(`No. of records read = ${this.messages.length}`) + logger.info(`No. of records updated for field isApplicationPageActive = true = ${jobsWithIsApplicationPageActiveSetToTrue.length}`) + logger.info(`No. of records updated for field isApplicationPageActive = false = ${jobsWithIsApplicationPageActiveSetToFalse.length}`) + logger.info(`No. of records : externalId not found = ${recordsWithExternalIdNotFound.length}`) + logger.info(`No. of records failed(all) = ${failure.length}`) + logger.info(`No. of records failed(excluding "externalId not found") = ${failure.length - recordsWithExternalIdNotFound.length}`) + logger.info(`No. of records skipped = ${skips.length}`) + } +} + +module.exports = Report