Merge pull request #432 from eisbilir/fix/job-import-script

fix: script to import job from RCRM

Author: maxceem

scripts/recruit-crm-job-import/README.md

@@ -16,6 +16,7 @@ By default the script creates jobs and resource bookings via `TC_API`.
 Follow the README for Taas API to deploy Taas API locally and then point the script to the local API by running:
 
 ``` bash
+export RCRM_IMPORT_CONFIG_NAMESAPCE=RCRM_IMPORT_
 export RCRM_IMPORT_TAAS_API_URL=http://localhost:3000/api/v5
 node scripts/recruit-crm-job-import scripts/recruit-crm-job-import/example_data.csv | tee /tmp/report.txt
 ```
@@ -27,38 +28,42 @@ DEBUG: processing line #1 - {"directProjectId":"24568","projectId":"(dynamic loa
 WARN: #1 - externalId is missing
 DEBUG: processed line #1
 DEBUG: processing line #2 - {"directProjectId":"24568","projectId":"(dynamic load)","externalId":"0","title":"taas-demo-job5","startDate":"10/26/2020","endDate":"01/29/2021","numPositions":"2","userHandle":"not_found_handle","jobid":"(dynamic load)","customerRate":"20","memberRate":"10","_lnum":2}
-ERROR: #2 - handle: not_found_handle user not found
+ERROR: #2 - id: 51ce2216-0dee-4dcf-bf7d-79f862e8d63c job created; handle: not_found_handle user not found
 DEBUG: processed line #2
 DEBUG: processing line #3 - {"directProjectId":"24568","projectId":"(dynamic load)","externalId":"0","title":"taas-demo-job5","startDate":"10/26/2020","endDate":"01/29/2021","numPositions":"2","userHandle":"nkumartest","jobid":"(dynamic load)","customerRate":"20","memberRate":"10","_lnum":3}
 DEBUG: userHandle: nkumartest userId: 57646ff9-1cd3-4d3c-88ba-eb09a395366c
-DEBUG: resourceBookingId: dc8b23d4-9987-4a7d-a587-2056283223de status: assigned
-INFO: #3 - id: 7c8ed989-35bf-4899-9c93-708630a7c63b job already exists; id: dc8b23d4-9987-4a7d-a587-2056283223de resource booking created; id: dc8b23d4-9987-4a7d-a587-2056283223de status: assigned resource booking updated
+INFO: #3 - id: 51ce2216-0dee-4dcf-bf7d-79f862e8d63c externalId: 0 job already exists; id: d49d2fbd-ba11-49dc-8eaa-5afafa7e993f resource booking created
 DEBUG: processed line #3
 DEBUG: processing line #4 - {"directProjectId":"24567","projectId":"(dynamic load)","externalId":"1212","title":"Dummy Description","startDate":"10/20/2020","endDate":"01/29/2021","numPositions":"2","userHandle":"pshah_manager","jobid":"(dynamic load)","customerRate":"150","memberRate":"100","_lnum":4}
 DEBUG: userHandle: pshah_manager userId: a55fe1bc-1754-45fa-9adc-cf3d6d7c377a
-DEBUG: resourceBookingId: 708469fb-ead0-4fc3-bef7-1ef4dd041428 status: assigned
-INFO: #4 - id: f61da880-5295-40c2-b6db-21e6cdef93f9 job created; id: 708469fb-ead0-4fc3-bef7-1ef4dd041428 resource booking created; id: 708469fb-ead0-4fc3-bef7-1ef4dd041428 status: assigned resource booking updated
+INFO: #4 - id: e0267551-24fe-48b5-9605-719852901de2 job created; id: f6285f03-056d-446f-a69b-6d275a97d68a resource booking created
 DEBUG: processed line #4
 DEBUG: processing line #5 - {"directProjectId":"24566","projectId":"(dynamic load)","externalId":"23850272","title":"33fromzaps330","startDate":"02/21/2021","endDate":"03/15/2021","numPositions":"7","userHandle":"nkumar2","jobid":"(dynamic load)","customerRate":"50","memberRate":"30","_lnum":5}
 DEBUG: userHandle: nkumar2 userId: 4b00d029-c87b-47b2-bfe2-0ab80d8b5774
-DEBUG: resourceBookingId: 7870c30b-e511-48f2-8687-499ab116174f status: assigned
-INFO: #5 - id: 72dc0399-5e4b-4783-9a27-ea07a4ce99a7 job created; id: 7870c30b-e511-48f2-8687-499ab116174f resource booking created; id: 7870c30b-e511-48f2-8687-499ab116174f status: assigned resource booking updated
+INFO: #5 - id: cd94784c-432d-4c46-b860-04a89e7b1099 job created; id: 98604c13-c6f3-4203-b74f-db376e9f02e4 resource booking created
 DEBUG: processed line #5
 DEBUG: processing line #6 - {"directProjectId":"24565","projectId":"(dynamic load)","externalId":"23843365","title":"Designer","startDate":"02/24/2021","endDate":"03/30/2021","numPositions":"1","userHandle":"GunaK-TopCoder","jobid":"(dynamic load)","customerRate":"70","memberRate":"70","_lnum":6}
 DEBUG: userHandle: GunaK-TopCoder userId: 2bba34d5-20e4-46d6-bfc1-05736b17afbb
-DEBUG: resourceBookingId: b2e705d3-6864-4697-96bb-dc2a288755bc status: assigned
-INFO: #6 - id: 7ff0737e-958c-494e-8a5a-592ac1c5d4ff job created; id: b2e705d3-6864-4697-96bb-dc2a288755bc resource booking created; id: b2e705d3-6864-4697-96bb-dc2a288755bc status: assigned resource booking updated
+INFO: #6 - id: 49883150-59c2-4e5b-b5c3-aaf6d11d0da2 job created; id: 5505b6b5-050c-421c-893f-b862b1a08092 resource booking created
 DEBUG: processed line #6
 DEBUG: processing line #7 - {"directProjectId":"24564","projectId":"(dynamic load)","externalId":"23836459","title":"demo-dev-19janV4","startDate":"01/20/2021","endDate":"01/30/2021","numPositions":"1","userHandle":"nkumar1","jobid":"(dynamic load)","customerRate":"400","memberRate":"200","_lnum":7}
 DEBUG: userHandle: nkumar1 userId: ab19a53b-0607-4a99-8bdd-f3b0cb552293
-DEBUG: resourceBookingId: 04299b4c-3f6e-4b3e-ae57-bf8232408cf9 status: assigned
-INFO: #7 - id: 73301ade-40ff-4103-bd50-37b8d2a98183 job created; id: 04299b4c-3f6e-4b3e-ae57-bf8232408cf9 resource booking created; id: 04299b4c-3f6e-4b3e-ae57-bf8232408cf9 status: assigned resource booking updated
+INFO: #7 - id: b03dc641-d6be-4a15-9c86-ef38f0e20c28 job created; id: 8e332107-453b-4ec5-b934-902c829e73a2 resource booking created
 DEBUG: processed line #7
 INFO: === summary ===
 INFO: total: 7
 INFO: success: 5
 INFO: failure: 1
 INFO: skips: 1
+INFO: jobs created: 5
+INFO: resource bookings created: 5
+INFO: jobs already exist: 1
+INFO: resource bookings already exist: 0
+INFO: validation errors: 0
+INFO: user not found: 1
+INFO: external id missing: 1
+INFO: request error: 0
+INFO: internal error: 0
 INFO: === summary ===
 INFO: done!
 ```

scripts/recruit-crm-job-import/helper.js

@@ -54,15 +54,15 @@ async function getJobByExternalId (externalId) {
  * Update the status of a resource booking.
  *
  * @param {String} resourceBookingId the resource booking id
- * @param {String} status the status for the resource booking
+ * @param {Object} data the data to update
  * @returns {Object} the result
  */
-async function updateResourceBookingStatus (resourceBookingId, status) {
+async function updateResourceBooking (resourceBookingId, data) {
   const token = await getM2MToken()
   const { body: resourceBooking } = await request.patch(`${config.TAAS_API_URL}/resourceBookings/${resourceBookingId}`)
     .set('Authorization', `Bearer ${token}`)
     .set('Content-Type', 'application/json')
-    .send({ status })
+    .send(data)
   return resourceBooking
 }
 
@@ -139,7 +139,7 @@ module.exports = {
   getPathnameFromCommandline: commonHelper.getPathnameFromCommandline,
   createJob,
   getJobByExternalId,
-  updateResourceBookingStatus,
+  updateResourceBooking,
   getResourceBookingByJobIdAndUserId,
   createResourceBooking,
   getUserByHandle,

scripts/recruit-crm-job-import/index.js

@@ -5,7 +5,7 @@
 const Joi = require('joi')
   .extend(require('@joi/date'))
 const _ = require('lodash')
-const dateFNS = require('date-fns')
+const moment = require('moment')
 const Report = require('./report')
 const config = require('./config')
 const helper = require('./helper')
@@ -18,12 +18,12 @@ const jobSchema = Joi.object({
   title: Joi.string().required(),
   startDate: Joi.date().format('MM/DD/YYYY').required(),
   endDate: Joi.date().format('MM/DD/YYYY').required(),
-  numPositions: Joi.number().integer().min(1),
+  numPositions: Joi.number().integer().min(1).required(),
   userHandle: Joi.string(),
   customerRate: Joi.number(),
   memberRate: Joi.number(),
   skills: Joi.array().default([]),
-  rateType: Joi.string().default('weekly')
+  rateType: Joi.string().default('weekly').valid('hourly', 'daily', 'weekly', 'monthly', 'annual')
 }).unknown(true)
 
 /**
@@ -67,33 +67,51 @@ async function processJob (job, info = []) {
     data.jobId = result.id
   } catch (err) {
     if (!(err.message && err.message.includes('job not found'))) {
+      err.info = info
       throw err
     }
-    const result = await helper.createJob(_.pick(data, ['projectId', 'externalId', 'title', 'numPositions', 'skills']))
+    const jobData = _.pick(data, ['projectId', 'externalId', 'title', 'numPositions', 'skills'])
+    if (data.numPositions === 1) {
+      jobData.status = 'assigned'
+    }
+    const result = await helper.createJob(jobData)
     info.push({ text: `id: ${result.id} job created`, tag: 'job_created' })
     data.jobId = result.id
   }
-  data.userId = (await helper.getUserByHandle(data.userHandle)).id
-  logger.debug(`userHandle: ${data.userHandle} userId: ${data.userId}`)
+  try {
+    data.userId = (await helper.getUserByHandle(data.userHandle)).id
+    logger.debug(`userHandle: ${data.userHandle} userId: ${data.userId}`)
+  } catch (err) {
+    if (!(err.message && err.message.includes('user not found'))) {
+      err.info = info
+      throw err
+    }
+    info.push({ text: err.message, tag: 'user_not_found' })
+    return { status: constants.ProcessingStatus.Failed, info }
+  }
   // create a resource booking if it does not already exist
   try {
     const result = await helper.getResourceBookingByJobIdAndUserId(data.jobId, data.userId)
     info.push({ text: `id: ${result.id} resource booking already exists`, tag: 'resource_booking_already_exists' })
     return { status: constants.ProcessingStatus.Successful, info }
   } catch (err) {
     if (!(err.message && err.message.includes('resource booking not found'))) {
+      err.info = info
+      throw err
+    }
+    try {
+      const resourceBookingData = _.pick(data, ['projectId', 'jobId', 'userId', 'memberRate', 'customerRate', 'rateType'])
+      resourceBookingData.startDate = moment(data.startDate).format('YYYY-MM-DD')
+      resourceBookingData.endDate = moment(data.endDate).format('YYYY-MM-DD')
+      resourceBookingData.status = moment(data.endDate).isBefore(moment()) ? 'closed' : 'placed'
+      const result = await helper.createResourceBooking(resourceBookingData)
+      info.push({ text: `id: ${result.id} resource booking created`, tag: 'resource_booking_created' })
+      return { status: constants.ProcessingStatus.Successful, info }
+    } catch (err) {
+      err.info = info
       throw err
     }
-    const result = await helper.createResourceBooking(_.pick(data, ['projectId', 'jobId', 'userId', 'startDate', 'endDate', 'memberRate', 'customerRate', 'rateType']))
-    info.push({ text: `id: ${result.id} resource booking created`, tag: 'resource_booking_created' })
-    data.resourceBookingId = result.id
   }
-  // update the resourceBooking based on startDate and endDate
-  const resourceBookingStatus = dateFNS.isBefore(data.endDate, dateFNS.startOfToday()) ? 'closed' : 'placed'
-  logger.debug(`resourceBookingId: ${data.resourceBookingId} status: ${resourceBookingStatus}`)
-  await helper.updateResourceBookingStatus(data.resourceBookingId, resourceBookingStatus)
-  info.push({ text: `id: ${data.resourceBookingId} status: ${resourceBookingStatus} resource booking updated`, tag: 'resource_booking_status_updated' })
-  return { status: constants.ProcessingStatus.Successful, info }
 }
 
 /**
@@ -111,10 +129,11 @@ async function main () {
       const result = await processJob(job)
       report.add({ lnum: job._lnum, ...result })
     } catch (err) {
+      const info = err.info || []
       if (err.response) {
-        report.add({ lnum: job._lnum, status: constants.ProcessingStatus.Failed, info: [{ text: err.response.error.toString().split('\n')[0], tag: 'request_error' }] })
+        report.add({ lnum: job._lnum, status: constants.ProcessingStatus.Failed, info: [{ text: err.response.error.toString().split('\n')[0], tag: 'request_error' }, ...info] })
       } else {
-        report.add({ lnum: job._lnum, status: constants.ProcessingStatus.Failed, info: [{ text: err.message, tag: 'internal_error' }] })
+        report.add({ lnum: job._lnum, status: constants.ProcessingStatus.Failed, info: [{ text: err.message, tag: 'internal_error' }, ...info] })
       }
     }
     report.print()

scripts/recruit-crm-job-import/report.js

@@ -44,6 +44,11 @@ class Report {
     const resourceBookingsCreated = groupsByTag.resource_booking_created || []
     const jobsAlreadyExist = groupsByTag.job_already_exists || []
     const resourceBookingsAlreadyExist = groupsByTag.resource_booking_already_exists || []
+    const validationErrors = groupsByTag.validation_error || []
+    const userNotFound = groupsByTag.user_not_found || []
+    const externalIdMissing = groupsByTag.external_id_missing || []
+    const requestError = groupsByTag.request_error || []
+    const internalError = groupsByTag.internal_error || []
     logger.info('=== summary ===')
     logger.info(`total: ${this.messages.length}`)
     logger.info(`success: ${success.length}`)
@@ -53,6 +58,11 @@ class Report {
     logger.info(`resource bookings created: ${resourceBookingsCreated.length}`)
     logger.info(`jobs already exist: ${jobsAlreadyExist.length}`)
     logger.info(`resource bookings already exist: ${resourceBookingsAlreadyExist.length}`)
+    logger.info(`validation errors: ${validationErrors.length}`)
+    logger.info(`user not found: ${userNotFound.length}`)
+    logger.info(`external id missing: ${externalIdMissing.length}`)
+    logger.info(`request error: ${requestError.length}`)
+    logger.info(`internal error: ${internalError.length}`)
     logger.info('=== summary ===')
   }
 }

src/eventHandlers/ResourceBookingEventHandler.js

@@ -102,11 +102,11 @@ async function assignJob (payload) {
     return
   }
   const job = await models.Job.findById(resourceBooking.jobId)
-  if (job.status === 'placed') {
+  if (job.status === 'placed' || job.status === 'assigned') {
     logger.debug({
       component: 'ResourceBookingEventHandler',
       context: 'assignJob',
-      message: `job with projectId ${job.projectId} is already placed`
+      message: `job with projectId ${job.projectId} is already ${job.status}`
     })
     return
   }