Merge pull request #120 from imcaizheng/recruit-crm-job-import

Create a script to import Jobs data from Recruit CRM

Author: maxceem

docs/swagger.yaml

@@ -1002,6 +1002,32 @@ paths:
             type: string
             enum: ['hourly', 'daily', 'weekly', 'monthly']
           description: The rate type.
+        - in: query
+          name: jobId
+          required: false
+          schema:
+            type: string
+            format: uuid
+          description: The job id.
+        - in: query
+          name: userId
+          required: false
+          schema:
+            type: string
+            format: uuid
+          description: The job id.
+        - in: query
+          name: projectId
+          required: false
+          schema:
+            type: integer
+          description: The project id.
+        - in: query
+          name: projectIds
+          required: false
+          schema:
+            type: string
+          description: comma separated project ids.
 
       responses:
         '200':

package-lock.json

@@ -50,7 +50,9 @@
     "winston": "^3.3.3"
   },
   "devDependencies": {
+    "@joi/date": "^2.0.1",
     "chai": "^4.2.0",
+    "csv-parser": "^3.0.0",
     "mocha": "^8.1.3",
     "nodemon": "^2.0.4",
     "nyc": "^15.1.0",

package.json

@@ -0,0 +1,76 @@
+Recruit CRM Data Import
+===
+
+# Configuration
+Configuration file is at `./scripts/recruit-crm-job-import/config.js`.
+
+
+# Usage
+``` bash
+node scripts/recruit-crm-job-import <pathname-to-a-csv-file>
+```
+
+By default the script creates jobs and resource bookings via `TC_API`.
+# Example
+
+Follow the README for Taas API to deploy Taas API locally and then point the script to the local API by running:
+
+``` bash
+export RCRM_IMPORT_TAAS_API_URL=http://localhost:3000/api/v5
+node scripts/recruit-crm-job-import scripts/recruit-crm-job-import/example_data.csv | tee /tmp/report.txt
+```
+
+The example output is:
+
+``` bash
+DEBUG: processing line #1 - {"directProjectId":"24568","projectId":"(dynamic load)","externalId":"","title":"taas-demo-job5","startDate":"10/26/2020","endDate":"01/29/2021","numPositions":"2","userHandle":"nkumartest","jobid":"(dynamic load)","customerRate":"20","memberRate":"10","_lnum":1}
+WARN: #1 - externalId is missing
+DEBUG: processed line #1
+DEBUG: processing line #2 - {"directProjectId":"24568","projectId":"(dynamic load)","externalId":"0","title":"taas-demo-job5","startDate":"10/26/2020","endDate":"01/29/2021","numPositions":"2","userHandle":"not_found_handle","jobid":"(dynamic load)","customerRate":"20","memberRate":"10","_lnum":2}
+ERROR: #2 - handle: not_found_handle user not found
+DEBUG: processed line #2
+DEBUG: processing line #3 - {"directProjectId":"24568","projectId":"(dynamic load)","externalId":"0","title":"taas-demo-job5","startDate":"10/26/2020","endDate":"01/29/2021","numPositions":"2","userHandle":"nkumartest","jobid":"(dynamic load)","customerRate":"20","memberRate":"10","_lnum":3}
+DEBUG: userHandle: nkumartest userId: 57646ff9-1cd3-4d3c-88ba-eb09a395366c
+DEBUG: resourceBookingId: dc8b23d4-9987-4a7d-a587-2056283223de status: assigned
+INFO: #3 - id: 7c8ed989-35bf-4899-9c93-708630a7c63b job already exists; id: dc8b23d4-9987-4a7d-a587-2056283223de resource booking created; id: dc8b23d4-9987-4a7d-a587-2056283223de status: assigned resource booking updated
+DEBUG: processed line #3
+DEBUG: processing line #4 - {"directProjectId":"24567","projectId":"(dynamic load)","externalId":"1212","title":"Dummy Description","startDate":"10/20/2020","endDate":"01/29/2021","numPositions":"2","userHandle":"pshah_manager","jobid":"(dynamic load)","customerRate":"150","memberRate":"100","_lnum":4}
+DEBUG: userHandle: pshah_manager userId: a55fe1bc-1754-45fa-9adc-cf3d6d7c377a
+DEBUG: resourceBookingId: 708469fb-ead0-4fc3-bef7-1ef4dd041428 status: assigned
+INFO: #4 - id: f61da880-5295-40c2-b6db-21e6cdef93f9 job created; id: 708469fb-ead0-4fc3-bef7-1ef4dd041428 resource booking created; id: 708469fb-ead0-4fc3-bef7-1ef4dd041428 status: assigned resource booking updated
+DEBUG: processed line #4
+DEBUG: processing line #5 - {"directProjectId":"24566","projectId":"(dynamic load)","externalId":"23850272","title":"33fromzaps330","startDate":"02/21/2021","endDate":"03/15/2021","numPositions":"7","userHandle":"nkumar2","jobid":"(dynamic load)","customerRate":"50","memberRate":"30","_lnum":5}
+DEBUG: userHandle: nkumar2 userId: 4b00d029-c87b-47b2-bfe2-0ab80d8b5774
+DEBUG: resourceBookingId: 7870c30b-e511-48f2-8687-499ab116174f status: assigned
+INFO: #5 - id: 72dc0399-5e4b-4783-9a27-ea07a4ce99a7 job created; id: 7870c30b-e511-48f2-8687-499ab116174f resource booking created; id: 7870c30b-e511-48f2-8687-499ab116174f status: assigned resource booking updated
+DEBUG: processed line #5
+DEBUG: processing line #6 - {"directProjectId":"24565","projectId":"(dynamic load)","externalId":"23843365","title":"Designer","startDate":"02/24/2021","endDate":"03/30/2021","numPositions":"1","userHandle":"GunaK-TopCoder","jobid":"(dynamic load)","customerRate":"70","memberRate":"70","_lnum":6}
+DEBUG: userHandle: GunaK-TopCoder userId: 2bba34d5-20e4-46d6-bfc1-05736b17afbb
+DEBUG: resourceBookingId: b2e705d3-6864-4697-96bb-dc2a288755bc status: assigned
+INFO: #6 - id: 7ff0737e-958c-494e-8a5a-592ac1c5d4ff job created; id: b2e705d3-6864-4697-96bb-dc2a288755bc resource booking created; id: b2e705d3-6864-4697-96bb-dc2a288755bc status: assigned resource booking updated
+DEBUG: processed line #6
+DEBUG: processing line #7 - {"directProjectId":"24564","projectId":"(dynamic load)","externalId":"23836459","title":"demo-dev-19janV4","startDate":"01/20/2021","endDate":"01/30/2021","numPositions":"1","userHandle":"nkumar1","jobid":"(dynamic load)","customerRate":"400","memberRate":"200","_lnum":7}
+DEBUG: userHandle: nkumar1 userId: ab19a53b-0607-4a99-8bdd-f3b0cb552293
+DEBUG: resourceBookingId: 04299b4c-3f6e-4b3e-ae57-bf8232408cf9 status: assigned
+INFO: #7 - id: 73301ade-40ff-4103-bd50-37b8d2a98183 job created; id: 04299b4c-3f6e-4b3e-ae57-bf8232408cf9 resource booking created; id: 04299b4c-3f6e-4b3e-ae57-bf8232408cf9 status: assigned resource booking updated
+DEBUG: processed line #7
+INFO: === summary ===
+INFO: total: 7
+INFO: success: 5
+INFO: failure: 1
+INFO: skips: 1
+INFO: === summary ===
+INFO: done!
+```
+
+To list all skipped lines:
+
+``` bash
+cat /tmp/report.txt | grep 'WARN'
+```
+
+To find out whether there are some users not found by user handles, run the following command:
+
+``` bash
+cat /tmp/report.txt | grep 'ERROR' | grep 'user not found'
+```

scripts/recruit-crm-job-import/README.md

@@ -0,0 +1,21 @@
+/*
+ * Configuration for the RCRM import script.
+ * Namespace is created to allow to configure the env variables for this script independently.
+ */
+
+const config = require('config')
+
+const namespace = process.env.RCRM_IMPORT_CONFIG_NAMESAPCE || 'RCRM_IMPORT_'
+
+module.exports = {
+  SLEEP_TIME: process.env[`${namespace}SLEEP_TIME`] || 500,
+  TAAS_API_URL: process.env[`${namespace}TAAS_API_URL`] || config.TC_API,
+
+  TC_API: process.env[`${namespace}TC_API`] || config.TC_API,
+  AUTH0_URL: process.env[`${namespace}AUTH0_URL`] || config.AUTH0_URL,
+  AUTH0_AUDIENCE: process.env[`${namespace}AUTH0_AUDIENCE`] || config.AUTH0_AUDIENCE,
+  TOKEN_CACHE_TIME: process.env[`${namespace}TOKEN_CACHE_TIME`] || config.TOKEN_CACHE_TIME,
+  AUTH0_CLIENT_ID: process.env[`${namespace}AUTH0_CLIENT_ID`] || config.AUTH0_CLIENT_ID,
+  AUTH0_CLIENT_SECRET: process.env[`${namespace}AUTH0_CLIENT_SECRET`] || config.AUTH0_CLIENT_SECRET,
+  AUTH0_PROXY_SERVER_URL: process.env[`${namespace}AUTH0_PROXY_SERVER_URL`] || config.AUTH0_PROXY_SERVER_URL
+}

scripts/recruit-crm-job-import/config.js

@@ -0,0 +1,22 @@
+/*
+ * Constants for the RCRM import script.
+ */
+
+module.exports = {
+  ProcessingStatus: {
+    Successful: 'successful',
+    Failed: 'failed',
+    Skipped: 'skipped'
+  },
+  fieldNameMap: {
+    DirectprojectId: 'directProjectId',
+    externalId: 'externalId',
+    title: 'title',
+    startDate: 'startDate',
+    endDate: 'endDate',
+    numPositions: 'numPositions',
+    userHandle: 'userHandle',
+    customerRate: 'customerRate',
+    memberRate: 'memberRate'
+  }
+}

scripts/recruit-crm-job-import/constants.js

@@ -0,0 +1,8 @@
+DirectprojectId,projectId,externalId,title,startDate,endDate,numPositions,userHandle,jobid,customerRate,memberRate
+24568,(dynamic load),,taas-demo-job5,10/26/2020,01/29/2021,2,nkumartest,(dynamic load),20,10
+24568,(dynamic load),0,taas-demo-job5,10/26/2020,01/29/2021,2,not_found_handle,(dynamic load),20,10
+24568,(dynamic load),0,taas-demo-job5,10/26/2020,01/29/2021,2,nkumartest,(dynamic load),20,10
+24567,(dynamic load),1212,Dummy Description,10/20/2020,01/29/2021,2,pshah_manager,(dynamic load),150,100
+24566,(dynamic load),23850272,33fromzaps330,02/21/2021,03/15/2021,7,nkumar2,(dynamic load),50,30
+24565,(dynamic load),23843365,Designer,02/24/2021,03/30/2021,1,GunaK-TopCoder,(dynamic load),70,70
+24564,(dynamic load),23836459,demo-dev-19janV4,01/20/2021,01/30/2021,1,nkumar1,(dynamic load),400,200

scripts/recruit-crm-job-import/example_data.csv

@@ -0,0 +1,142 @@
+/*
+ * Provide some commonly used functions for the RCRM import script.
+ */
+const config = require('./config')
+const request = require('superagent')
+const { getM2MToken } = require('../../src/common/helper')
+
+/**
+ * Sleep for a given number of milliseconds.
+ *
+ * @param {Number} milliseconds the sleep time
+ * @returns {undefined}
+ */
+async function sleep (milliseconds) {
+  return new Promise((resolve) => setTimeout(resolve, milliseconds))
+}
+
+/**
+ * Create a new job via taas api.
+ *
+ * @param {Object} data the job data
+ * @returns {Object} the result
+ */
+async function createJob (data) {
+  const token = await getM2MToken()
+  const { body: job } = await request.post(`${config.TAAS_API_URL}/jobs`)
+    .set('Authorization', `Bearer ${token}`)
+    .set('Content-Type', 'application/json')
+    .send(data)
+  return job
+}
+
+/**
+ * Find taas job by external id.
+ *
+ * @param {String} externalId the external id
+ * @returns {Object} the result
+ */
+async function getJobByExternalId (externalId) {
+  const token = await getM2MToken()
+  const { body: jobs } = await request.get(`${config.TAAS_API_URL}/jobs`)
+    .query({ externalId })
+    .set('Authorization', `Bearer ${token}`)
+  if (!jobs.length) {
+    throw new Error(`externalId: ${externalId} job not found`)
+  }
+  return jobs[0]
+}
+
+/**
+ * Update the status of a resource booking.
+ *
+ * @param {String} resourceBookingId the resource booking id
+ * @param {String} status the status for the resource booking
+ * @returns {Object} the result
+ */
+async function updateResourceBookingStatus (resourceBookingId, status) {
+  const token = await getM2MToken()
+  const { body: resourceBooking } = await request.patch(`${config.TAAS_API_URL}/resourceBookings/${resourceBookingId}`)
+    .set('Authorization', `Bearer ${token}`)
+    .set('Content-Type', 'application/json')
+    .send({ status })
+  return resourceBooking
+}
+
+/**
+ * Find taas resource booking by job id and user id.
+ *
+ * @param {String} jobId the job id
+ * @param {String} userId the user id
+ * @returns {Object} the result
+ */
+async function getResourceBookingByJobIdAndUserId (jobId, userId) {
+  const token = await getM2MToken()
+  const { body: resourceBookings } = await request.get(`${config.TAAS_API_URL}/resourceBookings`)
+    .query({ jobId, userId })
+    .set('Authorization', `Bearer ${token}`)
+  if (!resourceBookings.length) {
+    throw new Error(`jobId: ${jobId} userId: ${userId} resource booking not found`)
+  }
+  return resourceBookings[0]
+}
+
+/**
+ * Create a new resource booking via taas api.
+ *
+ * @param {Object} data the resource booking data
+ * @returns {Object} the result
+ */
+async function createResourceBooking (data) {
+  const token = await getM2MToken()
+  const { body: resourceBooking } = await request.post(`${config.TAAS_API_URL}/resourceBookings`)
+    .set('Authorization', `Bearer ${token}`)
+    .set('Content-Type', 'application/json')
+    .send(data)
+  return resourceBooking
+}
+
+/**
+ * Find user via /v5/users by user handle.
+ *
+ * @param {String} handle the user handle
+ * @returns {Object} the result
+ */
+async function getUserByHandle (handle) {
+  const token = await getM2MToken()
+  const { body: users } = await request.get(`${config.TC_API}/users`)
+    .query({ handle })
+    .set('Authorization', `Bearer ${token}`)
+  if (!users.length) {
+    throw new Error(`handle: ${handle} user not found`)
+  }
+  return users[0]
+}
+
+/**
+ * Find project via /v5/projects by Direct project id.
+ *
+ * @param {Number} directProjectId the Direct project id
+ * @returns {Object} the result
+ */
+async function getProjectByDirectProjectId (directProjectId) {
+  const token = await getM2MToken()
+  const { body: projects } = await request.get(`${config.TC_API}/projects`)
+    .query({ directProjectId })
+    .set('Authorization', `Bearer ${token}`)
+  if (!projects.length) {
+    throw new Error(`directProjectId: ${directProjectId} project not found`)
+  }
+  return projects[0]
+}
+
+module.exports = {
+  sleep,
+  createJob,
+  getJobByExternalId,
+  updateResourceBookingStatus,
+  getResourceBookingByJobIdAndUserId,
+  createResourceBooking,
+  getUserByHandle,
+  getProjectByDirectProjectId
+}