From c1b0578cfe1543ac06c1f64381ba50555d0c3b8c Mon Sep 17 00:00:00 2001 From: yoution Date: Wed, 28 Jul 2021 13:58:14 +0800 Subject: [PATCH 1/3] Send Weekly Surveys --- app.js | 4 +- config/default.js | 10 + docs/swagger.yaml | 44 ++++ ...21-07-26-add-send-weekly-survery-fields.js | 43 ++++ package.json | 1 + .../data/updateWorkPeriodSentSurveyField.js | 68 +++++ src/common/helper.js | 10 + src/common/surveyMonkey.js | 237 ++++++++++++++++++ src/models/ResourceBooking.js | 6 + src/models/WorkPeriod.js | 20 ++ src/services/ResourceBookingService.js | 28 ++- src/services/SurveyService.js | 134 ++++++++++ src/services/WorkPeriodService.js | 17 +- 13 files changed, 613 insertions(+), 9 deletions(-) create mode 100644 migrations/2021-07-26-add-send-weekly-survery-fields.js create mode 100644 scripts/data/updateWorkPeriodSentSurveyField.js create mode 100644 src/common/surveyMonkey.js create mode 100644 src/services/SurveyService.js diff --git a/app.js b/app.js index e6d79c69..08395ffc 100644 --- a/app.js +++ b/app.js @@ -14,6 +14,7 @@ const logger = require('./src/common/logger') const eventHandlers = require('./src/eventHandlers') const interviewService = require('./src/services/InterviewService') const { processScheduler } = require('./src/services/PaymentSchedulerService') +const { sendSurveys } = require('./src/services/SurveyService') // setup express app const app = express() @@ -98,7 +99,8 @@ const server = app.listen(app.get('port'), () => { eventHandlers.init() // schedule updateCompletedInterviews to run every hour schedule.scheduleJob('0 0 * * * *', interviewService.updateCompletedInterviews) - + // schedule sendSurveys + schedule.scheduleJob(config.WEEKLY_SURVEY.CRON, sendSurveys) // schedule payment processing schedule.scheduleJob(config.PAYMENT_PROCESSING.CRON, processScheduler) }) diff --git a/config/default.js b/config/default.js index cb589290..4675ff09 100644 --- a/config/default.js +++ b/config/default.js @@ -180,6 +180,16 @@ module.exports = { INTERNAL_MEMBER_GROUPS: process.env.INTERNAL_MEMBER_GROUPS || ['20000000', '20000001', '20000003', '20000010', '20000015'], // Topcoder skills cache time in minutes TOPCODER_SKILLS_CACHE_TIME: process.env.TOPCODER_SKILLS_CACHE_TIME || 60, + // weekly survey scheduler config + WEEKLY_SURVEY: { + CRON: process.env.WEEKLY_SURVEY_CRON || '0 1 * * 7', + BASE_URL: process.env.WEEKLY_SURVEY_BASE_URL || 'https://api.surveymonkey.net/v3/surveys', + JWT_TOKEN: process.env.WEEKLY_SURVEY_JWT_TOKEN || '', + SURVEY_ID: process.env.WEEKLY_SURVEY_SURVEY_ID || '', + SURVEY_MASTER_COLLECTOR_ID: process.env.WEEKLY_SURVEY_SURVEY_MASTER_COLLECTOR_ID || '', + SURVEY_MASTER_MESSAGE_ID: process.env.WEEKLY_SURVEY_SURVEY_MASTER_MESSAGE_ID || '', + SURVEY_CONTACT_GROUP_ID: process.env.WEEKLY_SURVEY_SURVEY_CONTACT_GROUP_ID || '' + }, // payment scheduler config PAYMENT_PROCESSING: { // switch off actual API calls in Payment Scheduler diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 6eb3ef5d..f0f5a0e5 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -4470,6 +4470,10 @@ components: format: float example: 13 description: "The member rate." + sendWeeklySurvey: + type: boolean + example: true, + description: "whether we should send weekly survey to this ResourceBooking or no" customerRate: type: integer format: float @@ -4527,6 +4531,10 @@ components: format: uuid example: "a55fe1bc-1754-45fa-9adc-cf3d6d7c377a" description: "The external id." + sendWeeklySurvey: + type: boolean + example: true, + description: "whether we should send weekly survey to this ResourceBooking or no" jobId: type: string format: uuid @@ -4584,6 +4592,10 @@ components: format: float example: 13.23 description: "The member rate." + sendWeeklySurvey: + type: boolean + example: true, + description: "whether we should send weekly survey to this ResourceBooking or no" customerRate: type: number format: float @@ -4620,6 +4632,22 @@ components: type: string format: uuid description: "The resource booking id." + sentSurvey: + type: boolean + example: true + description: "whether we've already sent a survey for this WorkPeriod of no" + sentSurveyError: + description: "error details if error happened during sending survey" + type: object + properties: + errorMessage: + type: string + example: "error message" + description: "The error message" + errorCode: + type: integer + example: 429 + description: "HTTP code of error" userHandle: type: string example: "eisbilir" @@ -4695,6 +4723,22 @@ components: type: integer example: 2 description: "The count of the days worked for that work period." + sentSurvey: + type: boolean + example: true + description: "whether we've already sent a survey for this WorkPeriod of no" + sentSurveyError: + description: "error details if error happened during sending survey" + type: object + properties: + errorMessage: + type: string + example: "error message" + description: "The error message" + errorCode: + type: integer + example: 429 + description: "HTTP code of error" WorkPeriodPayment: required: - id diff --git a/migrations/2021-07-26-add-send-weekly-survery-fields.js b/migrations/2021-07-26-add-send-weekly-survery-fields.js new file mode 100644 index 00000000..025d79d9 --- /dev/null +++ b/migrations/2021-07-26-add-send-weekly-survery-fields.js @@ -0,0 +1,43 @@ +const config = require('config') + +module.exports = { + up: async (queryInterface, Sequelize) => { + const transaction = await queryInterface.sequelize.transaction() + try { + await queryInterface.addColumn({ tableName: 'resource_bookings', schema: config.DB_SCHEMA_NAME }, 'send_weekly_survey', + { type: Sequelize.BOOLEAN, allowNull: false, defaultValue: true }, + { transaction }) + await queryInterface.addColumn({ tableName: 'work_periods', schema: config.DB_SCHEMA_NAME }, 'sent_survey', + { type: Sequelize.BOOLEAN, allowNull: false, defaultValue: false }, + { transaction }) + await queryInterface.addColumn({ tableName: 'work_periods', schema: config.DB_SCHEMA_NAME }, 'sent_survey_error', + { + type: Sequelize.JSONB({ + errorCode: { + field: 'error_code', + type: Sequelize.INTEGER, + }, + errorMessage: { + field: 'error_message', + type: Sequelize.STRING(255) + }, + }), allowNull: true }, { transaction }) + await transaction.commit() + } catch (err) { + await transaction.rollback() + throw err + } + }, + down: async (queryInterface, Sequelize) => { + const transaction = await queryInterface.sequelize.transaction() + try { + await queryInterface.removeColumn({ tableName: 'resource_bookings', schema: config.DB_SCHEMA_NAME }, 'send_weekly_survey', { transaction }) + await queryInterface.removeColumn({ tableName: 'work_periods', schema: config.DB_SCHEMA_NAME }, 'send_survey', { transaction }) + await queryInterface.removeColumn({ tableName: 'work_periods', schema: config.DB_SCHEMA_NAME }, 'sent_survey_error', { transaction } ) + await transaction.commit() + } catch (err) { + await transaction.rollback() + throw err + } + }, +} diff --git a/package.json b/package.json index 17c1887b..70b1913c 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "index:roles": "node scripts/es/reIndexRoles.js", "data:export": "node scripts/data/exportData.js", "data:import": "node scripts/data/importData.js", + "data:workperiod": "node scripts/data/updateWorkPeriodSentSurveyField.js", "migrate": "npx sequelize db:migrate", "migrate:undo": "npx sequelize db:migrate:undo", "test": "mocha test/unit/*.test.js --timeout 30000 --require test/prepare.js --exit", diff --git a/scripts/data/updateWorkPeriodSentSurveyField.js b/scripts/data/updateWorkPeriodSentSurveyField.js new file mode 100644 index 00000000..e641b177 --- /dev/null +++ b/scripts/data/updateWorkPeriodSentSurveyField.js @@ -0,0 +1,68 @@ +/* + * update WorkPeriod field `sentSurvey=true` + */ +const _ = require('lodash') +const moment = require('moment') +const logger = require('../../src/common/logger') +const { Op } = require('sequelize') +const models = require('../../src/models') + +const ResourceBooking = models.ResourceBooking +const WorkPeriod = models.WorkPeriod + +async function updateWorkPeriod () { + const transaction = await models.sequelize.transaction() + try { + // Start a transaction + const queryCriteria = { + attributes: ['sendWeeklySurvey', 'id'], + include: [{ + as: 'workPeriods', + model: WorkPeriod, + required: true, + where: { + [Op.and]: [ + { sentSurveyError: null }, + { sentSurvey: false }, + { paymentStatus: ['completed'] }, + { endDate: { [Op.lte]: moment().subtract(7, 'days').format('YYYY-MM-DD') } } + ] + } + }], + where: { + [Op.and]: [{ sendWeeklySurvey: true }] + }, + transaction + } + + const resourceBookings = await ResourceBooking.findAll(queryCriteria) + + _.forEach(resourceBookings, r => { + _.forEach(r.workPeriods, async w => { + // await w.update({sentSurvey: true}, {transaction: transaction} ) + await w.update({ sentSurvey: true }) + }) + }) + + // commit transaction only if all things went ok + logger.info({ + component: 'importData', + message: 'committing transaction to database...' + }) + await transaction.commit() + } catch (error) { + // logger.error({ + // component: 'importData', + // message: `Error while writing data of model: WorkPeriod` + // }) + // rollback all insert operations + if (transaction) { + logger.info({ + component: 'importData', + message: 'rollback database transaction...' + }) + transaction.rollback() + } + } +} +updateWorkPeriod() diff --git a/src/common/helper.js b/src/common/helper.js index 7f9625be..319eb86a 100644 --- a/src/common/helper.js +++ b/src/common/helper.js @@ -176,6 +176,7 @@ esIndexPropertyMapping[config.get('esConfig.ES_INDEX_RESOURCE_BOOKING')] = { endDate: { type: 'date', format: 'yyyy-MM-dd' }, memberRate: { type: 'float' }, customerRate: { type: 'float' }, + sendWeeklySurvey: { type: 'boolean' }, rateType: { type: 'keyword' }, billingAccountId: { type: 'integer', null_value: 0 }, workPeriods: { @@ -189,6 +190,14 @@ esIndexPropertyMapping[config.get('esConfig.ES_INDEX_RESOURCE_BOOKING')] = { }, projectId: { type: 'integer' }, userId: { type: 'keyword' }, + sentSurvey: { type: 'boolean' }, + sentSurveyError: { + type: 'nested', + properties: { + errorCode: { type: 'integer' }, + errorMessage: { type: 'keyword' } + } + }, startDate: { type: 'date', format: 'yyyy-MM-dd' }, endDate: { type: 'date', format: 'yyyy-MM-dd' }, daysWorked: { type: 'integer' }, @@ -2012,6 +2021,7 @@ async function getMembersSuggest (fragment) { } module.exports = { + encodeQueryString, getParamFromCliArgs, promptUser, sleep, diff --git a/src/common/surveyMonkey.js b/src/common/surveyMonkey.js new file mode 100644 index 00000000..c3a73499 --- /dev/null +++ b/src/common/surveyMonkey.js @@ -0,0 +1,237 @@ +/* + * surveymonkey api + * + */ + +const logger = require('./logger') +const config = require('config') +const _ = require('lodash') +const request = require('superagent') +const moment = require('moment') +const { encodeQueryString } = require('./helper') +/** + * This code uses several environment variables + * + * WEEKLY_SURVEY_SURVEY_CONTACT_GROUP_ID - the ID of contacts list which would be used to store all the contacts, + * see https://developer.surveymonkey.com/api/v3/#contact_lists-id + * WEEKLY_SURVEY_SURVEY_MASTER_COLLECTOR_ID - the ID of master collector - this collector should be created manually, + * and all other collectors would be created by copying this master collector. + * This is needed so we can make some config inside master collector which would + * be applied to all collectors. + * WEEKLY_SURVEY_SURVEY_MASTER_MESSAGE_ID - the ID of master message - similar to collector, this message would be created manually + * and then script would create copies of this message to use the same config. + */ + +const localLogger = { + debug: (message, context) => logger.debug({ component: 'SurveyMonkeyAPI', context, message }), + error: (message, context) => logger.error({ component: 'SurveyMonkeyAPI', context, message }), + info: (message, context) => logger.info({ component: 'SurveyMonkeyAPI', context, message }) +} + +function getRemainingRequestCountMessge (response) { + return `today has sent ${response.header['x-ratelimit-app-global-day-limit'] - response.header['x-ratelimit-app-global-day-remaining']} requests` +} + +function getErrorMessage (e) { + return { + errorCode: _.get(e, 'response.body.error.http_status_code', 400), + errorMessage: _.get(e, 'response.body.error.message', 'error message') + } +} + +function getSingleItem (lst, errorMessage) { + if (lst.length === 0) { + return null + } + + if (lst.length > 1) { + throw new Error(errorMessage) + } + + return lst[0].id +} + +/* + * get collector name + * + * format `Week Ending yyyy-nth(weeks)` + */ +function getCollectorName (dt) { + return 'Week Ending ' + moment(dt).year() + '-' + moment(dt).format('ww') +} + +/* + * search collector by name + */ +async function searchCollector (collectorName) { + const url = `${config.WEEKLY_SURVEY.BASE_URL}/surveys/${config.WEEKLY_SURVEY.SURVEY_ID}/collectors?${encodeQueryString({ name: collectorName })}` + try { + const response = await request + .get(url) + .set('Authorization', `Bearer ${config.WEEKLY_SURVEY.JWT_TOKEN}`) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json') + + localLogger.info(`URL ${url}, ${getRemainingRequestCountMessge(response)}`, 'searchCollector') + + return getSingleItem(response.body.data, 'More than 1 collector found by name ' + collectorName) + } catch (e) { + localLogger.error(`URL ${url} ${getErrorMessage(e)}, ${getRemainingRequestCountMessge(e.response)}`, 'searchCollector') + throw getErrorMessage(e) + } +} + +/* + * create a named collector if not created + * else return the collectId of the named collector + */ +async function createCollector (collectorName) { + let collectorID = await searchCollector(collectorName) + if (collectorID) { + return collectorID + } + + collectorID = await cloneCollector() + await renameCollector(collectorID, collectorName) + + return collectorID +} + +/* + * clone collector from MASTER_COLLECTOR + */ +async function cloneCollector () { + const body = { from_collector_id: `${config.WEEKLY_SURVEY.SURVEY_MASTER_COLLECTOR_ID}` } + const url = `${config.WEEKLY_SURVEY.BASE_URL}/surveys/${config.WEEKLY_SURVEY.SURVEY_ID}/collectors` + try { + const response = await request + .post(url) + .set('Authorization', `Bearer ${config.WEEKLY_SURVEY.JWT_TOKEN}`) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json') + .send(body) + localLogger.info(`URL ${url}, ${getRemainingRequestCountMessge(response)}`, 'cloneCollector') + return response.body.id + } catch (e) { + localLogger.error(`URL ${url} ${JSON.stringify(getErrorMessage(e))}, ${getRemainingRequestCountMessge(e.response)}`, 'cloneCollector') + throw getErrorMessage(e) + } +} + +/* + * rename collector + */ +async function renameCollector (collectorId, name) { + const body = { name: name } + // http.patch(BASE_URL + '/collectors/' + collectorId, body); + const url = `${config.WEEKLY_SURVEY.BASE_URL}/collectors/${collectorId}` + try { + const response = await request + .patch(url) + .set('Authorization', `Bearer ${config.WEEKLY_SURVEY.JWT_TOKEN}`) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json') + .send(body) + localLogger.info(`URL ${url}, ${getRemainingRequestCountMessge(response)}`, 'renameCollector') + } catch (e) { + localLogger.error(`URL ${url} ${JSON.stringify(getErrorMessage(e))}, ${getRemainingRequestCountMessge(e.response)}`, 'renameCollector') + throw getErrorMessage(e) + } +} + +/* + * create message + */ +async function createMessage (collectorId) { + const body = { + from_collector_id: `${config.WEEKLY_SURVEY.SURVEY_MASTER_COLLECTOR_ID}`, + from_message_id: `${config.WEEKLY_SURVEY.SURVEY_MASTER_MESSAGE_ID}` + } + // response = http.post(BASE_URL + '/collectors/' + collectorId + '/messages', body); + const url = `${config.WEEKLY_SURVEY.BASE_URL}/collectors/${collectorId}/messages` + try { + const response = await request + .post(url) + .set('Authorization', `Bearer ${config.WEEKLY_SURVEY.JWT_TOKEN}`) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json') + .send(body) + localLogger.info(`URL ${url}, ${getRemainingRequestCountMessge(response)}`, 'createMessage') + return response.body.id + } catch (e) { + localLogger.error(`URL ${url} ${JSON.stringify(getErrorMessage(e))}, ${getRemainingRequestCountMessge(e.response)}`, 'createMessage') + throw getErrorMessage(e) + } +} + +/** + * Add Contact Email to List for sending a survey + */ +async function upsertContactInSurveyMonkey (list) { + list = _.filter(list, p => p.email) + if (!list.length) { + return [] + } + const body = { + contacts: list + } + const url = `${config.WEEKLY_SURVEY.BASE_URL}/contact_lists/${config.WEEKLY_SURVEY.SURVEY_CONTACT_GROUP_ID}/contacts/bulk` + try { + const response = await request + .post(url) + .set('Authorization', `Bearer ${config.WEEKLY_SURVEY.JWT_TOKEN}`) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json') + .send(body) + + localLogger.info(`URL ${url}, ${getRemainingRequestCountMessge(response)}`, 'upsertContactInSurveyMonkey') + return _.concat(response.body.existing, response.body.succeeded) + } catch (e) { + localLogger.error(`URL ${url} ${JSON.stringify(getErrorMessage(e))}, ${getRemainingRequestCountMessge(e.response)}`, 'createMessage') + throw getErrorMessage(e) + } +} + +async function addContactsToSurvey (collectorId, messageId, contactIds) { + const url = `${config.WEEKLY_SURVEY.BASE_URL}/collectors/${collectorId}/messages/${messageId}/recipients/bulk` + const body = { contact_ids: _.map(contactIds, 'id') } + try { + const response = await request + .post(url) + .set('Authorization', `Bearer ${config.WEEKLY_SURVEY.JWT_TOKEN}`) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json') + .send(body) + localLogger.info(`URL ${url}, ${getRemainingRequestCountMessge(response)}`, 'addContactsToSurvey') + return response.body.id + } catch (e) { + localLogger.error(`URL ${url} ${JSON.stringify(getErrorMessage(e))}, ${getRemainingRequestCountMessge(e.response)}`, 'addContactsToSurvey') + throw getErrorMessage(e) + } +} + +async function sendSurveyAPI (collectorId, messageId) { + const url = `${config.WEEKLY_SURVEY.BASE_URL}/collectors/${collectorId}/messages/${messageId}/send` + try { + const response = await request + .post(url) + .set('Authorization', `Bearer ${config.WEEKLY_SURVEY.JWT_TOKEN}`) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json') + .send({}) + localLogger.info(`URL ${url}, ${getRemainingRequestCountMessge(response)}`, 'sendSurveyAPI') + return response.body.id + } catch (e) { + localLogger.error(`URL ${url} ${JSON.stringify(getErrorMessage(e))}, ${getRemainingRequestCountMessge(e.response)}`, 'sendSurveyAPI') + throw getErrorMessage(e) + } +} + +module.exports = { + getCollectorName, + createCollector, + createMessage, + upsertContactInSurveyMonkey, + addContactsToSurvey, + sendSurveyAPI +} diff --git a/src/models/ResourceBooking.js b/src/models/ResourceBooking.js index 580e6e96..21a222f1 100644 --- a/src/models/ResourceBooking.js +++ b/src/models/ResourceBooking.js @@ -122,6 +122,12 @@ module.exports = (sequelize) => { type: Sequelize.STRING(255), allowNull: false }, + sendWeeklySurvey: { + field: 'send_weekly_survey', + type: Sequelize.BOOLEAN, + defaultValue: true, + allowNull: false + }, billingAccountId: { field: 'billing_account_id', type: Sequelize.BIGINT diff --git a/src/models/WorkPeriod.js b/src/models/WorkPeriod.js index 720e4870..0204d9be 100644 --- a/src/models/WorkPeriod.js +++ b/src/models/WorkPeriod.js @@ -56,6 +56,26 @@ module.exports = (sequelize) => { type: Sequelize.UUID, allowNull: false }, + sentSurvey: { + field: 'send_survey', + type: Sequelize.BOOLEAN, + defaultValue: false, + allowNull: false + }, + sentSurveyError: { + field: 'sent_survey_error', + allowNull: true, + type: Sequelize.JSONB({ + errorCode: { + field: 'error_code', + type: Sequelize.INTEGER + }, + errorMessage: { + field: 'error_message', + type: Sequelize.STRING(255) + } + }) + }, userHandle: { field: 'user_handle', type: Sequelize.STRING(50), diff --git a/src/services/ResourceBookingService.js b/src/services/ResourceBookingService.js index 46d2fe62..e141f0c5 100644 --- a/src/services/ResourceBookingService.js +++ b/src/services/ResourceBookingService.js @@ -354,6 +354,7 @@ createResourceBooking.schema = Joi.object().keys({ projectId: Joi.number().integer().required(), userId: Joi.string().uuid().required(), jobId: Joi.string().uuid().allow(null), + sendWeeklySurvey: Joi.boolean().default(true), startDate: Joi.date().format('YYYY-MM-DD').allow(null), endDate: Joi.date().format('YYYY-MM-DD').when('startDate', { is: Joi.exist(), @@ -427,6 +428,7 @@ partiallyUpdateResourceBooking.schema = Joi.object().keys({ memberRate: Joi.number().allow(null), customerRate: Joi.number().allow(null), rateType: Joi.rateType(), + sendWeeklySurvey: Joi.boolean().allow(null), billingAccountId: Joi.number().allow(null) }).required() }).required() @@ -466,6 +468,7 @@ fullyUpdateResourceBooking.schema = Joi.object().keys({ customerRate: Joi.number().allow(null).default(null), rateType: Joi.rateType().required(), status: Joi.resourceBookingStatus().required(), + sendWeeklySurvey: Joi.boolean().allow(null), billingAccountId: Joi.number().allow(null).default(null) }).required() }).required() @@ -546,6 +549,10 @@ async function searchResourceBookings (currentUser, criteria, options) { if (!criteria.sortOrder) { criteria.sortOrder = 'desc' } + + if (_.has(criteria, 'workPeriods.sentSurveyError') && !criteria['workPeriods.sentSurveyError']) { + criteria['workPeriods.sentSurveyError'] = null + } // this option to return data from DB is only for internal usage, and it cannot be passed from the endpoint if (!options.returnFromDB) { try { @@ -590,7 +597,7 @@ async function searchResourceBookings (currentUser, criteria, options) { } esQuery.body.sort.push(sort) // Apply ResourceBooking filters - _.each(_.pick(criteria, ['status', 'startDate', 'endDate', 'rateType', 'projectId', 'jobId', 'userId', 'billingAccountId']), (value, key) => { + _.each(_.pick(criteria, ['sendWeeklySurvey', 'status', 'startDate', 'endDate', 'rateType', 'projectId', 'jobId', 'userId', 'billingAccountId']), (value, key) => { esQuery.body.query.bool.must.push({ term: { [key]: { @@ -626,7 +633,7 @@ async function searchResourceBookings (currentUser, criteria, options) { }) } // Apply WorkPeriod and WorkPeriodPayment filters - const workPeriodFilters = _.pick(criteria, ['workPeriods.paymentStatus', 'workPeriods.startDate', 'workPeriods.endDate', 'workPeriods.userHandle']) + const workPeriodFilters = _.pick(criteria, ['workPeriods.sentSurveyError', 'workPeriods.sentSurvey', 'workPeriods.paymentStatus', 'workPeriods.startDate', 'workPeriods.endDate', 'workPeriods.userHandle']) const workPeriodPaymentFilters = _.pick(criteria, ['workPeriods.payments.status', 'workPeriods.payments.days']) if (!_.isEmpty(workPeriodFilters) || !_.isEmpty(workPeriodPaymentFilters)) { const workPeriodsMust = [] @@ -637,7 +644,7 @@ async function searchResourceBookings (currentUser, criteria, options) { [key]: value } }) - } else { + } else if (key !== 'workPeriods.sentSurveyError') { workPeriodsMust.push({ term: { [key]: { @@ -666,6 +673,7 @@ async function searchResourceBookings (currentUser, criteria, options) { } }) } + esQuery.body.query.bool.must.push({ nested: { path: 'workPeriods', @@ -688,7 +696,9 @@ async function searchResourceBookings (currentUser, criteria, options) { r.workPeriods = _.filter(r.workPeriods, wp => { return _.every(_.omit(workPeriodFilters, 'workPeriods.userHandle'), (value, key) => { key = key.split('.')[1] - if (key === 'paymentStatus') { + if (key === 'sentSurveyError' && !workPeriodFilters['workPeriods.sentSurveyError']) { + return !wp[key] + } else if (key === 'paymentStatus') { return _.includes(value, wp[key]) } else { return wp[key] === value @@ -723,7 +733,7 @@ async function searchResourceBookings (currentUser, criteria, options) { logger.info({ component: 'ResourceBookingService', context: 'searchResourceBookings', message: 'fallback to DB query' }) const filter = { [Op.and]: [] } // Apply ResourceBooking filters - _.each(_.pick(criteria, ['status', 'startDate', 'endDate', 'rateType', 'projectId', 'jobId', 'userId']), (value, key) => { + _.each(_.pick(criteria, ['sendWeeklySurvey', 'status', 'startDate', 'endDate', 'rateType', 'projectId', 'jobId', 'userId']), (value, key) => { filter[Op.and].push({ [key]: value }) }) if (!_.isUndefined(criteria.billingAccountId)) { @@ -773,7 +783,7 @@ async function searchResourceBookings (currentUser, criteria, options) { queryCriteria.include[0].attributes = { exclude: _.map(queryOpt.excludeWP, f => _.split(f, '.')[1]) } } // Apply WorkPeriod filters - _.each(_.pick(criteria, ['workPeriods.startDate', 'workPeriods.endDate', 'workPeriods.paymentStatus']), (value, key) => { + _.each(_.pick(criteria, ['workPeriods.sentSurveyError', 'workPeriods.sentSurvey', 'workPeriods.startDate', 'workPeriods.endDate', 'workPeriods.paymentStatus']), (value, key) => { key = key.split('.')[1] queryCriteria.include[0].where[Op.and].push({ [key]: value }) }) @@ -869,6 +879,7 @@ searchResourceBookings.schema = Joi.object().keys({ Joi.string(), Joi.array().items(Joi.number().integer()) ), + sendWeeklySurvey: Joi.boolean(), billingAccountId: Joi.number().integer(), 'workPeriods.paymentStatus': Joi.alternatives( Joi.string(), @@ -891,6 +902,11 @@ searchResourceBookings.schema = Joi.object().keys({ return value }), 'workPeriods.userHandle': Joi.string(), + 'workPeriods.sentSurvey': Joi.boolean(), + 'workPeriods.sentSurveyError': Joi.object().keys({ + errorCode: Joi.number().integer().min(0), + errorMessage: Joi.string() + }).allow('').optional(), 'workPeriods.isFirstWeek': Joi.when(Joi.ref('workPeriods.startDate', { separator: false }), { is: Joi.exist(), then: Joi.boolean().default(false), diff --git a/src/services/SurveyService.js b/src/services/SurveyService.js new file mode 100644 index 00000000..0b80db4f --- /dev/null +++ b/src/services/SurveyService.js @@ -0,0 +1,134 @@ +const _ = require('lodash') +const logger = require('../common/logger') +const { searchResourceBookings } = require('./ResourceBookingService') +const { partiallyUpdateWorkPeriod } = require('./WorkPeriodService') +const { Scopes } = require('../../app-constants') +const { getUserById, getMemberDetailsByHandle } = require('../common/helper') +const { getCollectorName, createCollector, createMessage, upsertContactInSurveyMonkey, addContactsToSurvey, sendSurveyAPI } = require('../common/surveyMonkey') + +const resourceBookingCache = {} +const contactIdToWorkPeriodIdMap = {} +const emailToWorkPeriodIdMap = {} + +/** + * Scheduler process entrance + */ +async function sendSurveys () { + const currentUser = { + isMachine: true, + scopes: [Scopes.ALL_WORK_PERIOD, Scopes.ALL_WORK_PERIOD_PAYMENT] + } + + const criteria = { + fields: 'workPeriods,userId,id,sendWeeklySurvey', + sendWeeklySurvey: true, + 'workPeriods.paymentStatus': 'completed', + 'workPeriods.sentSurvey': false, + 'workPeriods.sentSurveyError': '', + jobIds: [], + page: 1, + perPage: 1000 + } + + const options = { + returnAll: false, + returnFromDB: false + } + try { + let resourceBookings = await searchResourceBookings(currentUser, criteria, options) + resourceBookings = resourceBookings.result + + logger.info({ component: 'SurveyService', context: 'sendSurvey', message: 'load workPeriod successfullly' }) + + const workPeriods = _.flatten(_.map(resourceBookings, 'workPeriods')) + + const collectors = {} + + // for each WorkPeriod make sure to creat a collector (one per week) + // so several WorkPeriods for the same week would be included into on collector + // and gather contacts (members) from each WorkPeriods + for (const workPeriod of workPeriods) { + // await partiallyUpdateWorkPeriod(currentUser, workPeriod.id, {sentSurvey: true}) + // await partiallyUpdateWorkPeriod(currentUser, workPeriod.id, {sentSurveyError: {errorCode: 23, errorMessage: "sf"}}) + try { + const collectorName = getCollectorName(workPeriod.endDate) + + // create collector and message for each week if not yet + if (!collectors[collectorName]) { + const collectorId = await createCollector(collectorName) + const messageId = await createMessage(collectorId) + // create map + contactIdToWorkPeriodIdMap[collectorName] = {} + emailToWorkPeriodIdMap[collectorName] = {} + collectors[collectorName] = { + collectorId, + messageId, + contacts: [] + } + } + + const resourceBooking = _.find(resourceBookings, (r) => r.id === workPeriod.resourceBookingId) + const userInfo = {} + if (!resourceBookingCache[resourceBooking.userId]) { + let user = await getUserById(resourceBooking.userId) + if (!user.email && user.handle) { + user = await getMemberDetailsByHandle(user.handle) + } + if (user.email) { + userInfo.email = user.email + if (user.firstName) { + userInfo.first_name = user.firstName + } + if (user.lastName) { + userInfo.last_name = user.lastName + } + resourceBookingCache[resourceBooking.userId] = userInfo + } + } + emailToWorkPeriodIdMap[collectorName][resourceBookingCache[resourceBooking.userId].email] = workPeriod.id + // resourceBookingCache[resourceBooking.userId].workPeriodId = workPeriod.id + collectors[collectorName].contacts.push(resourceBookingCache[resourceBooking.userId]) + } catch (e) { + await partiallyUpdateWorkPeriod(currentUser, workPeriod.id, { sentSurveyError: e }) + } + } + // add contacts + for (const collectorName in collectors) { + const collector = collectors[collectorName] + collectors[collectorName].contacts = await upsertContactInSurveyMonkey(collector.contacts) + + for (const contact of collectors[collectorName].contacts) { + contactIdToWorkPeriodIdMap[collectorName][contact.id] = emailToWorkPeriodIdMap[collectorName][contact.email] + } + } + + // send surveys + for (const collectorName in collectors) { + const collector = collectors[collectorName] + try { + await addContactsToSurvey( + collector.collectorId, + collector.messageId, + collector.contacts + ) + await sendSurveyAPI(collector.collectorId, collector.messageId) + + for (const contactId in contactIdToWorkPeriodIdMap[collectorName]) { + await partiallyUpdateWorkPeriod(currentUser, contactIdToWorkPeriodIdMap[collectorName][contactId], { sentSurvey: true }) + } + } catch (e) { + for (const contactId in contactIdToWorkPeriodIdMap[collectorName]) { + await partiallyUpdateWorkPeriod(currentUser, contactIdToWorkPeriodIdMap[collectorName][contactId], { sentSurveyError: e }) + } + } + } + + logger.info({ component: 'SurveyService', context: 'sendSurvey', message: 'send survey successfullly' }) + } catch (e) { + logger.error({ component: 'SurveyService', context: 'sendSurvey', message: 'Error : ' + e.message }) + } +} + +module.exports = { + sendSurveys +} diff --git a/src/services/WorkPeriodService.js b/src/services/WorkPeriodService.js index 8d018fb0..d28174a2 100644 --- a/src/services/WorkPeriodService.js +++ b/src/services/WorkPeriodService.js @@ -241,6 +241,7 @@ createWorkPeriod.schema = Joi.object().keys({ resourceBookingId: Joi.string().uuid().required(), startDate: Joi.workPeriodStartDate(), endDate: Joi.workPeriodEndDate(), + sentSurvey: Joi.boolean().default(true), daysWorked: Joi.number().integer().min(0).max(5).required(), daysPaid: Joi.number().default(0).forbidden(), paymentTotal: Joi.number().default(0).forbidden(), @@ -277,7 +278,9 @@ async function updateWorkPeriod (currentUser, id, data) { throw new errors.BadRequestError(`Maximum allowed daysWorked is (${thisWeek.daysWorked})`) } data.paymentStatus = helper.calculateWorkPeriodPaymentStatus(_.assign({}, oldValue, data)) - data.updatedBy = await helper.getUserId(currentUser.userId) + if (!currentUser.isMachine) { + data.updatedBy = await helper.getUserId(currentUser.userId) + } const updated = await workPeriod.update(data) const updatedDataWithoutPayments = _.omit(updated.toJSON(), ['payments']) const oldValueWithoutPayments = _.omit(oldValue, ['payments']) @@ -300,7 +303,12 @@ partiallyUpdateWorkPeriod.schema = Joi.object().keys({ currentUser: Joi.object().required(), id: Joi.string().uuid().required(), data: Joi.object().keys({ - daysWorked: Joi.number().integer().min(0).max(5) + daysWorked: Joi.number().integer().min(0).max(5), + sentSurvey: Joi.boolean(), + sentSurveyError: Joi.object().keys({ + errorCode: Joi.number().integer().min(0), + errorMessage: Joi.string() + }) }).required().min(1) }).required() @@ -499,6 +507,11 @@ searchWorkPeriods.schema = Joi.object().keys({ userHandle: Joi.string(), projectId: Joi.number().integer(), resourceBookingId: Joi.string().uuid(), + sentSurvey: Joi.boolean(), + sentSurveyError: Joi.object().keys({ + errorCode: Joi.number().integer().min(0), + errorMessage: Joi.string() + }), resourceBookingIds: Joi.alternatives( Joi.string(), Joi.array().items(Joi.string().uuid()) From d3b60400cf0f07165576b4e45adf198c6f4b7f38 Mon Sep 17 00:00:00 2001 From: yoution Date: Tue, 3 Aug 2021 10:42:10 +0800 Subject: [PATCH 2/3] fix: workly-surveys --- ...21-07-26-add-send-weekly-survery-fields.js | 5 +- package.json | 1 - .../data/updateWorkPeriodSentSurveyField.js | 68 ------------------- src/common/surveyMonkey.js | 4 +- src/models/WorkPeriod.js | 2 +- src/services/SurveyService.js | 37 +++++----- 6 files changed, 24 insertions(+), 93 deletions(-) delete mode 100644 scripts/data/updateWorkPeriodSentSurveyField.js diff --git a/migrations/2021-07-26-add-send-weekly-survery-fields.js b/migrations/2021-07-26-add-send-weekly-survery-fields.js index 025d79d9..06a45672 100644 --- a/migrations/2021-07-26-add-send-weekly-survery-fields.js +++ b/migrations/2021-07-26-add-send-weekly-survery-fields.js @@ -1,4 +1,5 @@ const config = require('config') +const moment = require('moment') module.exports = { up: async (queryInterface, Sequelize) => { @@ -22,6 +23,8 @@ module.exports = { type: Sequelize.STRING(255) }, }), allowNull: true }, { transaction }) + await queryInterface.sequelize.query(`UPDATE ${config.DB_SCHEMA_NAME}.work_periods SET sent_survey = true where payment_status = 'completed' and end_date <= '${moment().subtract(7, 'days').format('YYYY-MM-DD')}'`, + { transaction }) await transaction.commit() } catch (err) { await transaction.rollback() @@ -32,7 +35,7 @@ module.exports = { const transaction = await queryInterface.sequelize.transaction() try { await queryInterface.removeColumn({ tableName: 'resource_bookings', schema: config.DB_SCHEMA_NAME }, 'send_weekly_survey', { transaction }) - await queryInterface.removeColumn({ tableName: 'work_periods', schema: config.DB_SCHEMA_NAME }, 'send_survey', { transaction }) + await queryInterface.removeColumn({ tableName: 'work_periods', schema: config.DB_SCHEMA_NAME }, 'sent_survey', { transaction }) await queryInterface.removeColumn({ tableName: 'work_periods', schema: config.DB_SCHEMA_NAME }, 'sent_survey_error', { transaction } ) await transaction.commit() } catch (err) { diff --git a/package.json b/package.json index 70b1913c..17c1887b 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,6 @@ "index:roles": "node scripts/es/reIndexRoles.js", "data:export": "node scripts/data/exportData.js", "data:import": "node scripts/data/importData.js", - "data:workperiod": "node scripts/data/updateWorkPeriodSentSurveyField.js", "migrate": "npx sequelize db:migrate", "migrate:undo": "npx sequelize db:migrate:undo", "test": "mocha test/unit/*.test.js --timeout 30000 --require test/prepare.js --exit", diff --git a/scripts/data/updateWorkPeriodSentSurveyField.js b/scripts/data/updateWorkPeriodSentSurveyField.js deleted file mode 100644 index e641b177..00000000 --- a/scripts/data/updateWorkPeriodSentSurveyField.js +++ /dev/null @@ -1,68 +0,0 @@ -/* - * update WorkPeriod field `sentSurvey=true` - */ -const _ = require('lodash') -const moment = require('moment') -const logger = require('../../src/common/logger') -const { Op } = require('sequelize') -const models = require('../../src/models') - -const ResourceBooking = models.ResourceBooking -const WorkPeriod = models.WorkPeriod - -async function updateWorkPeriod () { - const transaction = await models.sequelize.transaction() - try { - // Start a transaction - const queryCriteria = { - attributes: ['sendWeeklySurvey', 'id'], - include: [{ - as: 'workPeriods', - model: WorkPeriod, - required: true, - where: { - [Op.and]: [ - { sentSurveyError: null }, - { sentSurvey: false }, - { paymentStatus: ['completed'] }, - { endDate: { [Op.lte]: moment().subtract(7, 'days').format('YYYY-MM-DD') } } - ] - } - }], - where: { - [Op.and]: [{ sendWeeklySurvey: true }] - }, - transaction - } - - const resourceBookings = await ResourceBooking.findAll(queryCriteria) - - _.forEach(resourceBookings, r => { - _.forEach(r.workPeriods, async w => { - // await w.update({sentSurvey: true}, {transaction: transaction} ) - await w.update({ sentSurvey: true }) - }) - }) - - // commit transaction only if all things went ok - logger.info({ - component: 'importData', - message: 'committing transaction to database...' - }) - await transaction.commit() - } catch (error) { - // logger.error({ - // component: 'importData', - // message: `Error while writing data of model: WorkPeriod` - // }) - // rollback all insert operations - if (transaction) { - logger.info({ - component: 'importData', - message: 'rollback database transaction...' - }) - transaction.rollback() - } - } -} -updateWorkPeriod() diff --git a/src/common/surveyMonkey.js b/src/common/surveyMonkey.js index c3a73499..5a65b0c8 100644 --- a/src/common/surveyMonkey.js +++ b/src/common/surveyMonkey.js @@ -57,7 +57,7 @@ function getSingleItem (lst, errorMessage) { * format `Week Ending yyyy-nth(weeks)` */ function getCollectorName (dt) { - return 'Week Ending ' + moment(dt).year() + '-' + moment(dt).format('ww') + return 'Week Ending ' + moment(dt).format('M/D/YYYY') } /* @@ -123,7 +123,6 @@ async function cloneCollector () { */ async function renameCollector (collectorId, name) { const body = { name: name } - // http.patch(BASE_URL + '/collectors/' + collectorId, body); const url = `${config.WEEKLY_SURVEY.BASE_URL}/collectors/${collectorId}` try { const response = await request @@ -147,7 +146,6 @@ async function createMessage (collectorId) { from_collector_id: `${config.WEEKLY_SURVEY.SURVEY_MASTER_COLLECTOR_ID}`, from_message_id: `${config.WEEKLY_SURVEY.SURVEY_MASTER_MESSAGE_ID}` } - // response = http.post(BASE_URL + '/collectors/' + collectorId + '/messages', body); const url = `${config.WEEKLY_SURVEY.BASE_URL}/collectors/${collectorId}/messages` try { const response = await request diff --git a/src/models/WorkPeriod.js b/src/models/WorkPeriod.js index 0204d9be..d2a3b12c 100644 --- a/src/models/WorkPeriod.js +++ b/src/models/WorkPeriod.js @@ -57,7 +57,7 @@ module.exports = (sequelize) => { allowNull: false }, sentSurvey: { - field: 'send_survey', + field: 'sent_survey', type: Sequelize.BOOLEAN, defaultValue: false, allowNull: false diff --git a/src/services/SurveyService.js b/src/services/SurveyService.js index 0b80db4f..8bdca8ca 100644 --- a/src/services/SurveyService.js +++ b/src/services/SurveyService.js @@ -26,13 +26,12 @@ async function sendSurveys () { 'workPeriods.sentSurvey': false, 'workPeriods.sentSurveyError': '', jobIds: [], - page: 1, - perPage: 1000 + page: 1 } const options = { - returnAll: false, - returnFromDB: false + returnAll: true, + returnFromDB: true } try { let resourceBookings = await searchResourceBookings(currentUser, criteria, options) @@ -48,8 +47,6 @@ async function sendSurveys () { // so several WorkPeriods for the same week would be included into on collector // and gather contacts (members) from each WorkPeriods for (const workPeriod of workPeriods) { - // await partiallyUpdateWorkPeriod(currentUser, workPeriod.id, {sentSurvey: true}) - // await partiallyUpdateWorkPeriod(currentUser, workPeriod.id, {sentSurveyError: {errorCode: 23, errorMessage: "sf"}}) try { const collectorName = getCollectorName(workPeriod.endDate) @@ -86,7 +83,6 @@ async function sendSurveys () { } } emailToWorkPeriodIdMap[collectorName][resourceBookingCache[resourceBooking.userId].email] = workPeriod.id - // resourceBookingCache[resourceBooking.userId].workPeriodId = workPeriod.id collectors[collectorName].contacts.push(resourceBookingCache[resourceBooking.userId]) } catch (e) { await partiallyUpdateWorkPeriod(currentUser, workPeriod.id, { sentSurveyError: e }) @@ -105,20 +101,22 @@ async function sendSurveys () { // send surveys for (const collectorName in collectors) { const collector = collectors[collectorName] - try { - await addContactsToSurvey( - collector.collectorId, - collector.messageId, - collector.contacts - ) - await sendSurveyAPI(collector.collectorId, collector.messageId) - - for (const contactId in contactIdToWorkPeriodIdMap[collectorName]) { - await partiallyUpdateWorkPeriod(currentUser, contactIdToWorkPeriodIdMap[collectorName][contactId], { sentSurvey: true }) + if (collector.contacts.length) { + try { + await addContactsToSurvey( + collector.collectorId, + collector.messageId, + collector.contacts + ) + await sendSurveyAPI(collector.collectorId, collector.messageId) + } catch (e) { + for (const contactId in contactIdToWorkPeriodIdMap[collectorName]) { + await partiallyUpdateWorkPeriod(currentUser, contactIdToWorkPeriodIdMap[collectorName][contactId], { sentSurveyError: e }) + } + continue } - } catch (e) { for (const contactId in contactIdToWorkPeriodIdMap[collectorName]) { - await partiallyUpdateWorkPeriod(currentUser, contactIdToWorkPeriodIdMap[collectorName][contactId], { sentSurveyError: e }) + await partiallyUpdateWorkPeriod(currentUser, contactIdToWorkPeriodIdMap[collectorName][contactId], { sentSurvey: true }) } } } @@ -126,6 +124,7 @@ async function sendSurveys () { logger.info({ component: 'SurveyService', context: 'sendSurvey', message: 'send survey successfullly' }) } catch (e) { logger.error({ component: 'SurveyService', context: 'sendSurvey', message: 'Error : ' + e.message }) + throw e } } From 19a487d357dcc6e072524ba0841edb8c7867bff1 Mon Sep 17 00:00:00 2001 From: yoution Date: Wed, 4 Aug 2021 17:29:52 +0800 Subject: [PATCH 3/3] fix: add catch for partiallyUpdateWorkPeriod --- src/services/SurveyService.js | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/services/SurveyService.js b/src/services/SurveyService.js index 8bdca8ca..942608ed 100644 --- a/src/services/SurveyService.js +++ b/src/services/SurveyService.js @@ -111,12 +111,20 @@ async function sendSurveys () { await sendSurveyAPI(collector.collectorId, collector.messageId) } catch (e) { for (const contactId in contactIdToWorkPeriodIdMap[collectorName]) { - await partiallyUpdateWorkPeriod(currentUser, contactIdToWorkPeriodIdMap[collectorName][contactId], { sentSurveyError: e }) + try { + await partiallyUpdateWorkPeriod(currentUser, contactIdToWorkPeriodIdMap[collectorName][contactId], { sentSurveyError: e }) + } catch (e) { + logger.error({ component: 'SurveyService', context: 'sendSurvey', message: 'Error : ' + e.message }) + } } continue } for (const contactId in contactIdToWorkPeriodIdMap[collectorName]) { - await partiallyUpdateWorkPeriod(currentUser, contactIdToWorkPeriodIdMap[collectorName][contactId], { sentSurvey: true }) + try { + await partiallyUpdateWorkPeriod(currentUser, contactIdToWorkPeriodIdMap[collectorName][contactId], { sentSurvey: true }) + } catch (e) { + logger.error({ component: 'SurveyService', context: 'sendSurvey', message: 'Error : ' + e.message }) + } } } } @@ -124,7 +132,6 @@ async function sendSurveys () { logger.info({ component: 'SurveyService', context: 'sendSurvey', message: 'send survey successfullly' }) } catch (e) { logger.error({ component: 'SurveyService', context: 'sendSurvey', message: 'Error : ' + e.message }) - throw e } }