Merge pull request #318 from topcoder-platform/feature/payment-scheduler

Payment scheduler

Author: maxceem

README.md

@@ -207,6 +207,7 @@ To be able to change and test `taas-es-processor` locally you can follow the nex
 | `npm run cov`                                                                                                             | Code Coverage Report.                                                                                                                              |
 | `npm run migrate`                                                                                                         | Run any migration files which haven't run yet.                                                                                                     |
 | `npm run migrate:undo`                                                                                                    | Revert most recent migration.                                                                                                                      |
+| `npm run demo-payment-scheduler`                                                                                          | Create 1000 Work Periods Payment records in with status "scheduled" and various "amount"                                                                         |
 
 ## Import and Export data
 

app-constants.js

@@ -86,8 +86,24 @@ const ChallengeStatus = {
 
 const WorkPeriodPaymentStatus = {
   COMPLETED: 'completed',
-  CANCELLED: 'cancelled',
-  SCHEDULED: 'scheduled'
+  SCHEDULED: 'scheduled',
+  IN_PROGRESS: 'in-progress',
+  FAILED: 'failed',
+  CANCELLED: 'cancelled'
+}
+
+const PaymentProcessingSwitch = {
+  ON: 'ON',
+  OFF: 'OFF'
+}
+
+const PaymentSchedulerStatus = {
+  START_PROCESS: 'start-process',
+  CREATE_CHALLENGE: 'create-challenge',
+  ASSIGN_MEMBER: 'assign-member',
+  ACTIVATE_CHALLENGE: 'activate-challenge',
+  GET_USER_ID: 'get-userId',
+  CLOSE_CHALLENGE: 'close-challenge'
 }
 
 module.exports = {
@@ -96,5 +112,7 @@ module.exports = {
   Scopes,
   Interviews,
   ChallengeStatus,
-  WorkPeriodPaymentStatus
+  WorkPeriodPaymentStatus,
+  PaymentSchedulerStatus,
+  PaymentProcessingSwitch
 }

app.js

@@ -13,6 +13,7 @@ const schedule = require('node-schedule')
 const logger = require('./src/common/logger')
 const eventHandlers = require('./src/eventHandlers')
 const interviewService = require('./src/services/InterviewService')
+const { processScheduler } = require('./src/services/PaymentSchedulerService')
 
 // setup express app
 const app = express()
@@ -97,6 +98,9 @@ const server = app.listen(app.get('port'), () => {
   eventHandlers.init()
   // schedule updateCompletedInterviews to run every hour
   schedule.scheduleJob('0 0 * * * *', interviewService.updateCompletedInterviews)
+
+  // schedule payment processing
+  schedule.scheduleJob(config.PAYMENT_PROCESSING.CRON, processScheduler)
 })
 
 if (process.env.NODE_ENV === 'test') {

config/default.js

@@ -175,5 +175,40 @@ module.exports = {
   // the minimum matching rate when searching roles by skills
   ROLE_MATCHING_RATE: process.env.ROLE_MATCHING_RATE || 0.70,
   // member groups representing Wipro or TopCoder employee
-  INTERNAL_MEMBER_GROUPS: process.env.INTERNAL_MEMBER_GROUPS || ['20000000', '20000001', '20000003', '20000010', '20000015']
+  INTERNAL_MEMBER_GROUPS: process.env.INTERNAL_MEMBER_GROUPS || ['20000000', '20000001', '20000003', '20000010', '20000015'],
+  // payment scheduler config
+  PAYMENT_PROCESSING: {
+    // switch off actual API calls in Payment Scheduler
+    SWITCH: process.env.PAYMENT_PROCESSING_SWITCH || 'OFF',
+    // the payment scheduler cron config
+    CRON: process.env.PAYMENT_PROCESSING_CRON || '0 */5 * * * *',
+    // the number of records processed by one time
+    BATCH_SIZE: parseInt(process.env.PAYMENT_PROCESSING_BATCH_SIZE || 50),
+    // in-progress expired to determine whether a record has been processed abnormally, moment duration format
+    IN_PROGRESS_EXPIRED: process.env.IN_PROGRESS_EXPIRED || 'PT1H',
+    // the number of max retry config
+    MAX_RETRY_COUNT: parseInt(process.env.PAYMENT_PROCESSING_MAX_RETRY_COUNT || 10),
+    // the time of retry base delay, unit: ms
+    RETRY_BASE_DELAY: parseInt(process.env.PAYMENT_PROCESSING_RETRY_BASE_DELAY || 100),
+    // the time of retry max delay, unit: ms
+    RETRY_MAX_DELAY: parseInt(process.env.PAYMENT_PROCESSING_RETRY_MAX_DELAY || 10000),
+    // the max time of one request, unit: ms
+    PER_REQUEST_MAX_TIME: parseInt(process.env.PAYMENT_PROCESSING_PER_REQUEST_MAX_TIME || 30000),
+    // the max time of one payment record, unit: ms
+    PER_PAYMENT_MAX_TIME: parseInt(process.env.PAYMENT_PROCESSING_PER_PAYMENT_MAX_TIME || 60000),
+    // the max records of payment of a minute
+    PER_MINUTE_PAYMENT_MAX_COUNT: parseInt(process.env.PAYMENT_PROCESSING_PER_MINUTE_PAYMENT_MAX_COUNT || 12),
+    // the max requests of challenge of a minute
+    PER_MINUTE_CHALLENGE_REQUEST_MAX_COUNT: parseInt(process.env.PAYMENT_PROCESSING_PER_MINUTE_CHALLENGE_REQUEST_MAX_COUNT || 60),
+    // the max requests of resource of a minute
+    PER_MINUTE_RESOURCE_REQUEST_MAX_COUNT: parseInt(process.env.PAYMENT_PROCESSING_PER_MINUTE_CHALLENGE_REQUEST_MAX_COUNT || 20),
+    // the default step fix delay, unit: ms
+    FIX_DELAY_STEP: parseInt(process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP || 500),
+    // the fix delay after step of create challenge, unit: ms
+    FIX_DELAY_STEP_CREATE_CHALLENGE: parseInt(process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP_CREATE_CHALLENGE || process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP || 500),
+    // the fix delay after step of assign member, unit: ms
+    FIX_DELAY_STEP_ASSIGN_MEMBER: parseInt(process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP_ASSIGN_MEMBER || process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP || 500),
+    // the fix delay after step of activate challenge, unit: ms
+    FIX_DELAY_STEP_ACTIVATE_CHALLENGE: parseInt(process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP_ACTIVATE_CHALLENGE || process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP || 500)
+  }
 }

docs/swagger.yaml

@@ -2403,7 +2403,7 @@ paths:
           required: false
           schema:
             type: string
-            enum: ["completed", "scheduled", "cancelled"]
+            enum: ["completed", "scheduled", "in-progress", "failed", "cancelled"]
           description: The payment status.
       responses:
         "200":
@@ -2519,7 +2519,7 @@ paths:
             application/json:
               schema:
                 $ref: "#/components/schemas/Error"
-    
+
   /work-period-payments/{id}:
     get:
       tags:
@@ -4830,8 +4830,22 @@ components:
           description: "The amount to be paid."
         status:
           type: string
-          enum: ["completed", "scheduled", "cancelled"]
+          enum: ["completed", "scheduled", "in-progress", "failed", "cancelled"]
           description: "The payment status."
+        statusDetails:
+          type: object
+          properties:
+            errorMessage:
+              type: string
+            errorCode:
+              type: integer
+            retry:
+              type: integer
+            step:
+              type: string
+            challengeId:
+              type: string
+              format: uuid
         billingAccountId:
           type: integer
           example: 80000071
@@ -4888,7 +4902,7 @@ components:
           description: "The amount to be paid."
         status:
           type: string
-          enum: ["completed", "scheduled", "cancelled"]
+          enum: ["completed", "scheduled", "in-progress", "failed", "cancelled"]
           description: "The payment status."
     WorkPeriodPaymentCreateRequestBody:
       required:
@@ -4979,7 +4993,7 @@ components:
           description: "The amount to be paid."
         status:
           type: string
-          enum: ["completed", "scheduled", "cancelled"]
+          enum: ["completed", "scheduled", "in-progress", "failed", "cancelled"]
           description: "The payment status."
     CheckRun:
       type: object

migrations/2021-05-29-create-payment-scheduler-table-add-status-details-to-payment.js

@@ -0,0 +1,112 @@
+'use strict';
+
+const config = require('config')
+const _ = require('lodash')
+const { PaymentSchedulerStatus } = require('../app-constants')
+
+/**
+ * Create `payment_schedulers` table & relations.
+ */
+module.exports = {
+  up: async (queryInterface, Sequelize) => {
+    const transaction = await queryInterface.sequelize.transaction()
+    try { 
+      await queryInterface.createTable('payment_schedulers', {
+        id: {
+          type: Sequelize.UUID,
+          primaryKey: true,
+          allowNull: false,
+          defaultValue: Sequelize.UUIDV4
+        },
+        challengeId: {
+          field: 'challenge_id',
+          type: Sequelize.UUID,
+          allowNull: false
+        },
+        workPeriodPaymentId: {
+          field: 'work_period_payment_id',
+          type: Sequelize.UUID,
+          allowNull: false,
+          references: {
+            model: {
+              tableName: 'work_period_payments',
+              schema: config.DB_SCHEMA_NAME
+            },
+            key: 'id'
+          }
+        },
+        step: {
+          type: Sequelize.ENUM(_.values(PaymentSchedulerStatus)),
+          allowNull: false
+        },
+        status: {
+          type: Sequelize.ENUM(
+            'in-progress',
+            'completed',
+            'failed'
+          ),
+          allowNull: false
+        },
+        userId: {
+          field: 'user_id',
+          type: Sequelize.BIGINT
+        },
+        userHandle: {
+          field: 'user_handle',
+          type: Sequelize.STRING,
+          allowNull: false
+        },
+        createdAt: {
+          field: 'created_at',
+          type: Sequelize.DATE
+        },
+        updatedAt: {
+          field: 'updated_at',
+          type: Sequelize.DATE
+        },
+        deletedAt: {
+          field: 'deleted_at',
+          type: Sequelize.DATE
+        }
+      }, { schema: config.DB_SCHEMA_NAME, transaction })
+      await queryInterface.addColumn({ tableName: 'work_period_payments', schema: config.DB_SCHEMA_NAME }, 'status_details',
+        { type: Sequelize.JSONB },
+        { transaction })
+      await queryInterface.changeColumn({ tableName: 'work_period_payments', schema: config.DB_SCHEMA_NAME }, 'challenge_id',
+        { type: Sequelize.UUID },
+        { transaction })
+      await queryInterface.sequelize.query(`ALTER TYPE ${config.DB_SCHEMA_NAME}.enum_work_period_payments_status ADD VALUE 'scheduled'`)
+      await queryInterface.sequelize.query(`ALTER TYPE ${config.DB_SCHEMA_NAME}.enum_work_period_payments_status ADD VALUE 'in-progress'`)
+      await queryInterface.sequelize.query(`ALTER TYPE ${config.DB_SCHEMA_NAME}.enum_work_period_payments_status ADD VALUE 'failed'`)
+      await transaction.commit()
+    } catch (err) {
+      await transaction.rollback()
+      throw err
+    }
+  },
+
+  down: async (queryInterface, Sequelize) => {
+    const table = { schema: config.DB_SCHEMA_NAME, tableName: 'payment_schedulers' }
+    const statusTypeName = `${table.schema}.enum_${table.tableName}_status`
+    const stepTypeName = `${table.schema}.enum_${table.tableName}_step`
+    const transaction = await queryInterface.sequelize.transaction()
+    try {
+      await queryInterface.dropTable(table, { transaction })
+      // drop enum type for status and step column
+      await queryInterface.sequelize.query(`DROP TYPE ${statusTypeName}`, { transaction })
+      await queryInterface.sequelize.query(`DROP TYPE ${stepTypeName}`, { transaction })
+
+      await queryInterface.changeColumn({ tableName: 'work_period_payments', schema: config.DB_SCHEMA_NAME }, 'challenge_id',
+        { type: Sequelize.UUID, allowNull: false },
+        { transaction })
+      await queryInterface.removeColumn({ tableName: 'work_period_payments', schema: config.DB_SCHEMA_NAME }, 'status_details',
+        { transaction })
+      await queryInterface.sequelize.query(`DELETE FROM pg_enum WHERE enumlabel in ('scheduled', 'in-progress', 'failed') AND enumtypid = (SELECT oid FROM pg_type WHERE typname = 'enum_work_period_payments_status')`,
+        { transaction })
+      await transaction.commit()
+    } catch (err) {
+      await transaction.rollback()
+      throw err
+    }
+  }
+};

package.json

@@ -27,6 +27,7 @@
     "local:init": "npm run local:reset && npm run data:import -- --force",
     "local:reset": "npm run delete-index -- --force || true && npm run create-index -- --force && npm run init-db force",
     "cov": "nyc --reporter=html --reporter=text npm run test",
+    "demo-payment-scheduler": "node scripts/demo-payment-scheduler/index.js && npm run index:all -- --force",
     "demo-payment": "node scripts/demo-payment"
   },
   "keywords": [],

scripts/demo-payment-scheduler/data.json

@@ -0,0 +1,103 @@
+{
+    "Job":{
+       "id":"43d695d4-e926-41d5-ad42-a899612b5246",
+       "projectId":17234,
+       "title":"Dummy title - at most 64 characters",
+       "numPositions":13,
+       "skills":[
+          "23e00d92-207a-4b5b-b3c9-4c5662644941",
+          "7d076384-ccf6-4e43-a45d-1b24b1e624aa",
+          "cbac57a3-7180-4316-8769-73af64893158",
+          "a2b4bc11-c641-4a19-9eb7-33980378f82e"
+       ],
+       "status":"in-review",
+       "isApplicationPageActive":false,
+       "createdBy":"57646ff9-1cd3-4d3c-88ba-eb09a395366c",
+       "updatedBy":"00000000-0000-0000-0000-000000000000",
+       "createdAt":"2021-05-09T21:21:10.394Z",
+       "updatedAt":"2021-05-09T21:21:14.010Z"
+    },
+    "ResourceBooking":{
+       "id":"41671764-0ded-46fd-b7de-2af5d5e4f3fc",
+       "projectId":17234,
+       "userId":"05e988b7-7d54-4c10-ada1-1a04870a88a8",
+       "jobId":"43d695d4-e926-41d5-ad42-a899612b5246",
+       "status":"placed",
+       "startDate":"2020-09-27",
+       "endDate":"2020-10-27",
+       "memberRate":13.23,
+       "customerRate":13,
+       "rateType":"hourly",
+       "billingAccountId":80000069,
+       "createdBy":"57646ff9-1cd3-4d3c-88ba-eb09a395366c",
+       "updatedBy":null,
+       "createdAt":"2021-05-09T21:25:46.728Z",
+       "updatedAt":"2021-05-09T21:25:46.728Z"
+    },
+    "WorkPeriods":[
+       {
+          "id":"4baae2cf-fd70-4ab3-9959-e826257b7e0f",
+          "resourceBookingId":"41671764-0ded-46fd-b7de-2af5d5e4f3fc",
+          "userHandle":"pshah_manager",
+          "projectId":17234,
+          "startDate":"2020-09-27",
+          "endDate":"2020-10-03",
+          "daysWorked":4,
+          "memberRate":27.06,
+          "customerRate":13.13,
+          "paymentStatus":"partially-completed",
+          "createdBy":"00000000-0000-0000-0000-000000000000",
+          "updatedBy":"57646ff9-1cd3-4d3c-88ba-eb09a395366c",
+          "createdAt":"2021-05-09T21:25:47.813Z",
+          "updatedAt":"2021-05-09T21:45:32.659Z"
+       },
+       {
+          "id":"9918e1b7-acbc-41ae-baa6-fdcb2386681d",
+          "resourceBookingId":"41671764-0ded-46fd-b7de-2af5d5e4f3fc",
+          "userHandle":"Shuchikr",
+          "projectId":17234,
+          "startDate":"2020-10-18",
+          "endDate":"2020-10-24",
+          "daysWorked":4,
+          "memberRate":4.08,
+          "customerRate":3.89,
+          "paymentStatus":"cancelled",
+          "createdBy":"00000000-0000-0000-0000-000000000000",
+          "updatedBy":"57646ff9-1cd3-4d3c-88ba-eb09a395366c",
+          "createdAt":"2021-05-09T21:25:47.834Z",
+          "updatedAt":"2021-05-09T21:45:37.647Z"
+       },
+       {
+          "id":"42e990c9-b14c-4496-9977-c3024aa90024",
+          "resourceBookingId":"41671764-0ded-46fd-b7de-2af5d5e4f3fc",
+          "userHandle":"vkumars",
+          "projectId":17234,
+          "startDate":"2020-10-25",
+          "endDate":"2020-10-31",
+          "daysWorked":3,
+          "memberRate":15.61,
+          "customerRate":9.76,
+          "paymentStatus":"pending",
+          "createdBy":"00000000-0000-0000-0000-000000000000",
+          "updatedBy":"57646ff9-1cd3-4d3c-88ba-eb09a395366c",
+          "createdAt":"2021-05-09T21:25:47.824Z",
+          "updatedAt":"2021-05-09T21:45:48.727Z"
+       },
+       {
+          "id":"8bf64481-ae7b-4e51-b48c-000cd90c87d1",
+          "resourceBookingId":"41671764-0ded-46fd-b7de-2af5d5e4f3fc",
+          "userHandle":"chandanant",
+          "projectId":17234,
+          "startDate":"2020-10-11",
+          "endDate":"2020-10-17",
+          "daysWorked":4,
+          "memberRate":10.82,
+          "customerRate":30.71,
+          "paymentStatus":"pending",
+          "createdBy":"00000000-0000-0000-0000-000000000000",
+          "updatedBy":"57646ff9-1cd3-4d3c-88ba-eb09a395366c",
+          "createdAt":"2021-05-09T21:25:47.815Z",
+          "updatedAt":"2021-05-09T21:45:41.810Z"
+       }
+    ]
+ }