Feature/payment scheduler

README.md

@@ -87,6 +87,9 @@
    tc-taas-es-processor | [2021-04-09T21:20:19.035Z] app INFO : Starting kafka consumer
    tc-taas-es-processor | 2021-04-09T21:20:21.292Z INFO no-kafka-client Joined group taas-es-processor generationId 1 as no-kafka-client-076538fc-60dd-4ca4-a2b9-520bdf73bc9e
    tc-taas-es-processor | 2021-04-09T21:20:21.293Z INFO no-kafka-client Elected as group leader
+   tc-taas-es-processor | 2021-04-09T21:20:21.449Z DEBUG no-kafka-client Subscribed to taas.role.update:0 offset 0 leader kafka:9093
+   tc-taas-es-processor | 2021-04-09T21:20:21.450Z DEBUG no-kafka-client Subscribed to taas.role.delete:0 offset 0 leader kafka:9093
+   tc-taas-es-processor | 2021-04-09T21:20:21.451Z DEBUG no-kafka-client Subscribed to taas.role.requested:0 offset 0 leader kafka:9093
    tc-taas-es-processor | 2021-04-09T21:20:21.452Z DEBUG no-kafka-client Subscribed to taas.jobcandidate.create:0 offset 0 leader kafka:9093
    tc-taas-es-processor | 2021-04-09T21:20:21.455Z DEBUG no-kafka-client Subscribed to taas.job.create:0 offset 0 leader kafka:9093
    tc-taas-es-processor | 2021-04-09T21:20:21.456Z DEBUG no-kafka-client Subscribed to taas.resourcebooking.delete:0 offset 0 leader kafka:9093
@@ -103,7 +106,7 @@
    tc-taas-es-processor | 2021-04-09T21:20:21.473Z DEBUG no-kafka-client Subscribed to taas.job.update:0 offset 0 leader kafka:9093
    tc-taas-es-processor | 2021-04-09T21:20:21.474Z DEBUG no-kafka-client Subscribed to taas.resourcebooking.update:0 offset 0 leader kafka:9093
    tc-taas-es-processor | [2021-04-09T21:20:21.475Z] app INFO : Initialized.......
-   tc-taas-es-processor | [2021-04-09T21:20:21.479Z] app INFO : taas.job.create,taas.job.update,taas.job.delete,taas.jobcandidate.create,taas.jobcandidate.update,taas.jobcandidate.delete,taas.resourcebooking.create,taas.resourcebooking.update,taas.resourcebooking.delete,taas.workperiod.create,taas.workperiod.update,taas.workperiod.delete,taas.workperiodpayment.create,taas.workperiodpayment.update,taas.workperiodpayment.delete
+   tc-taas-es-processor | [2021-04-09T21:20:21.479Z] app INFO : taas.job.create,taas.job.update,taas.job.delete,taas.jobcandidate.create,taas.jobcandidate.update,taas.jobcandidate.delete,taas.resourcebooking.create,taas.resourcebooking.update,taas.resourcebooking.delete,taas.workperiod.create,taas.workperiod.update,taas.workperiod.delete,taas.workperiodpayment.create,taas.workperiodpayment.update,taas.interview.requested,taas.interview.update,taas.interview.bulkUpdate,taas.role.requested,taas.role.update,taas.role.delete
    tc-taas-es-processor | [2021-04-09T21:20:21.480Z] app INFO : Kick Start.......
    tc-taas-es-processor | ********** Topcoder Health Check DropIn listening on port 3001
    tc-taas-es-processor | Topcoder Health Check DropIn started and ready to roll
@@ -194,6 +197,7 @@ To be able to change and test `taas-es-processor` locally you can follow the nex
 | `npm run index:jobs <jobId>`                                                                                              | Indexes job data from db into ES, if jobId is not given all data is indexed. Use `-- --force` flag to skip confirmation                            |
 | `npm run index:job-candidates <jobCandidateId>`                                                                           | Indexes job candidate data from db into ES, if jobCandidateId is not given all data is indexed. Use `-- --force` flag to skip confirmation         |
 | `npm run index:resource-bookings <resourceBookingsId>`                                                                    | Indexes resource bookings data from db into ES, if resourceBookingsId is not given all data is indexed. Use `-- --force` flag to skip confirmation |
+| `npm run index:roles <roleId>`                                                                                            | Indexes roles data from db into ES, if roleId is not given all data is indexed. Use `-- --force` flag to skip confirmation                         |
 | `npm run services:up`                                                                                                     | Start services via docker-compose for local development.                                                                                           |
 | `npm run services:down`                                                                                                   | Stop services via docker-compose for local development.                                                                                            |
 | `npm run services:logs -- -f <service_name>`                                                                              | View logs of some service inside docker-compose.                                                                                                   |
@@ -202,6 +206,7 @@ To be able to change and test `taas-es-processor` locally you can follow the nex
 | `npm run cov`                                                                                                             | Code Coverage Report.                                                                                                                              |
 | `npm run migrate`                                                                                                         | Run any migration files which haven't run yet.                                                                                                     |
 | `npm run migrate:undo`                                                                                                    | Revert most recent migration.                                                                                                                      |
+| `npm run demo-payment-scheduler`                                                                                          | Create 1000 Work Periods Payment records in with status "scheduled" and various "amount"                                                                         |
 
 ## Import and Export data
 

app-constants.js

@@ -49,7 +49,13 @@ const Scopes = {
   READ_INTERVIEW: 'read:taas-interviews',
   CREATE_INTERVIEW: 'create:taas-interviews',
   UPDATE_INTERVIEW: 'update:taas-interviews',
-  ALL_INTERVIEW: 'all:taas-interviews'
+  ALL_INTERVIEW: 'all:taas-interviews',
+  // role
+  READ_ROLE: 'read:taas-roles',
+  CREATE_ROLE: 'create:taas-roles',
+  UPDATE_ROLE: 'update:taas-roles',
+  DELETE_ROLE: 'delete:taas-roles',
+  ALL_ROLE: 'all:taas-roles'
 }
 
 // Interview related constants
@@ -76,6 +82,14 @@ const ChallengeStatus = {
   COMPLETED: 'Completed'
 }
 
+const WorkPeriodPaymentStatus = {
+  COMPLETED: 'completed',
+  SCHEDULED: 'scheduled',
+  IN_PROGRESS: 'in-progress',
+  FAILED: 'failed',
+  CANCELLED: 'cancelled'
+}
+
 const PaymentProcessingSwitch = {
   ON: 'ON',
   OFF: 'OFF'
@@ -87,5 +101,6 @@ module.exports = {
   Scopes,
   Interviews,
   ChallengeStatus,
+  WorkPeriodPaymentStatus,
   PaymentProcessingSwitch
 }

app.js

@@ -13,6 +13,7 @@ const schedule = require('node-schedule')
 const logger = require('./src/common/logger')
 const eventHandlers = require('./src/eventHandlers')
 const interviewService = require('./src/services/InterviewService')
+const { processScheduler } = require('./src/services/PaymentSchedulerService')
 
 // setup express app
 const app = express()
@@ -97,6 +98,9 @@ const server = app.listen(app.get('port'), () => {
   eventHandlers.init()
   // schedule updateCompletedInterviews to run every hour
   schedule.scheduleJob('0 0 * * * *', interviewService.updateCompletedInterviews)
+
+  // schedule payment processing
+  schedule.scheduleJob(config.PAYMENT_PROCESSING.CRON, processScheduler)
 })
 
 if (process.env.NODE_ENV === 'test') {

config/default.js

@@ -40,7 +40,7 @@ module.exports = {
 
   TOPCODER_USERS_API: process.env.TOPCODER_USERS_API || 'https://api.topcoder-dev.com/v3/users',
   // the api to find topcoder members
-  TOPCODER_MEMBERS_API: process.env.TOPCODER_MEMBERS_API || 'https://api.topcoder-dev.com/v3/members',
+  TOPCODER_MEMBERS_API: process.env.TOPCODER_MEMBERS_API || 'https://api.topcoder-dev.com/v5/members',
   // rate limit of requests to user api
   MAX_PARALLEL_REQUEST_TOPCODER_USERS_API: process.env.MAX_PARALLEL_REQUEST_TOPCODER_USERS_API || 100,
 
@@ -76,6 +76,8 @@ module.exports = {
     ES_INDEX_JOB_CANDIDATE: process.env.ES_INDEX_JOB_CANDIDATE || 'job_candidate',
     // the resource booking index
     ES_INDEX_RESOURCE_BOOKING: process.env.ES_INDEX_RESOURCE_BOOKING || 'resource_booking',
+    // the role index
+    ES_INDEX_ROLE: process.env.ES_INDEX_ROLE || 'role',
 
     // the max bulk size in MB for ES indexing
     MAX_BULK_REQUEST_SIZE_MB: process.env.MAX_BULK_REQUEST_SIZE_MB || 20,
@@ -131,6 +133,13 @@ module.exports = {
   TAAS_INTERVIEW_UPDATE_TOPIC: process.env.TAAS_INTERVIEW_UPDATE_TOPIC || 'taas.interview.update',
   // the interview bulk update Kafka message topic
   TAAS_INTERVIEW_BULK_UPDATE_TOPIC: process.env.TAAS_INTERVIEW_BULK_UPDATE_TOPIC || 'taas.interview.bulkUpdate',
+  // topics for role service
+  // the create role entity Kafka message topic
+  TAAS_ROLE_CREATE_TOPIC: process.env.TAAS_ROLE_CREATE_TOPIC || 'taas.role.requested',
+  // the update role entity Kafka message topic
+  TAAS_ROLE_UPDATE_TOPIC: process.env.TAAS_ROLE_UPDATE_TOPIC || 'taas.role.update',
+  // the delete role entity Kafka message topic
+  TAAS_ROLE_DELETE_TOPIC: process.env.TAAS_ROLE_DELETE_TOPIC || 'taas.role.delete',
 
   // the Kafka message topic for sending email
   EMAIL_TOPIC: process.env.EMAIL_TOPIC || 'external.action.email',
@@ -162,5 +171,38 @@ module.exports = {
   DEFAULT_TIMELINE_TEMPLATE_ID: process.env.DEFAULT_TIMELINE_TEMPLATE_ID || '53a307ce-b4b3-4d6f-b9a1-3741a58f77e6',
   DEFAULT_TRACK_ID: process.env.DEFAULT_TRACK_ID || '9b6fc876-f4d9-4ccb-9dfd-419247628825',
 
-  PAYMENT_PROCESSING_SWITCH: process.env.PAYMENT_PROCESSING_SWITCH || 'OFF'
+  PAYMENT_PROCESSING: {
+    // switch off actual API calls in Payment Scheduler
+    SWITCH: process.env.PAYMENT_PROCESSING_SWITCH || 'OFF',
+    // the payment scheduler cron config
+    CRON: process.env.PAYMENT_PROCESSING_CRON || '0 */5 * * * *',
+    // the number of records processed by one time
+    BATCH_SIZE: parseInt(process.env.PAYMENT_PROCESSING_BATCH_SIZE || 50),
+    // in-progress expired to determine whether a record has been processed abnormally, moment duration format
+    IN_PROGRESS_EXPIRED: process.env.IN_PROGRESS_EXPIRED || 'PT1H',
+    // the number of max retry config
+    MAX_RETRY_COUNT: parseInt(process.env.PAYMENT_PROCESSING_MAX_RETRY_COUNT || 10),
+    // the time of retry base delay, unit: ms
+    RETRY_BASE_DELAY: parseInt(process.env.PAYMENT_PROCESSING_RETRY_BASE_DELAY || 100),
+    // the time of retry max delay, unit: ms
+    RETRY_MAX_DELAY: parseInt(process.env.PAYMENT_PROCESSING_RETRY_MAX_DELAY || 10000),
+    // the max time of one request, unit: ms
+    PER_REQUEST_MAX_TIME: parseInt(process.env.PAYMENT_PROCESSING_PER_REQUEST_MAX_TIME || 30000),
+    // the max time of one payment record, unit: ms
+    PER_PAYMENT_MAX_TIME: parseInt(process.env.PAYMENT_PROCESSING_PER_PAYMENT_MAX_TIME || 60000),
+    // the max records of payment of a minute
+    PER_MINUTE_PAYMENT_MAX_COUNT: parseInt(process.env.PAYMENT_PROCESSING_PER_MINUTE_PAYMENT_MAX_COUNT || 12),
+    // the max requests of challenge of a minute
+    PER_MINUTE_CHALLENGE_REQUEST_MAX_COUNT: parseInt(process.env.PAYMENT_PROCESSING_PER_MINUTE_CHALLENGE_REQUEST_MAX_COUNT || 60),
+    // the max requests of resource of a minute
+    PER_MINUTE_RESOURCE_REQUEST_MAX_COUNT: parseInt(process.env.PAYMENT_PROCESSING_PER_MINUTE_CHALLENGE_REQUEST_MAX_COUNT || 20),
+    // the default step fix delay, unit: ms
+    FIX_DELAY_STEP: parseInt(process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP || 500),
+    // the fix delay between step one and step two, unit: ms
+    FIX_DELAY_STEP_1_2: parseInt(process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP_1_2 || process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP || 500),
+    // the fix delay between step two and step three, unit: ms
+    FIX_DELAY_STEP_2_3: parseInt(process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP_2_3 || process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP || 500),
+    // the fix delay between step three and step four, unit: ms
+    FIX_DELAY_STEP_3_4: parseInt(process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP_3_4 || process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP || 500)
+  }
 }