Skip to content

Commit f63256b

Browse files
maxceemmaxceem
authored
Merge pull request #308 from xxcxy/feature/payment-scheduler
Feature/payment scheduler
2 parents 232329c + a674ef9 commit f63256b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+22928
-7875
lines changed

‎README.md

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,9 @@
8787
tc-taas-es-processor | [2021-04-09T21:20:19.035Z] app INFO : Starting kafka consumer
8888
tc-taas-es-processor | 2021-04-09T21:20:21.292Z INFO no-kafka-client Joined group taas-es-processor generationId 1 as no-kafka-client-076538fc-60dd-4ca4-a2b9-520bdf73bc9e
8989
tc-taas-es-processor | 2021-04-09T21:20:21.293Z INFO no-kafka-client Elected as group leader
90+
tc-taas-es-processor | 2021-04-09T21:20:21.449Z DEBUG no-kafka-client Subscribed to taas.role.update:0 offset 0 leader kafka:9093
91+
tc-taas-es-processor | 2021-04-09T21:20:21.450Z DEBUG no-kafka-client Subscribed to taas.role.delete:0 offset 0 leader kafka:9093
92+
tc-taas-es-processor | 2021-04-09T21:20:21.451Z DEBUG no-kafka-client Subscribed to taas.role.requested:0 offset 0 leader kafka:9093
9093
tc-taas-es-processor | 2021-04-09T21:20:21.452Z DEBUG no-kafka-client Subscribed to taas.jobcandidate.create:0 offset 0 leader kafka:9093
9194
tc-taas-es-processor | 2021-04-09T21:20:21.455Z DEBUG no-kafka-client Subscribed to taas.job.create:0 offset 0 leader kafka:9093
9295
tc-taas-es-processor | 2021-04-09T21:20:21.456Z DEBUG no-kafka-client Subscribed to taas.resourcebooking.delete:0 offset 0 leader kafka:9093
@@ -103,7 +106,7 @@
103106
tc-taas-es-processor | 2021-04-09T21:20:21.473Z DEBUG no-kafka-client Subscribed to taas.job.update:0 offset 0 leader kafka:9093
104107
tc-taas-es-processor | 2021-04-09T21:20:21.474Z DEBUG no-kafka-client Subscribed to taas.resourcebooking.update:0 offset 0 leader kafka:9093
105108
tc-taas-es-processor | [2021-04-09T21:20:21.475Z] app INFO : Initialized.......
106-
tc-taas-es-processor | [2021-04-09T21:20:21.479Z] app INFO : taas.job.create,taas.job.update,taas.job.delete,taas.jobcandidate.create,taas.jobcandidate.update,taas.jobcandidate.delete,taas.resourcebooking.create,taas.resourcebooking.update,taas.resourcebooking.delete,taas.workperiod.create,taas.workperiod.update,taas.workperiod.delete,taas.workperiodpayment.create,taas.workperiodpayment.update,taas.workperiodpayment.delete
109+
tc-taas-es-processor | [2021-04-09T21:20:21.479Z] app INFO : taas.job.create,taas.job.update,taas.job.delete,taas.jobcandidate.create,taas.jobcandidate.update,taas.jobcandidate.delete,taas.resourcebooking.create,taas.resourcebooking.update,taas.resourcebooking.delete,taas.workperiod.create,taas.workperiod.update,taas.workperiod.delete,taas.workperiodpayment.create,taas.workperiodpayment.update,taas.interview.requested,taas.interview.update,taas.interview.bulkUpdate,taas.role.requested,taas.role.update,taas.role.delete
107110
tc-taas-es-processor | [2021-04-09T21:20:21.480Z] app INFO : Kick Start.......
108111
tc-taas-es-processor | ********** Topcoder Health Check DropIn listening on port 3001
109112
tc-taas-es-processor | Topcoder Health Check DropIn started and ready to roll
@@ -194,6 +197,7 @@ To be able to change and test `taas-es-processor` locally you can follow the nex
194197
| `npm run index:jobs <jobId>` | Indexes job data from db into ES, if jobId is not given all data is indexed. Use `-- --force` flag to skip confirmation |
195198
| `npm run index:job-candidates <jobCandidateId>` | Indexes job candidate data from db into ES, if jobCandidateId is not given all data is indexed. Use `-- --force` flag to skip confirmation |
196199
| `npm run index:resource-bookings <resourceBookingsId>` | Indexes resource bookings data from db into ES, if resourceBookingsId is not given all data is indexed. Use `-- --force` flag to skip confirmation |
200+
| `npm run index:roles <roleId>` | Indexes roles data from db into ES, if roleId is not given all data is indexed. Use `-- --force` flag to skip confirmation |
197201
| `npm run services:up` | Start services via docker-compose for local development. |
198202
| `npm run services:down` | Stop services via docker-compose for local development. |
199203
| `npm run services:logs -- -f <service_name>` | View logs of some service inside docker-compose. |
@@ -202,6 +206,7 @@ To be able to change and test `taas-es-processor` locally you can follow the nex
202206
| `npm run cov` | Code Coverage Report. |
203207
| `npm run migrate` | Run any migration files which haven't run yet. |
204208
| `npm run migrate:undo` | Revert most recent migration. |
209+
| `npm run demo-payment-scheduler` | Create 1000 Work Periods Payment records in with status "scheduled" and various "amount" |
205210
206211
## Import and Export data
207212

‎app-constants.js

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,13 @@ const Scopes = {
4949
READ_INTERVIEW: 'read:taas-interviews',
5050
CREATE_INTERVIEW: 'create:taas-interviews',
5151
UPDATE_INTERVIEW: 'update:taas-interviews',
52-
ALL_INTERVIEW: 'all:taas-interviews'
52+
ALL_INTERVIEW: 'all:taas-interviews',
53+
// role
54+
READ_ROLE: 'read:taas-roles',
55+
CREATE_ROLE: 'create:taas-roles',
56+
UPDATE_ROLE: 'update:taas-roles',
57+
DELETE_ROLE: 'delete:taas-roles',
58+
ALL_ROLE: 'all:taas-roles'
5359
}
5460

5561
// Interview related constants
@@ -76,6 +82,14 @@ const ChallengeStatus = {
7682
COMPLETED: 'Completed'
7783
}
7884

85+
const WorkPeriodPaymentStatus = {
86+
COMPLETED: 'completed',
87+
SCHEDULED: 'scheduled',
88+
IN_PROGRESS: 'in-progress',
89+
FAILED: 'failed',
90+
CANCELLED: 'cancelled'
91+
}
92+
7993
const PaymentProcessingSwitch = {
8094
ON: 'ON',
8195
OFF: 'OFF'
@@ -87,5 +101,6 @@ module.exports = {
87101
Scopes,
88102
Interviews,
89103
ChallengeStatus,
104+
WorkPeriodPaymentStatus,
90105
PaymentProcessingSwitch
91106
}

‎app.js

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ const schedule = require('node-schedule')
1313
const logger = require('./src/common/logger')
1414
const eventHandlers = require('./src/eventHandlers')
1515
const interviewService = require('./src/services/InterviewService')
16+
const { processScheduler } = require('./src/services/PaymentSchedulerService')
1617

1718
// setup express app
1819
const app = express()
@@ -97,6 +98,9 @@ const server = app.listen(app.get('port'), () => {
9798
eventHandlers.init()
9899
// schedule updateCompletedInterviews to run every hour
99100
schedule.scheduleJob('0 0 * * * *', interviewService.updateCompletedInterviews)
101+
102+
// schedule payment processing
103+
schedule.scheduleJob(config.PAYMENT_PROCESSING.CRON, processScheduler)
100104
})
101105

102106
if (process.env.NODE_ENV === 'test') {

‎config/default.js

Lines changed: 44 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ module.exports = {
4040

4141
TOPCODER_USERS_API: process.env.TOPCODER_USERS_API || 'https://api.topcoder-dev.com/v3/users',
4242
// the api to find topcoder members
43-
TOPCODER_MEMBERS_API: process.env.TOPCODER_MEMBERS_API || 'https://api.topcoder-dev.com/v3/members',
43+
TOPCODER_MEMBERS_API: process.env.TOPCODER_MEMBERS_API || 'https://api.topcoder-dev.com/v5/members',
4444
// rate limit of requests to user api
4545
MAX_PARALLEL_REQUEST_TOPCODER_USERS_API: process.env.MAX_PARALLEL_REQUEST_TOPCODER_USERS_API || 100,
4646

@@ -76,6 +76,8 @@ module.exports = {
7676
ES_INDEX_JOB_CANDIDATE: process.env.ES_INDEX_JOB_CANDIDATE || 'job_candidate',
7777
// the resource booking index
7878
ES_INDEX_RESOURCE_BOOKING: process.env.ES_INDEX_RESOURCE_BOOKING || 'resource_booking',
79+
// the role index
80+
ES_INDEX_ROLE: process.env.ES_INDEX_ROLE || 'role',
7981

8082
// the max bulk size in MB for ES indexing
8183
MAX_BULK_REQUEST_SIZE_MB: process.env.MAX_BULK_REQUEST_SIZE_MB || 20,
@@ -131,6 +133,13 @@ module.exports = {
131133
TAAS_INTERVIEW_UPDATE_TOPIC: process.env.TAAS_INTERVIEW_UPDATE_TOPIC || 'taas.interview.update',
132134
// the interview bulk update Kafka message topic
133135
TAAS_INTERVIEW_BULK_UPDATE_TOPIC: process.env.TAAS_INTERVIEW_BULK_UPDATE_TOPIC || 'taas.interview.bulkUpdate',
136+
// topics for role service
137+
// the create role entity Kafka message topic
138+
TAAS_ROLE_CREATE_TOPIC: process.env.TAAS_ROLE_CREATE_TOPIC || 'taas.role.requested',
139+
// the update role entity Kafka message topic
140+
TAAS_ROLE_UPDATE_TOPIC: process.env.TAAS_ROLE_UPDATE_TOPIC || 'taas.role.update',
141+
// the delete role entity Kafka message topic
142+
TAAS_ROLE_DELETE_TOPIC: process.env.TAAS_ROLE_DELETE_TOPIC || 'taas.role.delete',
134143

135144
// the Kafka message topic for sending email
136145
EMAIL_TOPIC: process.env.EMAIL_TOPIC || 'external.action.email',
@@ -162,5 +171,38 @@ module.exports = {
162171
DEFAULT_TIMELINE_TEMPLATE_ID: process.env.DEFAULT_TIMELINE_TEMPLATE_ID || '53a307ce-b4b3-4d6f-b9a1-3741a58f77e6',
163172
DEFAULT_TRACK_ID: process.env.DEFAULT_TRACK_ID || '9b6fc876-f4d9-4ccb-9dfd-419247628825',
164173

165-
PAYMENT_PROCESSING_SWITCH: process.env.PAYMENT_PROCESSING_SWITCH || 'OFF'
174+
PAYMENT_PROCESSING: {
175+
// switch off actual API calls in Payment Scheduler
176+
SWITCH: process.env.PAYMENT_PROCESSING_SWITCH || 'OFF',
177+
// the payment scheduler cron config
178+
CRON: process.env.PAYMENT_PROCESSING_CRON || '0 */5 * * * *',
179+
// the number of records processed by one time
180+
BATCH_SIZE: parseInt(process.env.PAYMENT_PROCESSING_BATCH_SIZE || 50),
181+
// in-progress expired to determine whether a record has been processed abnormally, moment duration format
182+
IN_PROGRESS_EXPIRED: process.env.IN_PROGRESS_EXPIRED || 'PT1H',
183+
// the number of max retry config
184+
MAX_RETRY_COUNT: parseInt(process.env.PAYMENT_PROCESSING_MAX_RETRY_COUNT || 10),
185+
// the time of retry base delay, unit: ms
186+
RETRY_BASE_DELAY: parseInt(process.env.PAYMENT_PROCESSING_RETRY_BASE_DELAY || 100),
187+
// the time of retry max delay, unit: ms
188+
RETRY_MAX_DELAY: parseInt(process.env.PAYMENT_PROCESSING_RETRY_MAX_DELAY || 10000),
189+
// the max time of one request, unit: ms
190+
PER_REQUEST_MAX_TIME: parseInt(process.env.PAYMENT_PROCESSING_PER_REQUEST_MAX_TIME || 30000),
191+
// the max time of one payment record, unit: ms
192+
PER_PAYMENT_MAX_TIME: parseInt(process.env.PAYMENT_PROCESSING_PER_PAYMENT_MAX_TIME || 60000),
193+
// the max records of payment of a minute
194+
PER_MINUTE_PAYMENT_MAX_COUNT: parseInt(process.env.PAYMENT_PROCESSING_PER_MINUTE_PAYMENT_MAX_COUNT || 12),
195+
// the max requests of challenge of a minute
196+
PER_MINUTE_CHALLENGE_REQUEST_MAX_COUNT: parseInt(process.env.PAYMENT_PROCESSING_PER_MINUTE_CHALLENGE_REQUEST_MAX_COUNT || 60),
197+
// the max requests of resource of a minute
198+
PER_MINUTE_RESOURCE_REQUEST_MAX_COUNT: parseInt(process.env.PAYMENT_PROCESSING_PER_MINUTE_CHALLENGE_REQUEST_MAX_COUNT || 20),
199+
// the default step fix delay, unit: ms
200+
FIX_DELAY_STEP: parseInt(process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP || 500),
201+
// the fix delay between step one and step two, unit: ms
202+
FIX_DELAY_STEP_1_2: parseInt(process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP_1_2 || process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP || 500),
203+
// the fix delay between step two and step three, unit: ms
204+
FIX_DELAY_STEP_2_3: parseInt(process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP_2_3 || process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP || 500),
205+
// the fix delay between step three and step four, unit: ms
206+
FIX_DELAY_STEP_3_4: parseInt(process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP_3_4 || process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP || 500)
207+
}
166208
}

0 commit comments

Comments
 (0)