Merge dev into feature/role-jd-parser2 - resolve conflict

Author: eisbilir

.circleci/config.yml

@@ -68,6 +68,7 @@ workflows:
             branches:
               only:
                 - dev
+                - change-validatations-in-job-jc
 
       # Production builds are exectuted only on tagged commits to the
       # master branch.

README.md

@@ -103,10 +103,11 @@
    tc-taas-es-processor | 2021-04-09T21:20:21.469Z DEBUG no-kafka-client Subscribed to taas.workperiodpayment.update:0 offset 0 leader kafka:9093
    tc-taas-es-processor | 2021-04-09T21:20:21.470Z DEBUG no-kafka-client Subscribed to taas.workperiodpayment.delete:0 offset 0 leader kafka:9093
    tc-taas-es-processor | 2021-04-09T21:20:21.471Z DEBUG no-kafka-client Subscribed to taas.workperiodpayment.create:0 offset 0 leader kafka:9093
+   tc-taas-es-processor | 2021-04-09T21:20:21.472Z DEBUG no-kafka-client Subscribed to taas.action.retry:0 offset 0 leader kafka:9093
    tc-taas-es-processor | 2021-04-09T21:20:21.473Z DEBUG no-kafka-client Subscribed to taas.job.update:0 offset 0 leader kafka:9093
    tc-taas-es-processor | 2021-04-09T21:20:21.474Z DEBUG no-kafka-client Subscribed to taas.resourcebooking.update:0 offset 0 leader kafka:9093
    tc-taas-es-processor | [2021-04-09T21:20:21.475Z] app INFO : Initialized.......
-   tc-taas-es-processor | [2021-04-09T21:20:21.479Z] app INFO : taas.job.create,taas.job.update,taas.job.delete,taas.jobcandidate.create,taas.jobcandidate.update,taas.jobcandidate.delete,taas.resourcebooking.create,taas.resourcebooking.update,taas.resourcebooking.delete,taas.workperiod.create,taas.workperiod.update,taas.workperiod.delete,taas.workperiodpayment.create,taas.workperiodpayment.update,taas.interview.requested,taas.interview.update,taas.interview.bulkUpdate,taas.role.requested,taas.role.update,taas.role.delete
+   tc-taas-es-processor | [2021-04-09T21:20:21.479Z] app INFO : common.error.reporting,taas.job.create,taas.job.update,taas.job.delete,taas.jobcandidate.create,taas.jobcandidate.update,taas.jobcandidate.delete,taas.resourcebooking.create,taas.resourcebooking.update,taas.resourcebooking.delete,taas.workperiod.create,taas.workperiod.update,taas.workperiod.delete,taas.workperiodpayment.create,taas.workperiodpayment.update,taas.interview.requested,taas.interview.update,taas.interview.bulkUpdate,taas.role.requested,taas.role.update,taas.role.delete,taas.action.retry
    tc-taas-es-processor | [2021-04-09T21:20:21.480Z] app INFO : Kick Start.......
    tc-taas-es-processor | ********** Topcoder Health Check DropIn listening on port 3001
    tc-taas-es-processor | Topcoder Health Check DropIn started and ready to roll
@@ -176,6 +177,19 @@ To be able to change and test `taas-es-processor` locally you can follow the nex
 2. Run `taas-es-processor` separately from the source code. As `npm run services:up` already run all the dependencies for both `taas-apis` and for `taas-es-processor`. The only thing you need to do for running `taas-es-processor` locally is clone the [taas-es-processor](https://github.com/topcoder-platform/taas-es-processor) repository and inside `taas-es-processor` folder run:
    - `nvm use` - to use correct Node version
    - `npm run install`
+   - Create `.env` file with the next environment variables. Values for **Auth0 config** should be shared with you on the forum.<br>
+
+      ```bash
+      # Auth0 config
+      AUTH0_URL=
+      AUTH0_AUDIENCE=
+      AUTH0_CLIENT_ID=
+      AUTH0_CLIENT_SECRET=
+      ```
+
+      - Values from this file would be automatically used by many `npm` commands.
+      - ⚠️ Never commit this file or its copy to the repository!
+
    - `npm run start`
 
 ## NPM Commands
@@ -206,6 +220,7 @@ To be able to change and test `taas-es-processor` locally you can follow the nex
 | `npm run cov`                                                                                                             | Code Coverage Report.                                                                                                                              |
 | `npm run migrate`                                                                                                         | Run any migration files which haven't run yet.                                                                                                     |
 | `npm run migrate:undo`                                                                                                    | Revert most recent migration.                                                                                                                      |
+| `npm run demo-payment-scheduler`                                                                                          | Create 1000 Work Periods Payment records in with status "scheduled" and various "amount"                                                                         |
 
 ## Import and Export data
 

app-constants.js

@@ -86,8 +86,24 @@ const ChallengeStatus = {
 
 const WorkPeriodPaymentStatus = {
   COMPLETED: 'completed',
-  CANCELLED: 'cancelled',
-  SCHEDULED: 'scheduled'
+  SCHEDULED: 'scheduled',
+  IN_PROGRESS: 'in-progress',
+  FAILED: 'failed',
+  CANCELLED: 'cancelled'
+}
+
+const PaymentProcessingSwitch = {
+  ON: 'ON',
+  OFF: 'OFF'
+}
+
+const PaymentSchedulerStatus = {
+  START_PROCESS: 'start-process',
+  CREATE_CHALLENGE: 'create-challenge',
+  ASSIGN_MEMBER: 'assign-member',
+  ACTIVATE_CHALLENGE: 'activate-challenge',
+  GET_USER_ID: 'get-userId',
+  CLOSE_CHALLENGE: 'close-challenge'
 }
 
 module.exports = {
@@ -96,5 +112,7 @@ module.exports = {
   Scopes,
   Interviews,
   ChallengeStatus,
-  WorkPeriodPaymentStatus
+  WorkPeriodPaymentStatus,
+  PaymentSchedulerStatus,
+  PaymentProcessingSwitch
 }

app.js

@@ -13,6 +13,7 @@ const schedule = require('node-schedule')
 const logger = require('./src/common/logger')
 const eventHandlers = require('./src/eventHandlers')
 const interviewService = require('./src/services/InterviewService')
+const { processScheduler } = require('./src/services/PaymentSchedulerService')
 
 // setup express app
 const app = express()
@@ -97,6 +98,9 @@ const server = app.listen(app.get('port'), () => {
   eventHandlers.init()
   // schedule updateCompletedInterviews to run every hour
   schedule.scheduleJob('0 0 * * * *', interviewService.updateCompletedInterviews)
+
+  // schedule payment processing
+  schedule.scheduleJob(config.PAYMENT_PROCESSING.CRON, processScheduler)
 })
 
 if (process.env.NODE_ENV === 'test') {

config/default.js

@@ -140,6 +140,8 @@ module.exports = {
   TAAS_ROLE_UPDATE_TOPIC: process.env.TAAS_ROLE_UPDATE_TOPIC || 'taas.role.update',
   // the delete role entity Kafka message topic
   TAAS_ROLE_DELETE_TOPIC: process.env.TAAS_ROLE_DELETE_TOPIC || 'taas.role.delete',
+  // special kafka topics
+  TAAS_ACTION_RETRY_TOPIC: process.env.TAAS_ACTION_RETRY_TOPIC || 'taas.action.retry',
 
   // the Kafka message topic for sending email
   EMAIL_TOPIC: process.env.EMAIL_TOPIC || 'external.action.email',
@@ -175,5 +177,40 @@ module.exports = {
   // member groups representing Wipro or TopCoder employee
   INTERNAL_MEMBER_GROUPS: process.env.INTERNAL_MEMBER_GROUPS || ['20000000', '20000001', '20000003', '20000010', '20000015'],
   // Topcoder skills cache time in minutes
-  TOPCODER_SKILLS_CACHE_TIME: process.env.TOPCODER_SKILLS_CACHE_TIME || 60
+  TOPCODER_SKILLS_CACHE_TIME: process.env.TOPCODER_SKILLS_CACHE_TIME || 60,
+  // payment scheduler config
+  PAYMENT_PROCESSING: {
+    // switch off actual API calls in Payment Scheduler
+    SWITCH: process.env.PAYMENT_PROCESSING_SWITCH || 'OFF',
+    // the payment scheduler cron config
+    CRON: process.env.PAYMENT_PROCESSING_CRON || '0 */5 * * * *',
+    // the number of records processed by one time
+    BATCH_SIZE: parseInt(process.env.PAYMENT_PROCESSING_BATCH_SIZE || 50),
+    // in-progress expired to determine whether a record has been processed abnormally, moment duration format
+    IN_PROGRESS_EXPIRED: process.env.IN_PROGRESS_EXPIRED || 'PT1H',
+    // the number of max retry config
+    MAX_RETRY_COUNT: parseInt(process.env.PAYMENT_PROCESSING_MAX_RETRY_COUNT || 10),
+    // the time of retry base delay, unit: ms
+    RETRY_BASE_DELAY: parseInt(process.env.PAYMENT_PROCESSING_RETRY_BASE_DELAY || 100),
+    // the time of retry max delay, unit: ms
+    RETRY_MAX_DELAY: parseInt(process.env.PAYMENT_PROCESSING_RETRY_MAX_DELAY || 10000),
+    // the max time of one request, unit: ms
+    PER_REQUEST_MAX_TIME: parseInt(process.env.PAYMENT_PROCESSING_PER_REQUEST_MAX_TIME || 30000),
+    // the max time of one payment record, unit: ms
+    PER_PAYMENT_MAX_TIME: parseInt(process.env.PAYMENT_PROCESSING_PER_PAYMENT_MAX_TIME || 60000),
+    // the max records of payment of a minute
+    PER_MINUTE_PAYMENT_MAX_COUNT: parseInt(process.env.PAYMENT_PROCESSING_PER_MINUTE_PAYMENT_MAX_COUNT || 12),
+    // the max requests of challenge of a minute
+    PER_MINUTE_CHALLENGE_REQUEST_MAX_COUNT: parseInt(process.env.PAYMENT_PROCESSING_PER_MINUTE_CHALLENGE_REQUEST_MAX_COUNT || 60),
+    // the max requests of resource of a minute
+    PER_MINUTE_RESOURCE_REQUEST_MAX_COUNT: parseInt(process.env.PAYMENT_PROCESSING_PER_MINUTE_CHALLENGE_REQUEST_MAX_COUNT || 20),
+    // the default step fix delay, unit: ms
+    FIX_DELAY_STEP: parseInt(process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP || 500),
+    // the fix delay after step of create challenge, unit: ms
+    FIX_DELAY_STEP_CREATE_CHALLENGE: parseInt(process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP_CREATE_CHALLENGE || process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP || 500),
+    // the fix delay after step of assign member, unit: ms
+    FIX_DELAY_STEP_ASSIGN_MEMBER: parseInt(process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP_ASSIGN_MEMBER || process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP || 500),
+    // the fix delay after step of activate challenge, unit: ms
+    FIX_DELAY_STEP_ACTIVATE_CHALLENGE: parseInt(process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP_ACTIVATE_CHALLENGE || process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP || 500)
+  }
 }

docs/swagger.yaml

@@ -174,7 +174,7 @@ paths:
           required: false
           schema:
             type: string
-            enum: ["hourly", "daily", "weekly", "monthly"]
+            enum: ["hourly", "daily", "weekly", "monthly","annual"]
           description: The rate type.
         - in: query
           name: status
@@ -1505,9 +1505,14 @@ paths:
           name: workPeriods.paymentStatus
           required: false
           schema:
-            type: string
-            enum: ["pending", "partially-completed", "completed", "cancelled"]
-          description: The payment status.
+            oneOf:
+              - type: array
+                items:
+                  type: string
+                  enum: ["pending", "partially-completed", "completed", "cancelled"]
+              - type: string
+                enum: ["pending", "partially-completed", "completed", "cancelled"]
+          description: comma separated payment status.
         - in: query
           name: workPeriods.startDate
           required: false
@@ -1956,9 +1961,14 @@ paths:
           name: paymentStatus
           required: false
           schema:
-            type: string
-            enum: ["pending", "partially-completed", "completed", "cancelled"]
-          description: The payment status.
+            oneOf:
+              - type: array
+                items:
+                  type: string
+                  enum: ["pending", "partially-completed", "completed", "cancelled"]
+              - type: string
+                enum: ["pending", "partially-completed", "completed", "cancelled"]
+          description: comma separated payment status.
         - in: query
           name: startDate
           required: false
@@ -2403,7 +2413,7 @@ paths:
           required: false
           schema:
             type: string
-            enum: ["completed", "scheduled", "cancelled"]
+            enum: ["completed", "scheduled", "in-progress", "failed", "cancelled"]
           description: The payment status.
       responses:
         "200":
@@ -4827,8 +4837,22 @@ components:
           description: "The amount to be paid."
         status:
           type: string
-          enum: ["completed", "scheduled", "cancelled"]
+          enum: ["completed", "scheduled", "in-progress", "failed", "cancelled"]
           description: "The payment status."
+        statusDetails:
+          type: object
+          properties:
+            errorMessage:
+              type: string
+            errorCode:
+              type: integer
+            retry:
+              type: integer
+            step:
+              type: string
+            challengeId:
+              type: string
+              format: uuid
         billingAccountId:
           type: integer
           example: 80000071
@@ -4885,7 +4909,7 @@ components:
           description: "The amount to be paid."
         status:
           type: string
-          enum: ["completed", "scheduled", "cancelled"]
+          enum: ["completed", "scheduled", "in-progress", "failed", "cancelled"]
           description: "The payment status."
     WorkPeriodPaymentCreateRequestBody:
       required:
@@ -4976,7 +5000,7 @@ components:
           description: "The amount to be paid."
         status:
           type: string
-          enum: ["completed", "scheduled", "cancelled"]
+          enum: ["completed", "scheduled", "in-progress", "failed", "cancelled"]
           description: "The payment status."
     CheckRun:
       type: object

local/kafka-client/topics.txt

@@ -20,3 +20,4 @@ taas.interview.requested
 taas.interview.update
 taas.interview.bulkUpdate
 external.action.email
+taas.action.retry

migrations/2021-05-29-create-payment-scheduler-table-add-status-details-to-payment.js

@@ -0,0 +1,111 @@
+'use strict';
+
+const config = require('config')
+const _ = require('lodash')
+const { PaymentSchedulerStatus } = require('../app-constants')
+
+/**
+ * Create `payment_schedulers` table & relations.
+ */
+module.exports = {
+  up: async (queryInterface, Sequelize) => {
+    const transaction = await queryInterface.sequelize.transaction()
+    try {
+      await queryInterface.createTable('payment_schedulers', {
+        id: {
+          type: Sequelize.UUID,
+          primaryKey: true,
+          allowNull: false,
+          defaultValue: Sequelize.UUIDV4
+        },
+        challengeId: {
+          field: 'challenge_id',
+          type: Sequelize.UUID,
+          allowNull: false
+        },
+        workPeriodPaymentId: {
+          field: 'work_period_payment_id',
+          type: Sequelize.UUID,
+          allowNull: false,
+          references: {
+            model: {
+              tableName: 'work_period_payments',
+              schema: config.DB_SCHEMA_NAME
+            },
+            key: 'id'
+          }
+        },
+        step: {
+          type: Sequelize.ENUM(_.values(PaymentSchedulerStatus)),
+          allowNull: false
+        },
+        status: {
+          type: Sequelize.ENUM(
+            'in-progress',
+            'completed',
+            'failed'
+          ),
+          allowNull: false
+        },
+        userId: {
+          field: 'user_id',
+          type: Sequelize.BIGINT
+        },
+        userHandle: {
+          field: 'user_handle',
+          type: Sequelize.STRING,
+          allowNull: false
+        },
+        createdAt: {
+          field: 'created_at',
+          type: Sequelize.DATE
+        },
+        updatedAt: {
+          field: 'updated_at',
+          type: Sequelize.DATE
+        },
+        deletedAt: {
+          field: 'deleted_at',
+          type: Sequelize.DATE
+        }
+      }, { schema: config.DB_SCHEMA_NAME, transaction })
+      await queryInterface.addColumn({ tableName: 'work_period_payments', schema: config.DB_SCHEMA_NAME }, 'status_details',
+        { type: Sequelize.JSONB },
+        { transaction })
+      await queryInterface.changeColumn({ tableName: 'work_period_payments', schema: config.DB_SCHEMA_NAME }, 'challenge_id',
+        { type: Sequelize.UUID },
+        { transaction })
+      await queryInterface.sequelize.query(`ALTER TYPE ${config.DB_SCHEMA_NAME}.enum_work_period_payments_status ADD VALUE 'in-progress'`)
+      await queryInterface.sequelize.query(`ALTER TYPE ${config.DB_SCHEMA_NAME}.enum_work_period_payments_status ADD VALUE 'failed'`)
+      await transaction.commit()
+    } catch (err) {
+      await transaction.rollback()
+      throw err
+    }
+  },
+
+  down: async (queryInterface, Sequelize) => {
+    const table = { schema: config.DB_SCHEMA_NAME, tableName: 'payment_schedulers' }
+    const statusTypeName = `${table.schema}.enum_${table.tableName}_status`
+    const stepTypeName = `${table.schema}.enum_${table.tableName}_step`
+    const transaction = await queryInterface.sequelize.transaction()
+    try {
+      await queryInterface.dropTable(table, { transaction })
+      // drop enum type for status and step column
+      await queryInterface.sequelize.query(`DROP TYPE ${statusTypeName}`, { transaction })
+      await queryInterface.sequelize.query(`DROP TYPE ${stepTypeName}`, { transaction })
+
+      await queryInterface.changeColumn({ tableName: 'work_period_payments', schema: config.DB_SCHEMA_NAME }, 'challenge_id',
+        { type: Sequelize.UUID, allowNull: false },
+        { transaction })
+      await queryInterface.removeColumn({ tableName: 'work_period_payments', schema: config.DB_SCHEMA_NAME }, 'status_details',
+        { transaction })
+      await queryInterface.sequelize.query(`DELETE FROM pg_enum WHERE enumlabel in ('in-progress', 'failed') AND enumtypid = (SELECT oid FROM pg_type WHERE typname = 'enum_work_period_payments_status')`,
+        { transaction })
+      await transaction.commit()
+    } catch (err) {
+      await transaction.rollback()
+      throw err
+    }
+  }
+};

package.json

@@ -27,6 +27,7 @@
     "local:init": "npm run local:reset && npm run data:import -- --force",
     "local:reset": "npm run delete-index -- --force || true && npm run create-index -- --force && npm run init-db force",
     "cov": "nyc --reporter=html --reporter=text npm run test",
+    "demo-payment-scheduler": "node scripts/demo-payment-scheduler/index.js && npm run index:all -- --force",
     "demo-payment": "node scripts/demo-payment"
   },
   "keywords": [],