Bulk milestone updates

config/default.js

@@ -35,5 +35,9 @@ module.exports = {
     ES_METADATA_INDEX: process.env.ES_METADATA_INDEX || 'metadata',
     ES_TYPE: process.env.ES_TYPE || 'doc', // ES 6.x accepts only 1 Type per index and it's mandatory to define it
     ES_METADATA_DEFAULT_ID: process.env.ES_METADATA_DEFAULT_ID || 1 // use for setting default id of metadata
-  }
+  },
+
+  // configuration for the stress test, see `test/stress/README.md`
+  STRESS_BASIC_QTY: process.env.STRESS_BASIC_QTY || 100,
+  STRESS_TESTER_TIMEOUT: process.env.STRESS_TESTER_TIMEOUT || 80
 }

package.json

@@ -10,7 +10,8 @@
     "sync:es": "node migrations/elasticsearch_sync.js",
     "view-data": "node test/common/view-data.js",
     "test": "NODE_ENV=test npm run sync:es && mocha test/e2e/*.test.js --timeout 30000 --exit && NODE_ENV=test npm run sync:es",
-    "test:cov": "nyc --reporter=html --reporter=text npm test"
+    "test:cov": "nyc --reporter=html --reporter=text npm test",
+    "test:stress": "NODE_ENV=test npm run sync:es && NODE_ENV=test node test/stress/doTest.js"
   },
   "author": "TCSCODER",
   "license": "none",
@@ -29,6 +30,7 @@
     "http-aws-es": "^6.0.0",
     "joi": "^14.3.1",
     "lodash": "^4.17.11",
+    "moment": "^2.24.0",
     "no-kafka": "^3.4.3",
     "tc-core-library-js": "github:appirio-tech/tc-core-library-js#v2.6",
     "topcoder-healthcheck-dropin": "^1.0.3",

src/services/ProcessorServiceMilestone.js

@@ -88,14 +88,8 @@ async function create (message) {
     return _.assign(doc._source, { milestones })
   }
 
-  // NOTE Disable indexing milestones when create at the moment, as it's now being indexed inside Project Service.
-  //      It's because adding a milestones may cause cascading updates of other milestones and in such cases we are doing
-  //      one ES index call instead of multiple calls. Otherwise ES may fail with error `version conflict`.
-  //      This would be turned on back, as soon as we get rid of such cascading updates inside Project Service.
-  //
-  // await helper.updateTimelineESPromise(message.timelineId, updateDocPromise)
-  // logger.debug(`Milestone created successfully in elasticsearch index, (milestoneId: ${message.id})`)
-  logger.debug(`TEMPORARY SKIPPED: Milestone created successfully in elasticsearch index, (milestoneId: ${message.id})`)
+  await helper.updateTimelineESPromise(message.timelineId, updateDocPromise)
+  logger.debug(`Milestone created successfully in elasticsearch index, (milestoneId: ${message.id})`)
 }
 
 create.schema = {
@@ -119,14 +113,8 @@ async function update (message) {
     return _.assign(doc._source, { milestones })
   }
 
-  // NOTE Disable indexing milestones when update at the moment, as it's now being indexed inside Project Service.
-  //      It's because updating a milestones may cause cascading updates of other milestones and in such cases we are doing
-  //      one ES index call instead of multiple calls. Otherwise ES may fail with error `version conflict`.
-  //      This would be turned on back, as soon as we get rid of such cascading updates inside Project Service.
-  //
-  // await helper.updateTimelineESPromise(message.timelineId, updateDocPromise)
-  // logger.debug(`Milestone updated successfully in elasticsearch index, (milestoneId: ${message.id})`)
-  logger.debug(`TEMPORARY SKIPPED: Milestone updated successfully in elasticsearch index, (milestoneId: ${message.id})`)
+  await helper.updateTimelineESPromise(message.timelineId, updateDocPromise)
+  logger.debug(`Milestone updated successfully in elasticsearch index, (milestoneId: ${message.id})`)
 }
 
 update.schema = {

test/e2e/processor.timeline.index.test.js

@@ -620,7 +620,7 @@ describe('TC Timeline And Nested Timeline Topic Tests', () => {
   })
 })
 
-xdescribe('TC Milestone Topic Tests', () => {
+describe('TC Milestone Topic Tests', () => {
   before(async () => {
     // runs before all tests in this block
     await ProcessorService.create(timelineCreatedMessage)

test/stress/README.md

@@ -0,0 +1,37 @@
+# Stress test
+
+## Overview
+
+This test is checking if any error would happen if we create, updated and delete milestones in **the same** timeline document in Elasticsearch `timelines` index at the same time. We are sending multiple Kafka events in parallel to trigger create, updated and delete milestone operations.
+
+1. First, the script should create initial data in the ES index:
+   - create one timeline in the `timeline` index with `2*STRESS_BASIC_QTY` milestones (it's important that all milestones belong to the same timeline)
+
+2. After initial data is created start an actual stress test by sending `3*STRESS_BASIC_QTY` Kafka messages in parallel:
+   - `STRESS_BASIC_QTY` Kafka messages to delete half of the initially created milestones
+   - `STRESS_BASIC_QTY` Kafka messages to update another half of initially created milestones
+   - `STRESS_BASIC_QTY` Kafka messages to create `STRESS_BASIC_QTY` new milestones in the same timeline
+
+## Configuration
+
+* `STRESS_BASIC_QTY`: The basic number of objects to use in stress test.
+
+* `STRESS_TESTER_TIMEOUT`: Number of seconds to wait after queueing create/update/delete requests and before validating data. Default is 80s, which is enough for `STRESS_BASIC_QTY=100`. This might have to be increased if `STRESS_BASIC_QTY` is higher than 100.
+
+## Run
+
+* Start processor
+
+  It should point the **test** ES, so set `NODE_ENV=test`.
+
+   ```
+   NODE_ENV=test npm start
+   ```
+
+* Run stress test
+
+  It would test using **test** ES, as this command sets `NODE_ENV=test`.
+
+  ```
+  npm run test:stress
+  ```