chore: es-db-compare script code structure enhancements

- added verification scripts and guides to the repo
- moved README to the script folder
- README and Verification guide improvements
- ignore paths configured for production use

Author: maxceem

docs/es-db-compare.md

@@ -0,0 +1,47 @@
+# Script to find mismatches between data in DB and ES
+
+We keep all the data in two places in DB (Database) and in ES (Elasticsearch Index). Every time we make any changes to the data in the DB all the changes are also reflected in ElasticSearch. Due to some circumstances data in ES and DB can become inconsistent.
+
+This script may be run to find all the inconsistencies between data we have  in ES and DB and create a report.
+
+## Configuration
+
+The following properties can be set from env variables:
+
+- `PROJECT_START_ID`: if set, only projects with id that large than or equal to the value are compared.
+- `PROJECT_END_ID`: if set, only projects with id that less than or equal to the value are compared.
+- `PROJECT_LAST_ACTIVITY_AT`: if set, only projects with property lastActivityAt that large than or equal to the value are compared.
+
+There could be some fields that always mismatch in ES and DB.
+The variable named `ignoredPaths` at `scripts/es-db-compare/constants.js` maintains a list of json paths which will be ignored
+during the comparation. You may need to modify/add/delete items in the list.
+
+### Required
+
+- `PROJECT_START_ID` and `PROJECT_END_ID` must exist together.
+- At least one of `PROJECT_START_ID` with `PROJECT_END_ID` or `PROJECT_LAST_ACTIVITY_AT` needs be set before running the script.
+
+## Usage
+
+Set up configuration and execute command `npm run es-db-compare` on the command line.
+It will then generate a HTML report with name `report.html` under the current directory.
+
+Example commands:
+
+- Generate a report comparing ALL the projects:
+
+  ```bash
+  PROJECT_LAST_ACTIVITY_AT=0 npm run es-db-compare
+  ```
+
+- Generate a report comparing projects that have been updated on **26 December 2019** or later:
+
+  ```bash
+  PROJECT_LAST_ACTIVITY_AT="2019-12-26" npm run es-db-compare
+  ```
+
+- Generate a report comparing projects with ID range:
+
+  ```bash
+    PROJECT_START_ID=5000 PROJECT_END_ID=6000 npm run es-db-compare
+    ```

scripts/es-db-compare/README.md

@@ -11,14 +11,7 @@
 
 const lodash = require('lodash');
 const scriptUtil = require('./util');
-
-const associations = {
-  phases: 'Phase',
-  members: 'Member',
-  invites: 'Invite',
-  attachments: 'Attachment',
-  timelines: 'Timeline',
-};
+const scriptConstants = require('./constants');
 
 /**
  * Process diff delta to extract project-related data.
@@ -241,8 +234,10 @@ function processDelta(delta, dbData, esData, finalData) {
     }
   };
 
-  if (delta.path.length > 2 && associations[delta.path[1]]) {
-    return processAssociation(delta, { modelName: associations[delta.path[1]], refPath: delta.path[1] });
+  if (delta.path.length > 2 && scriptConstants.associations.projects[delta.path[1]]) {
+    return processAssociation(delta, {
+      modelName: scriptConstants.associations.projects[delta.path[1]], refPath: delta.path[1],
+    });
   }
   if (delta.dataType === 'array' && delta.path.length === 1) {
     if (delta.type === 'delete') {

scripts/es-db-compare/compareProjects.js

@@ -6,19 +6,16 @@ module.exports = {
   // currently support only a subset of jsonpath notations
   // "*" means any index number
   ignoredPaths: [
-    'project.projectUrl',
-    'project.utm',
-    'metadata.milestoneTemplates.order',
+    // all project updatedAt (good to set for local verification)
+    // 'project.updatedAt',
+    // 'project.phases[*].updatedAt',
+    // 'project.phases[*].products[*].updatedAt',
+    // 'project.phases[*].products[*].timeline.updatedAt',
+    // 'project.phases[*].products[*].timeline.milestones[*].updatedAt',
+    // 'project.invites[*].updatedAt',
+    // 'project.members[*].updatedAt',
+    // 'project.attachments[*].updatedAt',
 
-    // all project updatedAt
-    'project.updatedAt',
-    'project.phases[*].updatedAt',
-    'project.phases[*].products[*].updatedAt',
-    'project.phases[*].products[*].timeline.updatedAt',
-    'project.phases[*].products[*].timeline.milestones[*].updatedAt',
-    'project.invites[*].updatedAt',
-    'project.members[*].updatedAt',
-    'project.attachments[*].updatedAt',
     // all project deletedAt
     'project.deletedAt',
     'project.phases[*].deletedAt',
@@ -28,6 +25,7 @@ module.exports = {
     'project.invites[*].deletedAt',
     'project.members[*].deletedAt',
     'project.attachments[*].deletedAt',
+
     // all project deletedBy
     'project.deletedBy',
     'project.phases[*].deletedBy',
@@ -38,17 +36,18 @@ module.exports = {
     'project.members[*].deletedBy',
     'project.attachments[*].deletedBy',
 
-    // all metadata updatedAt
-    'metadata.projectTemplates.updatedAt',
-    'metadata.productTemplates.updatedAt',
-    'metadata.projectTypes.updatedAt',
-    'metadata.productCategories.updatedAt',
-    'metadata.milestoneTemplates.updatedAt',
-    'metadata.orgConfigs.updatedAt',
-    'metadata.forms.updatedAt',
-    'metadata.planConfigs.updatedAt',
-    'metadata.priceConfigs.updatedAt',
-    'metadata.buildingBlocks.updatedAt',
+    // all metadata updatedAt (good to set for local verification)
+    // 'metadata.projectTemplates.updatedAt',
+    // 'metadata.productTemplates.updatedAt',
+    // 'metadata.projectTypes.updatedAt',
+    // 'metadata.productCategories.updatedAt',
+    // 'metadata.milestoneTemplates.updatedAt',
+    // 'metadata.orgConfigs.updatedAt',
+    // 'metadata.forms.updatedAt',
+    // 'metadata.planConfigs.updatedAt',
+    // 'metadata.priceConfigs.updatedAt',
+    // 'metadata.buildingBlocks.updatedAt',
+
     // all metadata deletedAt
     'metadata.projectTemplates.deletedAt',
     'metadata.productTemplates.deletedAt',
@@ -60,6 +59,7 @@ module.exports = {
     'metadata.planConfigs.deletedAt',
     'metadata.priceConfigs.deletedAt',
     'metadata.buildingBlocks.deletedAt',
+
     // all metadata deletedBy
     'metadata.projectTemplates.deletedBy',
     'metadata.productTemplates.deletedBy',
@@ -85,5 +85,13 @@ module.exports = {
       priceConfigs: 'PriceConfig',
       buildingBlocks: 'BuildingBlock',
     },
+
+    projects: {
+      phases: 'Phase',
+      members: 'Member',
+      invites: 'Invite',
+      attachments: 'Attachment',
+      timelines: 'Timeline',
+    },
   },
 };

scripts/es-db-compare/constants.js

@@ -120,7 +120,7 @@ function getTemplate() {
     }
     return 'unknown';
   });
-  const template = handlebars.compile(fs.readFileSync(path.join(__dirname, 'report.mustache')).toString());
+  const template = handlebars.compile(fs.readFileSync(path.join(__dirname, 'report.handlebars')).toString());
   return template;
 }