From abdba62b599435eca984f89f36e31b392d64c436 Mon Sep 17 00:00:00 2001 From: imcaizheng Date: Mon, 27 Jan 2020 20:35:33 +0800 Subject: [PATCH] initial commit --- scripts/es-db-compare/compareMetadata.js | 152 +++++++++++++ scripts/es-db-compare/compareProjects.js | 269 +++++++++++++++-------- scripts/es-db-compare/constants.js | 72 +++++- scripts/es-db-compare/index.js | 132 ++++++++++- scripts/es-db-compare/report.mustache | 83 ++++--- scripts/es-db-compare/util.js | 124 +++++++++-- 6 files changed, 660 insertions(+), 172 deletions(-) create mode 100644 scripts/es-db-compare/compareMetadata.js diff --git a/scripts/es-db-compare/compareMetadata.js b/scripts/es-db-compare/compareMetadata.js new file mode 100644 index 00000000..3a828efa --- /dev/null +++ b/scripts/es-db-compare/compareMetadata.js @@ -0,0 +1,152 @@ +/* eslint-disable no-console */ +/* eslint-disable consistent-return */ +/* eslint-disable no-restricted-syntax */ +/* eslint-disable no-param-reassign */ +/* + * Compare metadata between ES and DB. + */ +const lodash = require('lodash'); + +const scriptUtil = require('./util'); +const scriptConstants = require('./constants'); + +const hashKeyMapping = { + ProjectTemplate: 'id', + ProductTemplate: 'id', + ProjectType: 'key', + ProductCategory: 'key', + MilestoneTemplate: 'id', + OrgConfig: 'id', + Form: 'id', + PlanConfig: 'id', + PriceConfig: 'id', + BuildingBlock: 'id', +}; + +/** + * Process a single delta. + * + * @param {String} modelName the model name the delta belongs to + * @param {Object} delta the diff delta. + * @param {Object} dbData the data from DB + * @param {Object} esData the data from ES + * @param {Object} finalData the data patched + * @returns {undefined} + */ +function processDelta(modelName, delta, dbData, esData, finalData) { + const hashKey = hashKeyMapping[modelName]; + if (delta.dataType === 'array' && delta.path.length === 1) { + if (delta.type === 'delete') { + console.log(`one dbOnly found for ${modelName} with ${hashKey} ${delta.originalValue[hashKey]}`); + return { + type: 'dbOnly', + modelName, + hashKey, + hashValue: delta.originalValue[hashKey], + dbCopy: delta.originalValue, + }; + } + if (delta.type === 'add') { + console.log(`one esOnly found for ${modelName} with ${hashKey} ${delta.value[hashKey]}`); + return { + type: 'esOnly', + modelName, + hashKey, + hashValue: delta.value[hashKey], + esCopy: delta.value, + }; + } + } + if (['add', 'delete', 'modify'].includes(delta.type)) { + const path = scriptUtil.generateJSONPath(lodash.slice(delta.path, 1)); + const hashValue = lodash.get(finalData, lodash.slice(delta.path, 0, 1))[hashKey]; + const hashObject = lodash.set({}, hashKey, hashValue); + const dbCopy = lodash.find(dbData, hashObject); + const esCopy = lodash.find(esData, hashObject); + console.log(`one mismatch found for ${modelName} with ${hashKey} ${hashValue}`); + return { + type: 'mismatch', + kind: delta.type, + modelName, + hashKey, + hashValue, + path, + dbCopy, + esCopy, + }; + } +} + + +/** + * Compare Metadata data from ES and DB. + * + * @param {Object} dbData the data from DB + * @param {Object} esData the data from ES + * @returns {Object} the data to feed handlebars template + */ +function compareMetadata(dbData, esData) { + const data = { + nestedModels: {}, + }; + + const countInconsistencies = () => { + lodash.set(data, 'meta.totalObjects', 0); + lodash.map(data.nestedModels, (model) => { + const counts = Object.keys(model.mismatches).length + model.dbOnly.length + model.esOnly.length; + lodash.set(model, 'meta.counts', counts); + data.meta.totalObjects += counts; + }); + }; + + const storeDelta = (modelName, delta) => { + if (lodash.isUndefined(data.nestedModels[modelName])) { + data.nestedModels[modelName] = { + mismatches: {}, + dbOnly: [], + esOnly: [], + }; + } + if (delta.type === 'mismatch') { + if (lodash.isUndefined(data.nestedModels[modelName].mismatches[delta.hashValue])) { + data.nestedModels[modelName].mismatches[delta.hashValue] = []; + } + data.nestedModels[modelName].mismatches[delta.hashValue].push(delta); + return; + } + if (delta.type === 'dbOnly') { + data.nestedModels[modelName].dbOnly.push(delta); + return; + } + if (delta.type === 'esOnly') { + data.nestedModels[modelName].esOnly.push(delta); + } + }; + + for (const refPath of Object.keys(scriptConstants.associations.metadata)) { + const modelName = scriptConstants.associations.metadata[refPath]; + const { deltas, finalData } = scriptUtil.diffData( + dbData[refPath], + esData[refPath], + { + hashKey: hashKeyMapping[modelName], + modelPathExprssions: lodash.set({}, modelName, '[*]'), + }, + ); + for (const delta of deltas) { + if (scriptUtil.isIgnoredPath(`metadata.${refPath}`, delta.path)) { + continue; // eslint-disable-line no-continue + } + const deltaWithCopy = processDelta(modelName, delta, dbData[refPath], esData[refPath], finalData); + if (deltaWithCopy) { + storeDelta(modelName, deltaWithCopy); + } + } + } + countInconsistencies(); + return data; +} + +module.exports = { + compareMetadata, +}; diff --git a/scripts/es-db-compare/compareProjects.js b/scripts/es-db-compare/compareProjects.js index 35377747..f42a6e1b 100644 --- a/scripts/es-db-compare/compareProjects.js +++ b/scripts/es-db-compare/compareProjects.js @@ -9,7 +9,6 @@ * modules to compare other models. */ -const Diff = require('jsondiffpatch'); const lodash = require('lodash'); const scriptUtil = require('./util'); @@ -18,62 +17,22 @@ const associations = { members: 'Member', invites: 'Invite', attachments: 'Attachment', + timelines: 'Timeline', }; -const differ = Diff.create({ - objectHash: obj => obj.id, -}); - -/** - * The json diff patch may contains deltas with same path, - * one is "added to array", the other is "deleted from array". - * In such case they can be combined and treated as "modified at an index in the array". - * - * @param {Array} deltas the data to be filtered - * @returns {Array} filtered data - */ -function processSamePath(deltas) { - const result = []; - const groups = lodash.groupBy(deltas, 'path'); - for (const value of Object.values(groups)) { - if (value.length === 1) { - result.push(value[0]); - continue; // eslint-disable-line no-continue - } - if (value.length === 2) { - result.push(Object.assign({ type: 'modify' }, lodash.omit(value[0], 'type'))); - continue; // eslint-disable-line no-continue - } - throw new Error('Internal Error'); - } - return result; -} - -/** - * Transform or filter deltas before any further proccess. - * - * @param {Array} deltas the data to be processed - * @returns {Array} the result - */ -function preProcessDeltas(deltas) { - return processSamePath( - scriptUtil.flatten(deltas), - ); -} - /** * Process diff delta to extract project-related data. * - * @param {Object} delta the diff delta. See `util.flatten()` - * @param {Object} esData the data from ES + * @param {Object} delta the diff delta. * @param {Object} dbData the data from DB + * @param {Object} esData the data from ES * @param {Object} finalData the data patched * @returns {Object} Object project diff delta in a specific data structure */ -function processDelta(delta, esData, dbData, finalData) { +function processDelta(delta, dbData, esData, finalData) { const processMissingObject = (item, option) => { if (item.type === 'delete') { - const projectId = lodash.get(dbData, lodash.slice(item.path, 0, 1)).id; + const projectId = lodash.get(finalData, lodash.slice(item.path, 0, 1)).id; console.log(`one dbOnly found for ${option.modelName} with id ${item.originalValue.id}`); return { type: 'dbOnly', @@ -84,7 +43,7 @@ function processDelta(delta, esData, dbData, finalData) { }; } if (item.type === 'add') { - const projectId = lodash.get(esData, lodash.slice(item.path, 0, 1)).id; + const projectId = lodash.get(finalData, lodash.slice(item.path, 0, 1)).id; console.log(`one esOnly found for ${option.modelName} with id ${item.value.id}`); return { type: 'esOnly', @@ -96,6 +55,112 @@ function processDelta(delta, esData, dbData, finalData) { } }; + const processMilestone = (item) => { + const subPath = lodash.slice(item.path, 7); + if (item.dataType === 'array' && subPath.length === 1) { + return processMissingObject(item, { modelName: 'Milestone' }); + } + if (['add', 'delete', 'modify'].includes(item.type)) { + const path = scriptUtil.generateJSONPath(lodash.slice(subPath, 1)); + const id = lodash.get(finalData, lodash.slice(item.path, 0, 8)).id; + const projectId = lodash.get(finalData, lodash.slice(item.path, 0, 1)).id; + const phaseId = lodash.get(finalData, lodash.slice(item.path, 0, 3)).id; + const productId = lodash.get(finalData, lodash.slice(item.path, 0, 5)).id; + const dbCopy = lodash.find( + lodash.find( + lodash.find( + lodash.find(dbData, { id: projectId }).phases, + { id: phaseId }, + ).products, + { id: productId }, + ).timeline.milestones, + { id }, + ); + const esCopy = lodash.find( + lodash.find( + lodash.find( + lodash.find(esData, { id: projectId }).phases, + { id: phaseId }, + ).products, + { id: productId }, + ).timeline.milestones, + { id }, + ); + console.log(`one mismatch found for Milestone with id ${id}`); + return { + type: 'mismatch', + kind: item.type, + dataType: item.dataType, + projectId, + id, + modelName: 'Milestone', + path, + dbCopy, + esCopy, + }; + } + }; + + const processTimeline = (item) => { + if (item.path.length === 6 && item.type === 'modify') { + if (lodash.isNil(item.originalValue)) { + console.log(`one esOnly found for Timeline with id ${item.currentValue.id}`); + return { + type: 'esOnly', + projectId: lodash.get(finalData, lodash.slice(item.path, 0, 1)).id, + modelName: 'Timeline', + id: item.currentValue.id, + esCopy: item.currentValue, + }; + } + if (lodash.isNil(item.currentValue)) { + console.log(`one dbOnly found for Timeline with id ${item.originalValue.id}`); + return { + type: 'dbOnly', + projectId: lodash.get(finalData, lodash.slice(item.path, 0, 1)).id, + modelName: 'Timeline', + id: item.originalValue.id, + dbCopy: item.originalValue, + }; + } + throw new Error('Internal Error'); + } + const subPath = lodash.slice(item.path, 4); + if (['add', 'delete', 'modify'].includes(item.type)) { + const path = scriptUtil.generateJSONPath(lodash.slice(subPath, 2)); + const id = lodash.get(finalData, lodash.slice(item.path, 0, 5)).timeline.id; + const projectId = lodash.get(finalData, lodash.slice(item.path, 0, 1)).id; + const phaseId = lodash.get(finalData, lodash.slice(item.path, 0, 3)).id; + const productId = lodash.get(finalData, lodash.slice(item.path, 0, 5)).id; + const dbCopy = lodash.find( + lodash.find( + lodash.find(dbData, { id: projectId }).phases, + { id: phaseId }, + ).products, + { id: productId }, + ).timeline; + const esCopy = lodash.find( + lodash.find( + lodash.find(esData, { id: projectId }).phases, + { id: phaseId }, + ).products, + { id: productId }, + ).timeline; + console.log(`one mismatch found for Timeline with id ${id}`); + return { + type: 'mismatch', + kind: item.type, + dataType: item.dataType, + projectId, + id, + modelName: 'Timeline', + path, + dbCopy, + esCopy, + }; + } + }; + const processProduct = (item) => { const subPath = lodash.slice(item.path, 4); if (item.dataType === 'array' && subPath.length === 1) { @@ -137,6 +202,12 @@ function processDelta(delta, esData, dbData, finalData) { const processAssociation = (item, option) => { if (item.path[1] === 'phases' && item.path[3] === 'products') { + if (item.path[5] === 'timeline') { + if (item.path[6] === 'milestones') { + return processMilestone(item); + } + return processTimeline(item); + } return processProduct(item); } const subPath = lodash.slice(item.path, 2); @@ -174,7 +245,24 @@ function processDelta(delta, esData, dbData, finalData) { return processAssociation(delta, { modelName: associations[delta.path[1]], refPath: delta.path[1] }); } if (delta.dataType === 'array' && delta.path.length === 1) { - return processMissingObject(delta, { modelName: 'Project' }); + if (delta.type === 'delete') { + console.log(`one dbOnly found for Project with id ${delta.originalValue.id}`); + return { + type: 'dbOnly', + modelName: 'Project', + id: delta.originalValue.id, + dbCopy: delta.originalValue, + }; + } + if (delta.type === 'add') { + console.log(`one esOnly found for Project with id ${delta.value.id}`); + return { + type: 'esOnly', + modelName: 'Project', + id: delta.value.id, + esCopy: delta.value, + }; + } } if (['add', 'delete', 'modify'].includes(delta.type)) { const path = scriptUtil.generateJSONPath(lodash.slice(delta.path, 1)); @@ -199,46 +287,36 @@ function processDelta(delta, esData, dbData, finalData) { /** * Compare Project data from ES and DB. * - * @param {Object} esData the data from ES * @param {Object} dbData the data from DB + * @param {Object} esData the data from ES * @returns {Object} the data to feed handlebars template */ -function compareProjects(esData, dbData) { +function compareProjects(dbData, esData) { const data = { - project: { - rootMismatch: {}, - esOnly: [], - dbOnly: [], - }, - meta: { - esCopies: [], - dbCopies: [], - counts: { - Project: 0, - }, - uniqueDeltas: [], - }, + rootMismatch: {}, + esOnly: [], + dbOnly: [], }; - const storeDelta = (root, delta) => { + const storeDelta = (delta) => { if (delta.modelName === 'Project') { if (delta.type === 'esOnly') { - data[root].esOnly.push(delta); + data.esOnly.push(delta); return; } if (delta.type === 'dbOnly') { - data[root].dbOnly.push(delta); + data.dbOnly.push(delta); return; } } - if (!data[root].rootMismatch[delta.projectId]) { - data[root].rootMismatch[delta.projectId] = { project: [], associations: {} }; + if (!data.rootMismatch[delta.projectId]) { + data.rootMismatch[delta.projectId] = { project: [], associations: {} }; } if (delta.modelName === 'Project') { - data[root].rootMismatch[delta.projectId].project.push(delta); + data.rootMismatch[delta.projectId].project.push(delta); return; } - const currentAssociations = data[root].rootMismatch[delta.projectId].associations; + const currentAssociations = data.rootMismatch[delta.projectId].associations; if (!Object.keys(currentAssociations).includes(delta.modelName)) { currentAssociations[delta.modelName] = { mismatches: {}, @@ -257,31 +335,18 @@ function compareProjects(esData, dbData) { currentAssociations[delta.modelName][delta.type].push(delta); }; - const collectDataCopies = (delta) => { - if (delta.dbCopy) { - if (!lodash.find(data.meta.dbCopies, lodash.pick(delta, ['modelName', 'id']))) { - data.meta.dbCopies.push(delta); - } - } - if (delta.esCopy) { - if (!lodash.find(data.meta.esCopies, lodash.pick(delta, ['modelName', 'id']))) { - data.meta.esCopies.push(delta); - } - } - }; - const countInconsistencies = () => { lodash.set( - data.project, + data, 'meta.totalObjects', - data.project.dbOnly.length + data.project.esOnly.length, + data.dbOnly.length + data.esOnly.length, ); lodash.set( - data.project, + data, 'meta.totalProjects', - Object.keys(data.project.rootMismatch).length + data.project.dbOnly.length + data.project.esOnly.length, + Object.keys(data.rootMismatch).length + data.dbOnly.length + data.esOnly.length, ); - lodash.map(data.project.rootMismatch, (value) => { + lodash.map(data.rootMismatch, (value) => { const currentValue = value; lodash.set(currentValue, 'meta.counts', currentValue.project.length ? 1 : 0); lodash.map(currentValue.associations, (subObject) => { @@ -292,21 +357,33 @@ function compareProjects(esData, dbData) { ); currentValue.meta.counts += subObject.meta.counts; }); - data.project.meta.totalObjects += currentValue.meta.counts; + data.meta.totalObjects += currentValue.meta.counts; }); }; - const result = differ.diff(dbData, esData); - const finalData = differ.patch(Diff.clone(dbData), result); - const flattenedResult = preProcessDeltas(result); - for (const item of flattenedResult) { + const { deltas, finalData } = scriptUtil.diffData( + dbData, + esData, + { + hashKey: 'id', + modelPathExprssions: { + Project: '[*]', + Phase: '[*].phases[*]', + Product: '[*].phases[*].products[*]', + Milestone: '[*].phases[*].products[*].timeline.milestones[*]', + Invite: '[*].invites[*]', + Member: '[*].members[*]', + Attachment: '[*].attachments[*]', + }, + }, + ); + for (const item of deltas) { if (scriptUtil.isIgnoredPath('project', item.path)) { continue; // eslint-disable-line no-continue } - const delta = processDelta(item, esData, dbData, finalData); + const delta = processDelta(item, dbData, esData, finalData); if (delta) { - collectDataCopies(delta); - storeDelta('project', delta); + storeDelta(delta); } } countInconsistencies(); diff --git a/scripts/es-db-compare/constants.js b/scripts/es-db-compare/constants.js index 36b51fbb..a8dbd734 100644 --- a/scripts/es-db-compare/constants.js +++ b/scripts/es-db-compare/constants.js @@ -8,26 +8,82 @@ module.exports = { ignoredPaths: [ 'project.projectUrl', 'project.utm', + 'metadata.milestoneTemplates.order', - 'project.deletedAt', - 'project.phases[*].deletedAt', - 'project.phases[*].products[*].deletedAt', - 'project.invites[*].deletedAt', - 'project.members[*].deletedAt', - 'project.attachments[*].deletedAt', - + // all project updatedAt 'project.updatedAt', 'project.phases[*].updatedAt', 'project.phases[*].products[*].updatedAt', + 'project.phases[*].products[*].timeline.updatedAt', + 'project.phases[*].products[*].timeline.milestones[*].updatedAt', 'project.invites[*].updatedAt', 'project.members[*].updatedAt', 'project.attachments[*].updatedAt', - + // all project deletedAt + 'project.deletedAt', + 'project.phases[*].deletedAt', + 'project.phases[*].products[*].deletedAt', + 'project.phases[*].products[*].timeline.deletedAt', + 'project.phases[*].products[*].timeline.milestones[*].deletedAt', + 'project.invites[*].deletedAt', + 'project.members[*].deletedAt', + 'project.attachments[*].deletedAt', + // all project deletedBy 'project.deletedBy', 'project.phases[*].deletedBy', 'project.phases[*].products[*].deletedBy', + 'project.phases[*].products[*].timeline.deletedBy', + 'project.phases[*].products[*].timeline.milestones[*].deletedBy', 'project.invites[*].deletedBy', 'project.members[*].deletedBy', 'project.attachments[*].deletedBy', + + // all metadata updatedAt + 'metadata.projectTemplates.updatedAt', + 'metadata.productTemplates.updatedAt', + 'metadata.projectTypes.updatedAt', + 'metadata.productCategories.updatedAt', + 'metadata.milestoneTemplates.updatedAt', + 'metadata.orgConfigs.updatedAt', + 'metadata.forms.updatedAt', + 'metadata.planConfigs.updatedAt', + 'metadata.priceConfigs.updatedAt', + 'metadata.buildingBlocks.updatedAt', + // all metadata deletedAt + 'metadata.projectTemplates.deletedAt', + 'metadata.productTemplates.deletedAt', + 'metadata.projectTypes.deletedAt', + 'metadata.productCategories.deletedAt', + 'metadata.milestoneTemplates.deletedAt', + 'metadata.orgConfigs.deletedAt', + 'metadata.forms.deletedAt', + 'metadata.planConfigs.deletedAt', + 'metadata.priceConfigs.deletedAt', + 'metadata.buildingBlocks.deletedAt', + // all metadata deletedBy + 'metadata.projectTemplates.deletedBy', + 'metadata.productTemplates.deletedBy', + 'metadata.projectTypes.deletedBy', + 'metadata.productCategories.deletedBy', + 'metadata.milestoneTemplates.deletedBy', + 'metadata.orgConfigs.deletedBy', + 'metadata.forms.deletedBy', + 'metadata.planConfigs.deletedBy', + 'metadata.priceConfigs.deletedBy', + 'metadata.buildingBlocks.deletedBy', ], + associations: { + metadata: { + projectTemplates: 'ProjectTemplate', + productTemplates: 'ProductTemplate', + projectTypes: 'ProjectType', + productCategories: 'ProductCategory', + milestoneTemplates: 'MilestoneTemplate', + orgConfigs: 'OrgConfig', + forms: 'Form', + planConfigs: 'PlanConfig', + priceConfigs: 'PriceConfig', + buildingBlocks: 'BuildingBlock', + }, + }, }; diff --git a/scripts/es-db-compare/index.js b/scripts/es-db-compare/index.js index f74e152d..46bec184 100644 --- a/scripts/es-db-compare/index.js +++ b/scripts/es-db-compare/index.js @@ -1,4 +1,5 @@ /* eslint-disable no-console */ +/* eslint-disable no-param-reassign */ /* * Compare data between DB and ES and generate a report to be uploaded * to AWS S3. @@ -15,7 +16,9 @@ import { INVITE_STATUS } from '../../src/constants'; const handlebars = require('handlebars'); const path = require('path'); const fs = require('fs'); +const { compareMetadata } = require('./compareMetadata'); const { compareProjects } = require('./compareProjects'); +const scriptConstants = require('./constants'); const scriptConfig = { PROJECT_START_ID: process.env.PROJECT_START_ID, @@ -44,13 +47,17 @@ const es = util.getElasticSearchClient(); const ES_PROJECT_INDEX = config.get('elasticsearchConfig.indexName'); const ES_PROJECT_TYPE = config.get('elasticsearchConfig.docType'); +const ES_METADATA_INDEX = config.get('elasticsearchConfig.metadataIndexName'); +const ES_METADATA_TYPE = config.get('elasticsearchConfig.metadataDocType'); +const ES_TIMELINE_INDEX = config.get('elasticsearchConfig.timelineIndexName'); +const ES_TIMELINE_TYPE = config.get('elasticsearchConfig.timelineDocType'); /** * Get es search criteria. * * @returns {Object} the search criteria */ -function getESSearchCriteria() { +function getESSearchCriteriaForProject() { const filters = []; if (!lodash.isNil(scriptConfig.PROJECT_START_ID)) { filters.push({ @@ -118,12 +125,22 @@ function getTemplate() { } /** - * Get ES data. + * Get product timelines from ES. * * @returns {Promise} the ES data */ -async function getESData() { - const searchCriteria = getESSearchCriteria(); +async function getProductTimelinesFromES() { + const searchCriteria = { + index: ES_TIMELINE_INDEX, + type: ES_TIMELINE_TYPE, + body: { + query: { + match_phrase: { + reference: 'product', + }, + }, + }, + }; return es.search(searchCriteria) .then((docs) => { const rows = lodash.map(docs.hits.hits, single => single._source); // eslint-disable-line no-underscore-dangle @@ -132,17 +149,65 @@ async function getESData() { } /** - * Get DB data. + * Get projects from ES. + * + * @returns {Promise} the ES data + */ +async function getProjectsFromES() { + const searchCriteria = getESSearchCriteriaForProject(); + const projects = await es.search(searchCriteria) + .then((docs) => { + const rows = lodash.map(docs.hits.hits, single => single._source); // eslint-disable-line no-underscore-dangle + return rows; + }); + const timelines = await getProductTimelinesFromES(); + const timelinesGroup = lodash.groupBy(timelines, 'referenceId'); + lodash.map(projects, (project) => { + lodash.map(project.phases, (phase) => { + lodash.map(phase.products, (product) => { + product.timeline = lodash.get(timelinesGroup, [product.id, '0']) || null; + }); + }); + }); + return projects; +} + +/** + * Get metadata from ES. + * + * @returns {Promise} the ES data + */ +async function getMetadataFromES() { + const searchCriteria = { + index: ES_METADATA_INDEX, + type: ES_METADATA_TYPE, + }; + return es.search(searchCriteria) + .then((docs) => { + const rows = lodash.map(docs.hits.hits, single => single._source); // eslint-disable-line no-underscore-dangle + if (!rows.length) { + return lodash.reduce( + Object.keys(scriptConstants.associations.metadata), + (result, modleName) => { result[modleName] = []; }, + {}, + ); + } + return rows[0]; + }); +} + +/** + * Get projects from DB. * * @returns {Promise} the DB data */ -async function getDBData() { +async function getProjectsFromDB() { const filter = {}; if (!lodash.isNil(scriptConfig.PROJECT_START_ID)) { filter.id = { $between: [scriptConfig.PROJECT_START_ID, scriptConfig.PROJECT_END_ID] }; } if (!lodash.isNil(scriptConfig.PROJECT_LAST_ACTIVITY_AT)) { - filter.lastActivityAt = { $gte: scriptConfig.PROJECT_LAST_ACTIVITY_AT }; + filter.lastActivityAt = { $gte: new Date(scriptConfig.PROJECT_LAST_ACTIVITY_AT).toISOString() }; } return models.Project.findAll({ where: filter, @@ -172,24 +237,67 @@ async function getDBData() { return models.ProjectMember.getActiveProjectMembers(project.id) .then((currentProjectMembers) => { project.members = currentProjectMembers; - return project; + }).then(() => { + const promises = []; + lodash.map(project.phases, (phase) => { + lodash.map(phase.products, (product) => { + promises.push( + models.Timeline.findOne({ + where: { + reference: 'product', + referenceId: product.id, + }, + include: [{ + model: models.Milestone, + as: 'milestones', + }], + }).then((timeline) => { + product.timeline = timeline || null; + }), + ); + }); + }); + return Promise.all(promises) + .then(() => project); }); }); return Promise.all(projects); }).then(projects => JSON.parse(JSON.stringify(projects))); } +/** + * Get metadata from DB. + * + * @returns {Promise} the DB data + */ +async function getMetadataFromDB() { + const metadataAssociations = scriptConstants.associations.metadata; + const results = await Promise.all(lodash.map( + Object.values(metadataAssociations), + modelName => models[modelName].findAll(), + )); + return lodash.zipObject(Object.keys(metadataAssociations), JSON.parse(JSON.stringify(results))); +} + /** * Main function. * * @returns {Promise} void */ async function main() { - const esData = await getESData(); - const dbData = await getDBData(); + console.log('Processing Project...'); + const projectsFromDB = await getProjectsFromDB(); + const projectsFromES = await getProjectsFromES(); + const dataForProject = compareProjects(projectsFromDB, projectsFromES); + console.log('Processing Metadata...'); + const metadataFromDB = await getMetadataFromDB(); + const metadataFromES = await getMetadataFromES(); + const dataForMetadata = compareMetadata(metadataFromDB, metadataFromES); const template = getTemplate(); - const data = compareProjects(esData, dbData); - const report = template(data); + const report = template({ + metadata: dataForMetadata, + project: dataForProject, + }); fs.writeFileSync(reportPathname, report); console.log(`report is written to ${reportPathname}`); } diff --git a/scripts/es-db-compare/report.mustache b/scripts/es-db-compare/report.mustache index a69fa3cf..2ecd5ae9 100644 --- a/scripts/es-db-compare/report.mustache +++ b/scripts/es-db-compare/report.mustache @@ -1,21 +1,17 @@ Topcoder Project Service - ES/DB Comparison Report