From fc86113e5d5419aaea342eb736166e56e1bc207d Mon Sep 17 00:00:00 2001 From: LieutenantRoger Date: Thu, 5 Aug 2021 14:55:42 +0800 Subject: [PATCH 1/3] migration scripts updated --- scripts/withdrawn-migration/backup.js | 14 +++++++--- scripts/withdrawn-migration/migration.js | 35 +++++++++++++++--------- scripts/withdrawn-migration/restore.js | 35 +++++++++++++++--------- 3 files changed, 54 insertions(+), 30 deletions(-) diff --git a/scripts/withdrawn-migration/backup.js b/scripts/withdrawn-migration/backup.js index c737e143..927bc4c8 100644 --- a/scripts/withdrawn-migration/backup.js +++ b/scripts/withdrawn-migration/backup.js @@ -13,13 +13,17 @@ const currentStep = 'Backup' async function backup () { logger.info({ component: currentStep, message: '*************************** Backup process started ***************************' }) const filePath = path.join(__dirname, '/temp/') + if (fs.existsSync(filePath)) { + fs.rmdirSync(filePath, { recursive: true }) + fs.mkdirSync(filePath) + } const Op = Sequelize.Op const jobCandidates = await JobCandidate.findAll({ where: { status: 'placed' } }) - + let summary = 0 for (let i = 0; i < jobCandidates.length; i++) { const jc = jobCandidates[i] let job = null @@ -56,12 +60,12 @@ async function backup () { where: filter }) if (candidates && candidates.length > 0) { - fs.writeFile(filePath + 'jobcandidate-backup.json', JSON.stringify( + summary += candidates.length + fs.writeFile(filePath + `jobcandidate-backup-${jc.userId}.json`, JSON.stringify( candidates ), (err) => { if (!err) { - logger.info({ component: `${currentStep} Summary`, message: `Backup up finished. There are ${candidates.length} jobCandidates that need to be updated` }) - logger.info({ component: currentStep, message: '*************************** Backup process finished ***************************' }) + logger.info({ component: `${currentStep} Sub`, message: `There are ${candidates.length} jobCandidates that need to be updated for userId: ${jc.userId}` }) return } logger.error({ component: currentStep, message: err.message }) @@ -70,6 +74,8 @@ async function backup () { } } } + logger.info({ component: `${currentStep}`, message: `Report: there are ${summary} jobCandidates in total` }) + logger.info({ component: currentStep, message: '*************************** Backup process finished ***************************' }) } backup().then(() => { diff --git a/scripts/withdrawn-migration/migration.js b/scripts/withdrawn-migration/migration.js index ff923e6f..b9c90ed0 100644 --- a/scripts/withdrawn-migration/migration.js +++ b/scripts/withdrawn-migration/migration.js @@ -12,19 +12,28 @@ const currentStep = 'Migration' async function migration () { logger.info({ component: currentStep, message: '*************************** Migration process started ***************************' }) const filePath = path.join(__dirname, '/temp/') - const data = fs.readFileSync(filePath + 'jobCandidate-backup.json', 'utf-8') - const jobCandidates = JSON.parse(data) - let summary = 0 - for (var i = 0; i < jobCandidates.length; i++) { - const jc = await JobCandidate.findById(jobCandidates[i].id) - if (jc) { - const oldStatus = jc.status - const updated = await jc.update({ status: config.WITHDRAWN_STATUS_CHANGE_MAPPING[jobCandidates[i].status] }) - summary++ - logger.info({ component: currentStep, message: `jobCandidate with ${jc.id} status changed from ${oldStatus} to ${updated.status}` }) - } - }; - logger.info({ component: currentStep, message: `Totally updated ${summary} jobCandidates` }) + const files = [] + fs.readdirSync(filePath).forEach(async (file) => { + files.push(`${filePath}${file}`) + }) + let totalSum = 0 + for (let j = 0; j < files.length; j++) { + const data = fs.readFileSync(files[j], 'utf-8') + const jobCandidates = JSON.parse(data) + let summary = 0 + for (let i = 0; i < jobCandidates.length; i++) { + const jc = await JobCandidate.findById(jobCandidates[i].id) + if (jc) { + const oldStatus = jc.status + const updated = await jc.update({ status: config.WITHDRAWN_STATUS_CHANGE_MAPPING[jobCandidates[i].status] }) + summary++ + totalSum++ + logger.info({ component: currentStep, message: `jobCandidate with ${jc.id} status changed from ${oldStatus} to ${updated.status}` }) + } + }; + logger.info({ component: `${currentStep} Sub`, message: `Updated ${summary} jobCandidates from ${files[j]}` }) + } + logger.info({ component: currentStep, message: `Report: Totally Updated ${totalSum} jobCandidates` }) logger.info({ component: currentStep, message: '*************************** Migration process finished ***************************' }) } diff --git a/scripts/withdrawn-migration/restore.js b/scripts/withdrawn-migration/restore.js index 0150d75a..0f842a9f 100644 --- a/scripts/withdrawn-migration/restore.js +++ b/scripts/withdrawn-migration/restore.js @@ -11,19 +11,28 @@ const currentStep = 'Restore' async function restore () { logger.info({ component: currentStep, message: '*************************** Restore process started ***************************' }) const filePath = path.join(__dirname, '/temp/') - const data = fs.readFileSync(filePath + 'jobCandidate-backup.json', 'utf-8') - const jobCandidates = JSON.parse(data) - let summary = 0 - for (var i = 0; i < jobCandidates.length; i++) { - const jc = await JobCandidate.findById(jobCandidates[i].id) - if (jc) { - const oldStatus = jc.status - const updated = await jc.update({ status: jobCandidates[i].status }) - summary++ - logger.info({ component: currentStep, message: `jobCandidate with ${jc.id} status restored from ${oldStatus} to ${updated.status}` }) - } - }; - logger.info({ component: currentStep, message: `Totally restored ${summary} jobCandidates` }) + const files = [] + fs.readdirSync(filePath).forEach(async (file) => { + files.push(`${filePath}${file}`) + }) + let totalSum = 0 + for (let j = 0; j < files.length; j++) { + const data = fs.readFileSync(files[j], 'utf-8') + const jobCandidates = JSON.parse(data) + let summary = 0 + for (var i = 0; i < jobCandidates.length; i++) { + const jc = await JobCandidate.findById(jobCandidates[i].id) + if (jc) { + const oldStatus = jc.status + const updated = await jc.update({ status: jobCandidates[i].status }) + summary++ + totalSum++ + logger.info({ component: currentStep, message: `jobCandidate with ${jc.id} status restored from ${oldStatus} to ${updated.status}` }) + } + }; + logger.info({ component: `${currentStep} Sub`, message: `Restored ${summary} jobCandidates from ${files[j]}` }) + } + logger.info({ component: currentStep, message: `Report: Totally restored ${totalSum} jobCandidates` }) logger.info({ component: currentStep, message: '*************************** Restore process finished ***************************' }) } From 51bf20e40978d5080aae78cd1a0ec042bc54115c Mon Sep 17 00:00:00 2001 From: LieutenantRoger Date: Thu, 5 Aug 2021 15:13:07 +0800 Subject: [PATCH 2/3] folder creation --- config/default.js | 2 +- scripts/withdrawn-migration/backup.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/default.js b/config/default.js index b2a51bea..7f20c41d 100644 --- a/config/default.js +++ b/config/default.js @@ -189,7 +189,7 @@ module.exports = { SURVEY_COLLECTOR_PREFIX: process.env.WEEKLY_SURVEY_SURVEY_COLLECTOR_PREFIX || 'Week ending', SURVEY_MASTER_COLLECTOR_ID: process.env.WEEKLY_SURVEY_SURVEY_MASTER_COLLECTOR_ID || '', SURVEY_MASTER_MESSAGE_ID: process.env.WEEKLY_SURVEY_SURVEY_MASTER_MESSAGE_ID || '', - SURVEY_CONTACT_GROUP_ID: process.env.WEEKLY_SURVEY_SURVEY_CONTACT_GROUP_ID || '', + SURVEY_CONTACT_GROUP_ID: process.env.WEEKLY_SURVEY_SURVEY_CONTACT_GROUP_ID || '' }, // payment scheduler config PAYMENT_PROCESSING: { diff --git a/scripts/withdrawn-migration/backup.js b/scripts/withdrawn-migration/backup.js index 927bc4c8..70eb86e9 100644 --- a/scripts/withdrawn-migration/backup.js +++ b/scripts/withdrawn-migration/backup.js @@ -15,8 +15,8 @@ async function backup () { const filePath = path.join(__dirname, '/temp/') if (fs.existsSync(filePath)) { fs.rmdirSync(filePath, { recursive: true }) - fs.mkdirSync(filePath) } + fs.mkdirSync(filePath) const Op = Sequelize.Op const jobCandidates = await JobCandidate.findAll({ where: { From 8876cbb798b31809b69f018fa8fece00779d5698 Mon Sep 17 00:00:00 2001 From: LieutenantRoger Date: Thu, 5 Aug 2021 15:18:44 +0800 Subject: [PATCH 3/3] change naming conversion --- scripts/withdrawn-migration/backup.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/withdrawn-migration/backup.js b/scripts/withdrawn-migration/backup.js index 70eb86e9..3b3ea413 100644 --- a/scripts/withdrawn-migration/backup.js +++ b/scripts/withdrawn-migration/backup.js @@ -61,7 +61,7 @@ async function backup () { }) if (candidates && candidates.length > 0) { summary += candidates.length - fs.writeFile(filePath + `jobcandidate-backup-${jc.userId}.json`, JSON.stringify( + fs.writeFile(filePath + `jobcandidate-backup-${i+1}.json`, JSON.stringify( candidates ), (err) => { if (!err) {