From 232edcb5beca5572ab0ded91a14493afb1843374 Mon Sep 17 00:00:00 2001 From: LieutenantRoger Date: Mon, 19 Sep 2022 22:51:43 +0800 Subject: [PATCH 1/6] fix-es-query --- .circleci/config.yml | 1 + src/services/challengeService.js | 18 +++++++++++++++--- src/services/syncService.js | 7 ++++--- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 005ff39..454f225 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -70,6 +70,7 @@ workflows: branches: only: - develop + - fix-es-query # Production builds are exectuted only on tagged commits to the # master branch. diff --git a/src/services/challengeService.js b/src/services/challengeService.js index ec066b2..a9da2fa 100644 --- a/src/services/challengeService.js +++ b/src/services/challengeService.js @@ -343,7 +343,7 @@ async function getChallengeIDsFromV4 (filter, perPage, page = 1) { * @param {Number} page * @returns {Object} { total, ids } */ -async function getChallengeIDsFromV5 (filter, perPage, page = 1) { +async function getChallengeIDsFromV5 (filter, perPage, lastDate) { // logger.warn(`getChallengeIDsFromV5 ${JSON.stringify(filter)} perPage ${perPage} page ${page}`) const boolQuery = [] const mustQuery = [] @@ -373,7 +373,6 @@ async function getChallengeIDsFromV5 (filter, perPage, page = 1) { type: config.get('ES.CHALLENGE_ES_TYPE'), // refresh: config.get('ES.ES_REFRESH'), size: perPage, - from: perPage * (page - 1), body: { _source: ['legacyId', 'id'], version: 'true', @@ -390,6 +389,9 @@ async function getChallengeIDsFromV5 (filter, perPage, page = 1) { ] } } + if (lastDate) { + esQuery.body.search_after = [Number(lastDate)] + } // Search with constructed query let docs // logger.warn(`V5 Challenge IDs Query ${JSON.stringify(esQuery)}`) @@ -408,10 +410,20 @@ async function getChallengeIDsFromV5 (filter, perPage, page = 1) { // logger.warn(JSON.stringify(docs)) // Extract data from hits if (docs.hits.total > 0) { + let newLastDate = null + const hits = docs.hits.hits + logger.info(`ES Search Result Length -> ${hits.length}`) + const endSortDate = docs.hits.hits[hits.length - 1].sort + if (endSortDate && endSortDate.length) { + logger.info(`LastSortDate: ${JSON.stringify(endSortDate)}`) + newLastDate = endSortDate[0] + } + logger.info(`newLastDate Timestamp: ${newLastDate}`) return { total: docs.hits.total, ids: _.map(docs.hits.hits, hit => _.toNumber(hit._source.legacyId)), - v5Ids: _.map(docs.hits.hits, hit => hit._source.id) + v5Ids: _.map(docs.hits.hits, hit => hit._source.id), + lastDate: newLastDate } } return false diff --git a/src/services/syncService.js b/src/services/syncService.js index 8d94454..6be57ca 100644 --- a/src/services/syncService.js +++ b/src/services/syncService.js @@ -213,7 +213,7 @@ async function getV4ChallengeIds (filter) { } async function getV5LegacyChallengeIds (filter) { - let page = 1 + let lastKey = null let running = true let v5Ids = [] const perPage = 1000 @@ -221,12 +221,13 @@ async function getV5LegacyChallengeIds (filter) { while (running) { // logger.debug(`V5 Challenge IDs - Getting ${page}`) - const { total, ids } = await challengeService.getChallengeIDsFromV5(filter, perPage, page) + const { total, ids, lastDate } = await challengeService.getChallengeIDsFromV5(filter, perPage, lastKey) + // Record the last call's end timestamp + lastKey = lastDate; if (ids && ids.length > 0) { // logger.warn(`IDs ${JSON.stringify(ids)}`) combinedTotal = total v5Ids = _.concat(v5Ids, ids) - page += 1 } else { running = false } From 136bdbbf1ccc271f615c3489542b8b2d60ef31e3 Mon Sep 17 00:00:00 2001 From: LieutenantRoger Date: Tue, 20 Sep 2022 17:06:38 +0800 Subject: [PATCH 2/6] switch sort key --- src/services/challengeService.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/services/challengeService.js b/src/services/challengeService.js index 8fde017..ef7deb9 100644 --- a/src/services/challengeService.js +++ b/src/services/challengeService.js @@ -384,9 +384,11 @@ async function getChallengeIDsFromV5 (filter, perPage, lastDate) { } : { match_all: {} }, - sort: [ - { updated: 'desc' } - ] + sort: [{ + created: { + order: 'desc' + } + }] } } if (lastDate) { From f4fa9a091382e3b37657a4a0e976931d40a07923 Mon Sep 17 00:00:00 2001 From: LieutenantRoger Date: Tue, 20 Sep 2022 17:21:09 +0800 Subject: [PATCH 3/6] refine code --- src/services/challengeService.js | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/services/challengeService.js b/src/services/challengeService.js index ef7deb9..25ef445 100644 --- a/src/services/challengeService.js +++ b/src/services/challengeService.js @@ -384,11 +384,9 @@ async function getChallengeIDsFromV5 (filter, perPage, lastDate) { } : { match_all: {} }, - sort: [{ - created: { - order: 'desc' - } - }] + sort: [ + { updated: 'desc' } + ] } } if (lastDate) { @@ -411,11 +409,11 @@ async function getChallengeIDsFromV5 (filter, perPage, lastDate) { } // logger.warn(JSON.stringify(docs)) // Extract data from hits - if (docs.hits.total > 0) { - let newLastDate = null - const hits = docs.hits.hits - logger.info(`ES Search Result Length -> ${hits.length}`) - const endSortDate = docs.hits.hits[hits.length - 1].sort + let result = _.map(docs.hits.hits, item => item._source) + let newLastDate = null + if (result.length > 0) { + logger.info(`ES Search Result Length -> ${result.length}`) + const endSortDate = docs.hits.hits[result.length - 1].sort if (endSortDate && endSortDate.length) { logger.info(`LastSortDate: ${JSON.stringify(endSortDate)}`) newLastDate = endSortDate[0] From 52cd47af183e60d014357790a008fb5f613bdfa1 Mon Sep 17 00:00:00 2001 From: LieutenantRoger Date: Tue, 20 Sep 2022 17:25:02 +0800 Subject: [PATCH 4/6] refine code v2 --- src/services/challengeService.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/services/challengeService.js b/src/services/challengeService.js index 25ef445..b96f822 100644 --- a/src/services/challengeService.js +++ b/src/services/challengeService.js @@ -410,6 +410,7 @@ async function getChallengeIDsFromV5 (filter, perPage, lastDate) { // logger.warn(JSON.stringify(docs)) // Extract data from hits let result = _.map(docs.hits.hits, item => item._source) + logger.info(`ES Search Hits Total -> ${docs.hits.total}`) let newLastDate = null if (result.length > 0) { logger.info(`ES Search Result Length -> ${result.length}`) @@ -426,7 +427,12 @@ async function getChallengeIDsFromV5 (filter, perPage, lastDate) { lastDate: newLastDate } } - return false + return { + total: docs.hits.total, + ids: [], + v5Ids: [], + lastDate: newLastDate + } } async function getChallengeListingFromV4ES (legacyId) { From 104b6793fcf077e72e8249c8f03dac3aa7252d6a Mon Sep 17 00:00:00 2001 From: LieutenantRoger Date: Wed, 21 Sep 2022 01:37:32 +0800 Subject: [PATCH 5/6] enable sync queue error log --- src/services/challengeSyncStatusService.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/services/challengeSyncStatusService.js b/src/services/challengeSyncStatusService.js index 40ff221..1c9dcf5 100644 --- a/src/services/challengeSyncStatusService.js +++ b/src/services/challengeSyncStatusService.js @@ -73,6 +73,7 @@ async function getSyncProgress (filter, perPage = 100, page = 1) { docs = await getESClient().search(esQuery) } catch (e) { // Catch error when the ES is fresh and has no data + logger.error(`Sync Queue Challenge IDs try/catch ${JSON.stringify(e)}`) docs = { hits: { total: 0, From fc1ebbe46b8f81b46887742cb5d76b5736d3af30 Mon Sep 17 00:00:00 2001 From: LieutenantRoger Date: Fri, 30 Sep 2022 20:18:37 +0800 Subject: [PATCH 6/6] reset ci --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 454f225..005ff39 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -70,7 +70,6 @@ workflows: branches: only: - develop - - fix-es-query # Production builds are exectuted only on tagged commits to the # master branch.