Skip to content
This repository was archived by the owner on Jan 23, 2025. It is now read-only.

Commit e5eb8c9

Browse files
rootelementrootelement
committed
More migrations
1 parent 7d762a8 commit e5eb8c9

File tree

6 files changed

+156
-49
lines changed

6 files changed

+156
-49
lines changed

‎config/default.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ module.exports = {
3232
GROUPS_API_URL: process.env.GROUPS_API_URL || 'https://api.topcoder-dev.com/v5/groups',
3333
TERMS_API_URL: process.env.TERMS_API_URL || 'https://api.topcoder-dev.com/v5/terms',
3434
V4_CHALLENGE_API_URL: process.env.V4_CHALLENGE_API_URL || 'https://api.topcoder-dev.com/v4/challenges',
35+
V3_MEMBER_API_URL: process.env.V3_MEMBER_API_URL || 'https://api.topcoder-dev.com/v3/members',
3536
CREATED_DATE_BEGIN: process.env.CREATED_DATE_BEGIN,
3637
POPULATE_MIGRATION_TABLE_DATE_BEGIN: process.env.POPULATE_MIGRATION_TABLE_DATE_BEGIN || process.env.CREATED_DATE_BEGIN || new Date(),
3738

‎package-lock.json

Lines changed: 23 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,12 @@
88
"bluebird": "^3.5.5",
99
"config": "^3.1.0",
1010
"cors": "^2.8.5",
11+
"csv-parser": "^2.3.3",
1112
"dynamoose": "^1.11.1",
1213
"elasticsearch": "^16.1.1",
1314
"express": "^4.17.1",
1415
"express-interceptor": "^1.2.0",
16+
"fs": "0.0.1-security",
1517
"hashmap": "^2.4.0",
1618
"http-aws-es": "^6.0.0",
1719
"informix-wrapper": "git+https://github.com/appirio-tech/informix-wrapper.git#less-logs",

‎src/scripts/migrations/005-mmatches.js

Lines changed: 116 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,10 @@ const logger = require('../../util/logger')
1212
const challengeService = require('../../services/challengeService')
1313
const { getV4ESClient, getM2MToken } = require('../../util/helper')
1414
const translationService = require('../../services/translationService')
15-
// const resourceService = require('../../services/resourceService')
15+
// const csv = require('csv-parser')
16+
const fs = require('fs')
17+
const resourceService = require('../../services/resourceService')
18+
const { v5 } = require('uuid')
1619

1720
const memberHandleCache = new HashMap()
1821

@@ -23,6 +26,8 @@ const migrationFunction = {
2326
let page = 0
2427
let batch = 1
2528

29+
const challengeJson = { challenges: [] }
30+
2631
while (!finish) {
2732
logger.info(`Batch-${batch} - Loading challenges`)
2833
const challenges = await getMatchesFromES(page, perPage)
@@ -31,8 +36,23 @@ const migrationFunction = {
3136
// // logger.info(`Updating ${challenges}`)
3237
for (const challenge of challenges) {
3338
logger.debug(`Loading challenge ${challenge.id}`)
34-
// get challenge from v4 api
3539
const c = await challengeService.getMMatchFromV4API(challenge.id)
40+
if (!c) {
41+
logger.error(`Challenge Not Found - ID: ${challenge.id}, RoundID: ${challenge.roundId}`)
42+
continue
43+
}
44+
45+
const v5ChallengeLookup = await challengeService.getChallengeIDsFromV5({ legacyId: challenge.id }, 10)
46+
// logger.debug(JSON.stringify(v5ChallengeLookup))
47+
if (v5ChallengeLookup && v5ChallengeLookup.v5Ids && v5ChallengeLookup.v5Ids[0]) {
48+
logger.debug('Skipping!')
49+
continue
50+
51+
// for (const id of v5ChallengeLookup.v5Ids) {
52+
// logger.warn(`Deleting Entry for ${id}`)
53+
// await challengeService.deleteChallenge(id)
54+
// }
55+
}
3656

3757
const v5TrackProperties = translationService.convertV4TrackToV5(
3858
'DATA_SCIENCE',
@@ -90,48 +110,88 @@ const migrationFunction = {
90110
newChallenge.updated = challengeEndDate
91111

92112
let handlesToLookup = []
93-
for (const registrant of c.registrants) {
94-
// build cache
95-
if (!getMemberIdFromCache(registrant.handle)) {
96-
handlesToLookup.push(registrant.handle)
113+
if (c.registrants && c.registrants.length > 0) {
114+
for (const registrant of c.registrants) {
115+
// build cache
116+
if (!getMemberIdFromCache(registrant.handle)) {
117+
handlesToLookup.push(registrant.handle)
118+
} else {
119+
// logger.debug(`Handle Found in Cache ${registrant.handle}`)
120+
}
121+
if (handlesToLookup.length >= 15) {
122+
await cacheHandles(handlesToLookup)
123+
handlesToLookup = []
124+
}
97125
}
98-
if (handlesToLookup.length >= 25) {
99-
await cacheHandles(handlesToLookup)
100-
handlesToLookup = []
126+
await cacheHandles(handlesToLookup)
127+
128+
// csv
129+
// challenge id, member id, member handle, submission id, score
130+
const winners = _.map(c.winners, w => {
131+
return {
132+
handle: w.submitter,
133+
userId: getMemberIdFromCache(w.submitter),
134+
points: w.points,
135+
submissionId: w.submissionId
136+
}
137+
})
138+
139+
const sortedWinners = _.orderBy(winners, ['points'], ['desc'])
140+
141+
// console.log('Sorted Winners', JSON.stringify(sortedWinners))
142+
let placement = 1
143+
let counter = 1
144+
let lastPointsValue = null
145+
const calculatedWinners = []
146+
for (const winner of sortedWinners) {
147+
// calculate rank
148+
if (lastPointsValue && lastPointsValue > winner.points) {
149+
placement = counter
150+
}
151+
const calculatedWinner = {}
152+
calculatedWinner.handle = _.toString(winner.handle)
153+
calculatedWinner.userId = _.toString(winner.userId)
154+
calculatedWinner.placement = placement
155+
lastPointsValue = winner.points
156+
counter += 1
157+
158+
calculatedWinners.push(calculatedWinner)
101159
}
102-
}
103-
await cacheHandles(handlesToLookup)
160+
// console.log('Calculated Winners', JSON.stringify(calculatedWinners))
161+
162+
newChallenge.winners = calculatedWinners
163+
164+
const savedChallengeId = await challengeService.save(newChallenge)
165+
// const savedChallengeId = uuid() // FOR TESTING
166+
// logger.debug(`Challenge: ${JSON.stringify(c.submissions)}`)
104167

105-
const winners = _.map(c.winners, w => {
106-
return {
107-
handle: w.submitter
108-
// placement: w.rank // TODO :: missing placement?
109-
// TODO :: missing points as an object property
168+
const thisChallenge = {
169+
challengeId: challenge.id,
170+
submissions: c.submissions
110171
}
111-
})
112-
newChallenge.winners = winners
113-
114-
// const savedChallenge = await challengeService.save(newChallenge)
115-
const savedChallenge = { id: uuid() }
116-
logger.debug(`Challenge: ${JSON.stringify(newChallenge)}`)
117-
118-
for (const registrant of c.registrants) {
119-
const memberId = await getMemberIdFromCache(registrant.handle)
120-
const newResource = {
121-
// legacyId: resource.id,
122-
created: moment(registrant.registrationDate).utc().format(),
123-
createdBy: registrant.handle,
124-
updated: moment(registrant.registrationDate).utc().format(),
125-
updatedBy: registrant.handle,
126-
memberId: _.toString(memberId),
127-
memberHandle: registrant.handle,
128-
challengeId: savedChallenge.id,
129-
roleId: config.SUBMITTER_ROLE_ID
172+
challengeJson.challenges.push(thisChallenge)
173+
fs.writeFileSync('src/scripts/files/challenges.json', JSON.stringify(challengeJson))
174+
175+
for (const registrant of c.registrants) {
176+
const memberId = await getMemberIdFromCache(registrant.handle)
177+
const newResource = {
178+
// legacyId: resource.id,
179+
created: moment(registrant.registrationDate).utc().format(),
180+
createdBy: registrant.handle,
181+
updated: moment(registrant.registrationDate).utc().format(),
182+
updatedBy: registrant.handle,
183+
memberId: _.toString(memberId),
184+
memberHandle: registrant.handle,
185+
challengeId: savedChallengeId,
186+
roleId: config.SUBMITTER_ROLE_ID
187+
}
188+
await resourceService.saveResource(newResource)
189+
// logger.debug(`Resource: ${JSON.stringify(newResource)}`)
130190
}
131-
// await resourceService.save(newResource)
132-
// logger.debug(`Resource: ${JSON.stringify(newResource)}`)
191+
} else {
192+
logger.warn(`No Registrants for Challenge ${challenge.id}`)
133193
}
134-
return
194+
// return
135195
}
136196
} else {
137197
logger.info('Finished')
@@ -159,16 +219,29 @@ async function cacheHandles (handles) {
159219
logger.debug(`Caching ${handles.length} handles`)
160220
// curl --location --request GET 'https://api.topcoder-dev.com/v3/members/_search/?fields=userId%2Chandle%2CfirstName%2Cemail%2ClastName&query=handleLower:upbeat%20OR%20handleLower:tonyj'
161221
const ids = _.map(handles, h => `handleLower:${h}`)
162-
const query = ids.join('%20OR%20')
222+
const query = encodeURIComponent(escapeChars(ids.join('%20OR%20')))
163223
const token = await getM2MToken()
164-
const url = `https://api.topcoder-dev.com/v3/members/_search?fields=userId%2Chandle&query=${query}` // TODO COnfig
224+
const url = `${config.V3_MEMBER_API_URL}/_search?fields=userId%2Chandle&query=${query}`
165225
const res = await request.get(url).set({ Authorization: `Bearer ${token}` })
166226
const handleArray = _.get(res.body, 'result.content')
167227
for (const h of handleArray) {
168228
cacheMemberIdForHandle(h.handle, h.userId)
169229
}
170230
}
171231

232+
// + - && || ! ( ) { } [ ] ^ " ~ * ? : \
233+
function escapeChars (str) {
234+
str = str.replace(/]/g, '\\]')
235+
str = str.replace(/\[/g, '\\[')
236+
str = str.replace(/-/g, '\\-')
237+
str = str.replace(/{/g, '\\{')
238+
str = str.replace(/}/g, '\\}')
239+
str = str.replace(/\)/g, '\\)')
240+
str = str.replace(/\(/g, '\\(')
241+
str = str.replace(/\//g, '\\/')
242+
return str
243+
}
244+
172245
function convertPhases (v4PhasesArray) {
173246
let challengeEndDate = moment()
174247
let challengeStartDate = null
@@ -210,11 +283,12 @@ async function getMatchesFromES (page = 0, perPage = 10) {
210283
from: page * perPage,
211284
body: {
212285
query: {
213-
match_all: { }
286+
// match_phrase: { id: 16492 }
287+
match_all: {}
214288
}
215289
}
216290
}
217-
logger.debug(`ES Query ${JSON.stringify(esQuery)}`)
291+
// logger.debug(`ES Query ${JSON.stringify(esQuery)}`)
218292
// Search with constructed query
219293
let docs
220294
try {

‎src/services/challengeService.js

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -363,8 +363,8 @@ async function getChallengeIDsFromV5 (filter, perPage, page = 1) {
363363
// refresh: config.get('ES.ES_REFRESH'),
364364
size: perPage,
365365
from: perPage * (page - 1),
366-
_source: ['legacyId'],
367366
body: {
367+
_source: ['legacyId', 'id'],
368368
version: 'true',
369369
query: mustQuery.length > 0 ? {
370370
bool: {
@@ -386,7 +386,7 @@ async function getChallengeIDsFromV5 (filter, perPage, page = 1) {
386386
docs = await getESClient().search(esQuery)
387387
} catch (e) {
388388
// Catch error when the ES is fresh and has no data
389-
logger.error(e)
389+
// logger.error(`V5 Challenge IDs try/catch ${JSON.stringify(e)}`)
390390
docs = {
391391
hits: {
392392
total: 0,
@@ -396,7 +396,13 @@ async function getChallengeIDsFromV5 (filter, perPage, page = 1) {
396396
}
397397
// logger.warn(JSON.stringify(docs))
398398
// Extract data from hits
399-
if (docs.hits.total > 0) return { total: docs.hits.total, ids: _.map(docs.hits.hits, hit => _.toNumber(hit._source.legacyId)) }
399+
if (docs.hits.total > 0) {
400+
return {
401+
total: docs.hits.total,
402+
ids: _.map(docs.hits.hits, hit => _.toNumber(hit._source.legacyId)),
403+
v5Ids: _.map(docs.hits.hits, hit => hit._source.id)
404+
}
405+
}
400406
return false
401407
}
402408

@@ -804,15 +810,16 @@ async function getChallengeFromV5API (legacyId) {
804810
async function getMMatchFromV4API (legacyId) {
805811
const token = await getM2MToken()
806812
const url = `${config.V4_CHALLENGE_API_URL}/${legacyId}`
807-
console.log(url)
813+
// console.log(url)
808814
let res = null
809815
try {
810816
res = await axios.get(url, { headers: { Authorization: `Bearer ${token}` } })
811817
} catch (e) {
812-
logger.error(`Axios Error: ${JSON.stringify(e)}`)
818+
// logger.error(`Axios Error: ${JSON.stringify(e)}`)
819+
return false
813820
}
814821
// console.log(res.data)
815-
return res.data.result.content || null
822+
return res.data.result.content || false
816823
}
817824

818825
async function getChallengeSubmissionsFromV5API (challengeId, type) {

‎src/services/resourceService.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -220,7 +220,7 @@ async function getResourcesFromV5API (challengeId, roleId) {
220220
}
221221
let res = null
222222
try {
223-
// logger.debug(`Getting Resources from v5 ${challengeId} - ${roleId} url ${url}`)
223+
logger.debug(`Getting Resources from v5 ${challengeId} - ${roleId} url ${url}`)
224224
res = await axios.get(url, { headers: { Authorization: `Bearer ${token}` } })
225225
} catch (e) {
226226
logger.error(`get from v5 error ${JSON.stringify(e)}`)

0 commit comments

Comments
 (0)