Skip to content
This repository was archived by the owner on Mar 13, 2025. It is now read-only.

Commit 2bf09e4

Browse files
callmekatootiecallmekatootie
authored
Merge pull request #54 from topcoder-platform/enrich-2
Updates to the winning submission for enrich processor
2 parents 211f395 + 8ac769b commit 2bf09e4

File tree

7 files changed

+514
-106
lines changed

7 files changed

+514
-106
lines changed

‎README.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,9 @@ Configuration for the application is at config/default.js and config/production.
3737
- UBAHN_AGGREGATE_TOPIC: Kafka topic that is used to combine all create, update and delete message(s)
3838
- ES.HOST: Elasticsearch host
3939
- ES.DOCUMENTS: Elasticsearch index, type and id mapping for resources.
40+
- ATTRIBUTE_GROUP_PIPELINE_ID: The pipeline id for enrichment with attribute group. Default is `attributegroup-pipeline`
41+
- SKILL_PROVIDER_PIPELINE_ID: The pipeline id for enrichment with skill provider. Default is `skillprovider-pipeline`
42+
- USER_PIPELINE_ID: The pipeline id for enrichment of user details. Default is `user-pipeline`
4043

4144
For `ES.DOCUMENTS` configuration, you will find multiple other configurations below it. Each has default values that you can override using the environment variables
4245

‎config/default.js

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,6 @@ module.exports = {
5151
// ElasticSearch
5252
ES: {
5353
HOST: process.env.ES_HOST || 'http://localhost:9200',
54-
ENRICH_USER_PIPELINE_NAME: process.env.ENRICH_USER_PIPELINE_NAME || 'enrich_user',
5554
// es mapping: _index, _type, _id
5655
DOCUMENTS: {
5756
achievementprovider: {
@@ -64,7 +63,8 @@ module.exports = {
6463
},
6564
attributegroup: {
6665
index: process.env.ATTRIBUTE_GROUP_INDEX || 'attribute_group',
67-
type: '_doc'
66+
type: '_doc',
67+
pipelineId: process.env.ATTRIBUTE_GROUP_PIPELINE_ID || 'attributegroup-pipeline'
6868
},
6969
organization: {
7070
index: process.env.ORGANIZATION_INDEX || 'organization',
@@ -80,11 +80,13 @@ module.exports = {
8080
},
8181
skillprovider: {
8282
index: process.env.SKILL_PROVIDER_INDEX || 'skill_provider',
83-
type: '_doc'
83+
type: '_doc',
84+
pipelineId: process.env.SKILL_PROVIDER_PIPELINE_ID || 'skillprovider-pipeline'
8485
},
8586
user: {
8687
index: process.env.USER_INDEX || 'user',
87-
type: '_doc'
88+
type: '_doc',
89+
pipelineId: process.env.USER_PIPELINE_ID || 'user-pipeline'
8890
},
8991
// sub resources under user
9092
achievement: {

‎scripts/constants.js

Lines changed: 124 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -6,42 +6,136 @@
66
const config = require('config')
77

88
const topResources = {
9+
skillprovider: {
10+
index: config.get('ES.DOCUMENTS.skillprovider.index'),
11+
type: config.get('ES.DOCUMENTS.skillprovider.type'),
12+
enrich: {
13+
policyName: 'skillprovider-policy',
14+
matchField: 'id',
15+
enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy']
16+
},
17+
pipeline: {
18+
id: config.get('ES.DOCUMENTS.skillprovider.pipelineId'),
19+
field: 'skillProviderId',
20+
targetField: 'skillprovider',
21+
maxMatches: '1'
22+
}
23+
},
24+
25+
role: {
26+
index: config.get('ES.DOCUMENTS.role.index'),
27+
type: config.get('ES.DOCUMENTS.role.type'),
28+
enrich: {
29+
policyName: 'role-policy',
30+
matchField: 'id',
31+
enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy']
32+
}
33+
},
34+
935
achievementprovider: {
1036
index: config.get('ES.DOCUMENTS.achievementprovider.index'),
11-
enrichPolicy: 'achievementprovider-policy',
12-
type: config.get('ES.DOCUMENTS.achievementprovider.type')
13-
},
14-
attribute: {
15-
index: config.get('ES.DOCUMENTS.attribute.index'),
16-
enrichPolicy: 'attribute-policy',
17-
type: config.get('ES.DOCUMENTS.attribute.type')
37+
type: config.get('ES.DOCUMENTS.achievementprovider.type'),
38+
enrich: {
39+
policyName: 'achievementprovider-policy',
40+
matchField: 'id',
41+
enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy']
42+
}
1843
},
44+
1945
attributegroup: {
2046
index: config.get('ES.DOCUMENTS.attributegroup.index'),
21-
type: config.get('ES.DOCUMENTS.attributegroup.type')
22-
},
23-
organization: {
24-
index: config.get('ES.DOCUMENTS.organization.index'),
25-
enrichPolicy: 'organization-policy',
26-
type: config.get('ES.DOCUMENTS.organization.type')
27-
},
28-
role: {
29-
index: config.get('ES.DOCUMENTS.role.index'),
30-
enrichPolicy: 'role-policy',
31-
type: config.get('ES.DOCUMENTS.role.type')
47+
type: config.get('ES.DOCUMENTS.attributegroup.type'),
48+
enrich: {
49+
policyName: 'attributegroup-policy',
50+
matchField: 'id',
51+
enrichFields: ['id', 'name', 'organizationId', 'created', 'updated', 'createdBy', 'updatedBy']
52+
},
53+
pipeline: {
54+
id: config.get('ES.DOCUMENTS.attributegroup.pipelineId'),
55+
field: 'attributeGroupId',
56+
targetField: 'attributegroup',
57+
maxMatches: '1'
58+
}
3259
},
60+
3361
skill: {
3462
index: config.get('ES.DOCUMENTS.skill.index'),
35-
enrichPolicy: 'skill-policy',
36-
type: config.get('ES.DOCUMENTS.skill.type')
63+
type: config.get('ES.DOCUMENTS.skill.type'),
64+
enrich: {
65+
policyName: 'skill-policy',
66+
matchField: 'id',
67+
enrichFields: ['id', 'skillProviderId', 'name', 'externalId', 'uri', 'created', 'updated', 'createdBy', 'updatedBy', 'skillprovider']
68+
},
69+
ingest: {
70+
pipeline: {
71+
id: config.get('ES.DOCUMENTS.skillprovider.pipelineId')
72+
}
73+
}
3774
},
38-
skillprovider: {
39-
index: config.get('ES.DOCUMENTS.skillprovider.index'),
40-
type: config.get('ES.DOCUMENTS.skillprovider.type')
75+
76+
attribute: {
77+
index: config.get('ES.DOCUMENTS.attribute.index'),
78+
type: config.get('ES.DOCUMENTS.attribute.type'),
79+
enrich: {
80+
policyName: 'attribute-policy',
81+
matchField: 'id',
82+
enrichFields: ['id', 'name', 'attributeGroupId', 'created', 'updated', 'createdBy', 'updatedBy', 'attributegroup']
83+
},
84+
ingest: {
85+
pipeline: {
86+
id: config.get('ES.DOCUMENTS.attributegroup.pipelineId')
87+
}
88+
}
4189
},
90+
91+
organization: {
92+
index: config.get('ES.DOCUMENTS.organization.index'),
93+
type: config.get('ES.DOCUMENTS.organization.type')
94+
},
95+
4296
user: {
4397
index: config.get('ES.DOCUMENTS.user.index'),
44-
type: config.get('ES.DOCUMENTS.user.type')
98+
type: config.get('ES.DOCUMENTS.user.type'),
99+
pipeline: {
100+
id: config.get('ES.DOCUMENTS.user.pipelineId'),
101+
processors: [
102+
{
103+
referenceField: config.get('ES.DOCUMENTS.achievement.userField'),
104+
enrichPolicyName: 'achievementprovider-policy',
105+
field: '_ingest._value.achievementsProviderId',
106+
targetField: '_ingest._value.achievementprovider',
107+
maxMatches: '1'
108+
},
109+
{
110+
referenceField: config.get('ES.DOCUMENTS.externalprofile.userField'),
111+
enrichPolicyName: 'organization-policy',
112+
field: '_ingest._value.organizationId',
113+
targetField: '_ingest._value.organization',
114+
maxMatches: '1'
115+
},
116+
{
117+
referenceField: config.get('ES.DOCUMENTS.userattribute.userField'),
118+
enrichPolicyName: 'attribute-policy',
119+
field: '_ingest._value.attributeId',
120+
targetField: '_ingest._value.attribute',
121+
maxMatches: '1'
122+
},
123+
{
124+
referenceField: config.get('ES.DOCUMENTS.userrole.userField'),
125+
enrichPolicyName: 'role-policy',
126+
field: '_ingest._value.roleId',
127+
targetField: '_ingest._value.role',
128+
maxMatches: '1'
129+
},
130+
{
131+
referenceField: config.get('ES.DOCUMENTS.userskill.userField'),
132+
enrichPolicyName: 'skill-policy',
133+
field: '_ingest._value.skillId',
134+
targetField: '_ingest._value.skill',
135+
maxMatches: '1'
136+
}
137+
]
138+
}
45139
}
46140
}
47141

@@ -72,7 +166,12 @@ const userResources = {
72166
const organizationResources = {
73167
organizationskillprovider: {
74168
propertyName: config.get('ES.DOCUMENTS.organizationskillprovider.orgField'),
75-
relateKey: 'skillProviderId'
169+
relateKey: 'skillProviderId',
170+
enrich: {
171+
policyName: 'organization-policy',
172+
matchField: 'id',
173+
enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy', 'skillProviders']
174+
}
76175
}
77176
}
78177

‎scripts/db/dropAll.js

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,24 +4,63 @@
44
const _ = require('lodash')
55
const models = require('../../src/models')
66
const logger = require('../../src/common/logger')
7-
const { topResources, modelToESIndexMapping } = require('../constants')
7+
const {
8+
topResources,
9+
organizationResources,
10+
modelToESIndexMapping
11+
} = require('../constants')
812
const { getESClient } = require('../../src/common/es-client')
913

1014
async function main () {
1115
const client = getESClient()
16+
17+
try {
18+
logger.info('Deleting all pipelines...')
19+
await client.ingest.deletePipeline({
20+
id: topResources.user.pipeline.id
21+
})
22+
await client.ingest.deletePipeline({
23+
id: topResources.skillprovider.pipeline.id
24+
})
25+
await client.ingest.deletePipeline({
26+
id: topResources.attributegroup.pipeline.id
27+
})
28+
logger.info('Successfully deleted')
29+
} catch (e) {
30+
console.error(e)
31+
logger.warn('Delete all ingest pipelines failed')
32+
}
33+
1234
const keys = Object.keys(models)
1335
for (let i = 0; i < keys.length; i++) {
1436
const key = keys[i]
1537
if (models[key].tableName) {
1638
const esResourceName = modelToESIndexMapping[key]
1739
try {
1840
if (_.includes(_.keys(topResources), esResourceName)) {
41+
if (topResources[esResourceName].enrich) {
42+
logger.info(`Deleting enrich policy for ${esResourceName}`)
43+
await client.enrich.deletePolicy({
44+
name: topResources[esResourceName].enrich.policyName
45+
})
46+
logger.info(`Successfully deleted enrich policy for ${esResourceName}`)
47+
}
48+
logger.info(`Deleting index for ${esResourceName}`)
1949
await client.indices.delete({
2050
index: topResources[esResourceName].index
2151
})
52+
logger.info(`Successfully deleted enrich policy for ${esResourceName}`)
53+
} else if (_.includes(_.keys(organizationResources), esResourceName)) {
54+
logger.info('Deleting enrich policy for organization')
55+
await client.enrich.deletePolicy({
56+
name: organizationResources[esResourceName].enrich.policyName
57+
})
58+
logger.info('Successfully deleted enrich policy for organization')
2259
}
2360

61+
logger.info(`Deleting data in QLDB for ${esResourceName}`)
2462
await models.DBHelper.clear(models[key])
63+
logger.info(`Successfully deleted data in QLDB for ${esResourceName}`)
2564
} catch (e) {
2665
console.error(e)
2766
logger.warn(`drop table ${key} failed`)

0 commit comments

Comments
 (0)