diff --git a/README.md b/README.md index ac72430..049d748 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,13 @@ The following parameters can be set in config files or in env variables: - ELASTICCLOUD_ID: The elastic cloud id, if your elasticsearch instance is hosted on elastic cloud. DO NOT provide a value for ES_HOST if you are using this - ELASTICCLOUD_USERNAME: The elastic cloud username for basic authentication. Provide this only if your elasticsearch instance is hosted on elastic cloud - ELASTICCLOUD_PASSWORD: The elastic cloud password for basic authentication. Provide this only if your elasticsearch instance is hosted on elastic cloud +- AUTH0_URL: The auth0 url, Default is 'https://topcoder-dev.auth0.com/oauth/token' +- AUTH0_AUDIENCE: The auth0 audience for accessing ubahn api(s), Default is 'https://m2m.topcoder-dev.com/' +- AUTH0_CLIENT_ID: The auth0 client id +- AUTH0_CLIENT_SECRET: The auth0 client secret +- AUTH0_PROXY_SERVER_URL: The auth0 proxy server url +- TOKEN_CACHE_TIME: The token cache time +- TOPCODER_GROUP_API: The topcoder groups api, Default is 'https://api.topcoder-dev.com/v5/groups' There is a `/health` endpoint that checks for the health of the app. This sets up an expressjs server and listens on the environment variable `PORT`. It's not part of the configuration file and needs to be passed as an environment variable @@ -89,7 +96,7 @@ Configuration for the tests is at `config/test.js`, only add such new configurat docker-compose up -d ``` -3. initialize Elasticsearch, create configured Elasticsearch index: `npm run init-es force` +3. initialize Elasticsearch. Execute the `insert-data` script in the [API repository](https://github.com/topcoder-platform/u-bahn-api) to set it up and then clear only the data ## Local deployment diff --git a/config/default.js b/config/default.js index 9eee37d..3e698d7 100644 --- a/config/default.js +++ b/config/default.js @@ -14,6 +14,15 @@ module.exports = { // Kafka group id KAFKA_GROUP_ID: process.env.KAFKA_GROUP_ID || 'ubahn-processor-es', + TOPCODER_GROUP_API: process.env.TOPCODER_GROUP_API || 'https://api.topcoder-dev.com/v5/groups', + + AUTH0_URL: process.env.AUTH0_URL || 'https://topcoder-dev.auth0.com/oauth/token', // Auth0 credentials + AUTH0_AUDIENCE: process.env.AUTH0_AUDIENCE || 'https://m2m.topcoder-dev.com/', + TOKEN_CACHE_TIME: process.env.TOKEN_CACHE_TIME, + AUTH0_CLIENT_ID: process.env.AUTH0_CLIENT_ID, + AUTH0_CLIENT_SECRET: process.env.AUTH0_CLIENT_SECRET, + AUTH0_PROXY_SERVER_URL: process.env.AUTH0_PROXY_SERVER_URL, + UBAHN_CREATE_TOPIC: process.env.UBAHN_CREATE_TOPIC || 'u-bahn.action.create', UBAHN_UPDATE_TOPIC: process.env.UBAHN_UPDATE_TOPIC || 'u-bahn.action.update', UBAHN_DELETE_TOPIC: process.env.UBAHN_DELETE_TOPIC || 'u-bahn.action.delete', @@ -84,6 +93,26 @@ module.exports = { }, organization: { enrichPolicyName: process.env.ORGANIZATION_ENRICH_POLICYNAME || 'organization-policy' + }, + // sub resources under user + achievement: { + userField: process.env.USER_ACHIEVEMENT_PROPERTY_NAME || 'achievements' + }, + externalprofile: { + userField: process.env.USER_EXTERNALPROFILE_PROPERTY_NAME || 'externalProfiles' + }, + userattribute: { + userField: process.env.USER_ATTRIBUTE_PROPERTY_NAME || 'attributes' + }, + userrole: { + userField: process.env.USER_ROLE_PROPERTY_NAME || 'roles' + }, + userskill: { + userField: process.env.USER_SKILL_PROPERTY_NAME || 'skills' + }, + // sub resources under organization + organizationskillprovider: { + orgField: process.env.ORGANIZATION_SKILLPROVIDER_PROPERTY_NAME || 'skillProviders' } } } diff --git a/package.json b/package.json index 03dd99d..4aabcf7 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,6 @@ "start": "node src/app.js", "lint": "standard", "lint:fix": "standard --fix", - "init-es": "node test/common/init-es.js", "view-data": "node test/common/view-data.js", "test": "mocha test/unit/test.js --require test/unit/prepare.js --timeout 20000 --exit", "test:cov": "nyc --reporter=html --reporter=text mocha test/unit/test.js --require test/unit/prepare.js --timeout 20000 --exit", @@ -35,6 +34,8 @@ "get-parameter-names": "^0.3.0", "lodash": "^4.17.19", "no-kafka": "^3.4.3", + "axios": "^0.19.2", + "tc-core-library-js": "appirio-tech/tc-core-library-js.git#v2.6", "topcoder-healthcheck-dropin": "^1.0.3", "winston": "^3.2.1" }, diff --git a/src/common/constants.js b/src/common/constants.js index 1a6901e..d4adfb4 100644 --- a/src/common/constants.js +++ b/src/common/constants.js @@ -10,14 +10,18 @@ const topResources = { index: config.get('ES.ACHIEVEMENT_PROVIDER_INDEX'), type: config.get('ES.ACHIEVEMENT_PROVIDER_TYPE'), enrich: { - policyName: config.get('ES.ENRICHMENT.achievementprovider.enrichPolicyName') + policyName: config.get('ES.ENRICHMENT.achievementprovider.enrichPolicyName'), + matchField: 'id', + enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy'] } }, attribute: { index: config.get('ES.ATTRIBUTE_INDEX'), type: config.get('ES.ATTRIBUTE_TYPE'), enrich: { - policyName: config.get('ES.ENRICHMENT.attribute.enrichPolicyName') + policyName: config.get('ES.ENRICHMENT.attribute.enrichPolicyName'), + matchField: 'id', + enrichFields: ['id', 'name', 'attributeGroupId', 'created', 'updated', 'createdBy', 'updatedBy', 'attributegroup'] }, ingest: { pipeline: { @@ -29,7 +33,15 @@ const topResources = { index: config.get('ES.ATTRIBUTE_GROUP_INDEX'), type: config.get('ES.ATTRIBUTE_GROUP_TYPE'), enrich: { - policyName: config.get('ES.ENRICHMENT.attributegroup.enrichPolicyName') + policyName: config.get('ES.ENRICHMENT.attributegroup.enrichPolicyName'), + matchField: 'id', + enrichFields: ['id', 'name', 'organizationId', 'created', 'updated', 'createdBy', 'updatedBy'] + }, + pipeline: { + id: config.get('ES.ENRICHMENT.attributegroup.pipelineId'), + field: 'attributeGroupId', + targetField: 'attributegroup', + maxMatches: '1' } }, organization: { @@ -40,14 +52,18 @@ const topResources = { index: config.get('ES.ROLE_INDEX'), type: config.get('ES.ROLE_TYPE'), enrich: { - policyName: config.get('ES.ENRICHMENT.role.enrichPolicyName') + policyName: config.get('ES.ENRICHMENT.role.enrichPolicyName'), + matchField: 'id', + enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy'] } }, skill: { index: config.get('ES.SKILL_INDEX'), type: config.get('ES.SKILL_TYPE'), enrich: { - policyName: config.get('ES.ENRICHMENT.skill.enrichPolicyName') + policyName: config.get('ES.ENRICHMENT.skill.enrichPolicyName'), + matchField: 'id', + enrichFields: ['id', 'skillProviderId', 'name', 'externalId', 'uri', 'created', 'updated', 'createdBy', 'updatedBy', 'skillprovider'] }, ingest: { pipeline: { @@ -59,7 +75,15 @@ const topResources = { index: config.get('ES.SKILL_PROVIDER_INDEX'), type: config.get('ES.SKILL_PROVIDER_TYPE'), enrich: { - policyName: config.get('ES.ENRICHMENT.skillprovider.enrichPolicyName') + policyName: config.get('ES.ENRICHMENT.skillprovider.enrichPolicyName'), + matchField: 'id', + enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy'] + }, + pipeline: { + id: config.get('ES.ENRICHMENT.skillprovider.pipelineId'), + field: 'skillProviderId', + targetField: 'skillprovider', + maxMatches: '1' } }, user: { @@ -69,6 +93,39 @@ const topResources = { pipeline: { id: config.get('ES.ENRICHMENT.user.pipelineId') } + }, + pipeline: { + id: config.get('ES.ENRICHMENT.user.pipelineId'), + processors: [ + { + referenceField: config.get('ES.ENRICHMENT.achievement.userField'), + enrichPolicyName: config.get('ES.ENRICHMENT.achievementprovider.enrichPolicyName'), + field: '_ingest._value.achievementsProviderId', + targetField: '_ingest._value.achievementprovider', + maxMatches: '1' + }, + { + referenceField: config.get('ES.ENRICHMENT.userattribute.userField'), + enrichPolicyName: config.get('ES.ENRICHMENT.attribute.enrichPolicyName'), + field: '_ingest._value.attributeId', + targetField: '_ingest._value.attribute', + maxMatches: '1' + }, + { + referenceField: config.get('ES.ENRICHMENT.userrole.userField'), + enrichPolicyName: config.get('ES.ENRICHMENT.role.enrichPolicyName'), + field: '_ingest._value.roleId', + targetField: '_ingest._value.role', + maxMatches: '1' + }, + { + referenceField: config.get('ES.ENRICHMENT.userskill.userField'), + enrichPolicyName: config.get('ES.ENRICHMENT.skill.enrichPolicyName'), + field: '_ingest._value.skillId', + targetField: '_ingest._value.skill', + maxMatches: '1' + } + ] } } } @@ -106,7 +163,12 @@ const organizationResources = { organizationskillprovider: { propertyName: config.get('ES.ORGANIZATION_SKILLPROVIDER_PROPERTY_NAME'), relateKey: 'skillProviderId', - validate: payload => validProperties(payload, ['organizationId', 'skillProviderId']) + validate: payload => validProperties(payload, ['organizationId', 'skillProviderId']), + enrich: { + policyName: config.get('ES.ENRICHMENT.organization.enrichPolicyName'), + matchField: 'id', + enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy', 'skillProviders'] + } } } diff --git a/src/common/helper.js b/src/common/helper.js index e09fe77..188e762 100644 --- a/src/common/helper.js +++ b/src/common/helper.js @@ -8,6 +8,10 @@ const elasticsearch = require('@elastic/elasticsearch') const _ = require('lodash') const Joi = require('@hapi/joi') const { Mutex } = require('async-mutex') +const axios = require('axios') +const logger = require('./logger') +const m2mAuth = require('tc-core-library-js').auth.m2m +const topcoderM2M = m2mAuth(_.pick(config, ['AUTH0_URL', 'AUTH0_AUDIENCE', 'TOKEN_CACHE_TIME', 'AUTH0_PROXY_SERVER_URL'])) AWS.config.region = config.ES.AWS_REGION @@ -18,6 +22,34 @@ let transactionId const esClientMutex = new Mutex() const mutexReleaseMap = {} +/* Function to get M2M token + * (Topcoder APIs only) + * @returns {Promise} + */ +async function getTopcoderM2Mtoken () { + return topcoderM2M.getMachineToken(config.AUTH0_CLIENT_ID, config.AUTH0_CLIENT_SECRET) +} + +/** + * Returns the user in Topcoder identified by the email + * @param {String} email The user email + */ +async function getUserGroup (memberId) { + const url = config.TOPCODER_GROUP_API + const token = await getTopcoderM2Mtoken() + const params = { memberId, membershipType: 'user', page: 1 } + + logger.debug(`request GET ${url} with params: ${JSON.stringify(params)}`) + let groups = [] + let groupRes = await axios.get(url, { headers: { Authorization: `Bearer ${token}` }, params }) + while (groupRes.data.length > 0) { + groups = _.concat(groups, _.map(groupRes.data, g => _.pick(g, 'id', 'name'))) + params.page = params.page + 1 + groupRes = await axios.get(url, { headers: { Authorization: `Bearer ${token}` }, params }) + } + return groups +} + /** * Get Kafka options * @return {Object} the Kafka options @@ -201,6 +233,7 @@ module.exports = { getESClient, validProperties, getUser, + getUserGroup, updateUser, getOrg, updateOrg, diff --git a/src/services/ProcessorService.js b/src/services/ProcessorService.js index c27046a..d5a5f21 100644 --- a/src/services/ProcessorService.js +++ b/src/services/ProcessorService.js @@ -49,6 +49,12 @@ async function processCreate (message, transactionId) { user[userResource.propertyName] = [] } + // import groups for a new user + if (resource === 'externalprofile' && message.payload.externalId) { + const userGroups = await helper.getUserGroup(message.payload.externalId) + user[config.get('ES.USER_GROUP_PROPERTY_NAME')] = _.unionBy(user[config.get('ES.USER_GROUP_PROPERTY_NAME')], userGroups, 'id') + } + // check the resource does not exist if (_.some(user[userResource.propertyName], [userResource.relateKey, relateId])) { logger.error(`Can't create existed ${resource} with the ${userResource.relateKey}: ${relateId}, userId: ${message.payload.userId}`) diff --git a/test/common/init-es.js b/test/common/init-es.js index ad8c87d..5e9dae0 100644 --- a/test/common/init-es.js +++ b/test/common/init-es.js @@ -11,146 +11,51 @@ * node src/init-es * node src/init-es force */ - -const config = require('config') -const _ = require('lodash') const logger = require('../../src/common/logger') const helper = require('../../src/common/helper') -const { topResources, userResources } = require('../../src/common/constants') +const { topResources } = require('../../src/common/constants') let client -let needsNestedTypes = ['user'] - /** - * Initialize elastic search index - * @param {Boolean} isForce boolean flag indicate it is forced operation + * Check if elastic search is empty */ -const init = async (isForce) => { +const checkEmpty = async () => { if (!client) { client = await helper.getESClient() } - if (isForce) { - await clearES() - } for (const key in topResources) { - const exists = await client.indices.exists({ index: topResources[key].index }) - if (exists.body) { - logger.info(`The index ${topResources[key].index} exists.`) - } else { - logger.info(`The index ${topResources[key].index} will be created.`) - await client.indices.create({ - index: topResources[key].index, - body: { - mappings: { - properties: _(topResources[key].mappingFields).map(p => [p, { type: 'keyword' }]).fromPairs() - } - } - }) - if (needsNestedTypes.includes(key)) { - for (const childKey in userResources) { - if (userResources[childKey].isNested) { - await client.indices.putMapping({ - index: topResources[key].index, - type: topResources[key].type, - include_type_name: true, - body: { - properties: { - [userResources[childKey].propertyName]: { - type: 'nested' - } - } - } - }) - } - } + try { + const { body } = await client.search({ index: topResources[key].index }) + if (body.hits.total.value > 0) { + return false } + } catch (err) { + // ignore } } - const processors = [] - for (const key in userResources) { - logger.info(`The enrich policy ${key}-policy will be created.`) - const top = topResources[userResources[key].relateTopResource] - await client.enrich.putPolicy({ - name: top.enrichPolicy, - body: { - match: { - indices: top.index, - match_field: 'id', - enrich_fields: top.mappingFields - } - } - }) - await client.enrich.executePolicy({ name: top.enrichPolicy }) - processors.push({ - foreach: { - field: userResources[key].propertyName, - ignore_missing: true, - processor: { - enrich: { - policy_name: top.enrichPolicy, - ignore_missing: true, - field: `_ingest._value.${userResources[key].relateKey}`, - target_field: '_ingest._value' - } - } - } - }) - } - - logger.info(`The pipeline ${config.ES.ENRICH_USER_PIPELINE_NAME} will be created.`) - await client.ingest.putPipeline({ - id: config.ES.ENRICH_USER_PIPELINE_NAME, - body: { - processors - } - }) + return true } /** - * Delete elastic search index + * Clear elastic search data */ -const clearES = async () => { - try { - logger.info(`Delete pipeline ${config.ES.ENRICH_USER_PIPELINE_NAME} if any.`) - await client.ingest.deletePipeline({ id: config.ES.ENRICH_USER_PIPELINE_NAME }) - } catch (err) { - // ignore - } - for (const key in userResources) { - try { - const policyName = topResources[userResources[key].relateTopResource].enrichPolicy - logger.info(`Delete enrich policy ${policyName} if any.`) - await client.enrich.deletePolicy({ - name: policyName - }) - } catch (err) { - // ignore - } +const clearData = async () => { + if (!client) { + client = await helper.getESClient() } for (const key in topResources) { - logger.info(`Delete index ${topResources[key].index} if any.`) + logger.info(`Clear index ${topResources[key].index} data if any.`) try { - await client.indices.delete({ index: topResources[key].index }) + await client.deleteByQuery({ index: topResources[key].index, body: { query: { match_all: {} } } }) } catch (err) { // ignore + logger.logFullError(err) } } } -if (!module.parent) { - const isForce = process.argv.length === 3 && process.argv[2] === 'force' - - init(isForce).then(() => { - logger.info('done') - process.exit() - }).catch((e) => { - logger.error(e) - process.exit() - }) -} - module.exports = { - init, - clearES + checkEmpty, + clearData } diff --git a/test/common/testData.js b/test/common/testData.js index 53fd8c4..3e80b2d 100644 --- a/test/common/testData.js +++ b/test/common/testData.js @@ -1,14 +1,14 @@ module.exports = { fields: { achievement: { - createIndex: 1, - updateIndex: 1, + createIndex: 2, + updateIndex: 2, deleteIndex: 0, requiredFields: ['payload.resource', 'payload.userId', 'payload.achievementsProviderId'] }, achievementprovider: { - createIndex: 2, - updateIndex: 2, + createIndex: 1, + updateIndex: 1, deleteIndex: 1, requiredFields: ['payload.resource', 'payload.id'] }, @@ -37,14 +37,14 @@ module.exports = { requiredFields: ['payload.resource', 'payload.id'] }, skill: { - createIndex: 7, - updateIndex: 7, + createIndex: 8, + updateIndex: 8, deleteIndex: 6, requiredFields: ['payload.resource', 'payload.id'] }, skillprovider: { - createIndex: 8, - updateIndex: 8, + createIndex: 7, + updateIndex: 7, deleteIndex: 7, requiredFields: ['payload.resource', 'payload.id'] }, @@ -92,13 +92,9 @@ module.exports = { timestamp: '2019-07-08T00:00:00.000Z', 'mime-type': 'application/json', payload: { - resource: 'achievement', - userId: '391a3656-9a01-47d4-8c6d-64b68c44f212', - achievementsProviderId: 'c77326d8-ef16-4be0-b844-d5c384b7bb8b', - name: 'achievement', - uri: 'https://google.com', - certifierId: 'b8726ca1-557e-4502-8f9b-25044b9c123d', - certifiedDate: '2019-07-08T00:00:00.000Z', + resource: 'achievementprovider', + id: '7b4f98b1-5831-45fe-a71f-8454d11eb8e8', + name: 'achievementprovider', originalTopic: 'u-bahn.action.create' } }, @@ -108,9 +104,13 @@ module.exports = { timestamp: '2019-07-08T00:00:00.000Z', 'mime-type': 'application/json', payload: { - resource: 'achievementprovider', - id: '7b4f98b1-5831-45fe-a71f-8454d11eb8e8', - name: 'achievementprovider', + resource: 'achievement', + userId: '391a3656-9a01-47d4-8c6d-64b68c44f212', + achievementsProviderId: '7b4f98b1-5831-45fe-a72f-8454d11eb8e8', + name: 'achievement', + uri: 'https://google.com', + certifierId: 'b8726ca1-557e-4502-8f9b-25044b9c123d', + certifiedDate: '2019-07-08T00:00:00.000Z', originalTopic: 'u-bahn.action.create' } }, @@ -170,12 +170,9 @@ module.exports = { timestamp: '2019-07-08T00:00:00.000Z', 'mime-type': 'application/json', payload: { - resource: 'skill', - id: 'a75d95d7-6ab8-472d-8103-19d7e642e8f7', - skillProviderId: '63061b84-9784-4b71-b695-4a777eeb7601', - externalId: 'ba395d36-6ce8-4bd1-9d6c-754f0389abcb', - uri: 'https://google.com', - name: 'skill', + resource: 'skillprovider', + id: '2375564d-c5eb-4b80-9b35-465c6b700ac1', + name: 'skillprovider', originalTopic: 'u-bahn.action.create' } }, @@ -185,9 +182,12 @@ module.exports = { timestamp: '2019-07-08T00:00:00.000Z', 'mime-type': 'application/json', payload: { - resource: 'skillprovider', - id: '2375564d-c5eb-4b80-9b35-465c6b700ac1', - name: 'skillprovider', + resource: 'skill', + id: 'a75d95d7-6ab8-472d-8103-19d7e642e8f7', + skillProviderId: '2375564d-c5eb-4b80-9b35-465c6b700ac1', + externalId: 'ba395d36-6ce8-4bd1-9d6c-754f0389abcb', + uri: 'https://google.com', + name: 'skill', originalTopic: 'u-bahn.action.create' } }, @@ -212,7 +212,7 @@ module.exports = { payload: { resource: 'userrole', userId: '391a3656-9a01-47d4-8c6d-64b68c44f212', - roleId: '22028da5-0563-48e8-b84c-e480eb8ed98c', + roleId: '288446f1-02dc-4fc7-b74e-ab7ea3033a57', originalTopic: 'u-bahn.action.create' } }, @@ -224,7 +224,7 @@ module.exports = { payload: { resource: 'userskill', userId: '391a3656-9a01-47d4-8c6d-64b68c44f212', - skillId: '8a8c8d3a-9165-4dae-8a8c-f828cbe01d5d', + skillId: 'b75d95d7-6ab8-472d-8103-19d7e642e8f7', metricValue: 'userskill', certifierId: '7cf786d9-a8c0-48ed-a7cc-09dcf91d904c', certifiedDate: '2019-07-08T00:00:00.000Z', @@ -263,11 +263,9 @@ module.exports = { timestamp: '2019-07-08T00:00:00.000Z', 'mime-type': 'application/json', payload: { - resource: 'achievement', - userId: '391a3656-9a01-47d4-8c6d-64b68c44f212', - achievementsProviderId: 'c77326d8-ef16-4be0-b844-d5c384b7bb8b', + resource: 'achievementprovider', + id: '7b4f98b1-5831-45fe-a71f-8454d11eb8e8', name: 'update_name', - uri: 'https://facebook.com', originalTopic: 'u-bahn.action.update' } }, @@ -277,9 +275,11 @@ module.exports = { timestamp: '2019-07-08T00:00:00.000Z', 'mime-type': 'application/json', payload: { - resource: 'achievementprovider', - id: '7b4f98b1-5831-45fe-a71f-8454d11eb8e8', + resource: 'achievement', + userId: '391a3656-9a01-47d4-8c6d-64b68c44f212', + achievementsProviderId: '7b4f98b1-5831-45fe-a72f-8454d11eb8e8', name: 'update_name', + uri: 'https://facebook.com', originalTopic: 'u-bahn.action.update' } }, @@ -339,12 +339,9 @@ module.exports = { timestamp: '2019-07-08T00:00:00.000Z', 'mime-type': 'application/json', payload: { - resource: 'skill', - id: 'a75d95d7-6ab8-472d-8103-19d7e642e8f7', - skillProviderId: '63061b84-9784-4b71-b695-4a777eeb7601', - externalId: 'ba395d36-6ce8-4bd1-9d6c-754f0389abcb', - uri: 'https://facebook.com', - name: 'update_skill', + resource: 'skillprovider', + id: '2375564d-c5eb-4b80-9b35-465c6b700ac1', + name: 'update_skillprovider', originalTopic: 'u-bahn.action.update' } }, @@ -354,9 +351,12 @@ module.exports = { timestamp: '2019-07-08T00:00:00.000Z', 'mime-type': 'application/json', payload: { - resource: 'skillprovider', - id: '2375564d-c5eb-4b80-9b35-465c6b700ac1', - name: 'update_skillprovider', + resource: 'skill', + id: 'a75d95d7-6ab8-472d-8103-19d7e642e8f7', + skillProviderId: '2375564d-c5eb-4b80-9b35-465c6b700ac1', + externalId: 'ba395d36-6ce8-4bd1-9d6c-754f0389abcb', + uri: 'https://facebook.com', + name: 'update_skill', originalTopic: 'u-bahn.action.update' } }, @@ -381,7 +381,7 @@ module.exports = { payload: { resource: 'userskill', userId: '391a3656-9a01-47d4-8c6d-64b68c44f212', - skillId: '8a8c8d3a-9165-4dae-8a8c-f828cbe01d5d', + skillId: 'b75d95d7-6ab8-472d-8103-19d7e642e8f7', metricValue: 'update_userskill', originalTopic: 'u-bahn.action.update' } @@ -537,7 +537,7 @@ module.exports = { payload: { resource: 'achievement', userId: '391a3656-9a01-47d4-8c6d-64b68c44f212', - achievementsProviderId: 'c77326d8-ef16-4be0-b844-d5c384b7bb8b', + achievementsProviderId: '7b4f98b1-5831-45fe-a72f-8454d11eb8e8', originalTopic: 'u-bahn.action.delete' } }, @@ -639,7 +639,7 @@ module.exports = { payload: { resource: 'userrole', userId: '391a3656-9a01-47d4-8c6d-64b68c44f212', - roleId: '22028da5-0563-48e8-b84c-e480eb8ed98c', + roleId: '288446f1-02dc-4fc7-b74e-ab7ea3033a57', originalTopic: 'u-bahn.action.delete' } }, @@ -651,7 +651,7 @@ module.exports = { payload: { resource: 'userskill', userId: '391a3656-9a01-47d4-8c6d-64b68c44f212', - skillId: '8a8c8d3a-9165-4dae-8a8c-f828cbe01d5d', + skillId: 'b75d95d7-6ab8-472d-8103-19d7e642e8f7', originalTopic: 'u-bahn.action.delete' } }, diff --git a/test/common/testHelper.js b/test/common/testHelper.js index ca89be5..cba1d1a 100644 --- a/test/common/testHelper.js +++ b/test/common/testHelper.js @@ -61,7 +61,39 @@ async function getESGroupRecord (userId, groupId) { return _.find(user[propertyName], { id: groupId }) } +/** + * Get expect value. + * @param {object} payload message payload + * @param {array} relationRecord test payload list + */ +function getExpectValue (payload, relationRecord) { + const result = _.omit(payload, ['resource', 'originalTopic']) + if (topResources[payload.resource] && topResources[payload.resource].ingest) { + if (topResources[payload.resource].pipeline) { + _.each(topResources[payload.resource].pipeline.processors, p => { + const relationResource = _.keys(_.pickBy(topResources, value => value.enrich && value.enrich.policyName === p.policyName))[0] + if (relationResource) { + const record = _.find(relationRecord, r => r.payload.resource === relationResource && payload[p.field] === r.payload.id) + if (record) { + result[p.referenceField] = _.pick(record.payload, topResources[relationResource].enrich.enrichFields) + } + } + }) + } else { + const relationResource = _.keys(_.pickBy(topResources, value => value.enrich && value.pipeline && value.pipeline.id === topResources[payload.resource].ingest.pipeline.id))[0] + if (relationResource) { + const record = _.find(relationRecord, r => r.payload.resource === relationResource && payload[topResources[relationResource].pipeline.field] === r.payload[topResources[relationResource].enrich.matchField]) + if (record) { + result[topResources[relationResource].pipeline.targetField] = _.pick(record.payload, topResources[relationResource].enrich.enrichFields) + } + } + } + } + return result +} + module.exports = { getESRecord, - getESGroupRecord + getESGroupRecord, + getExpectValue } diff --git a/test/e2e/test.js b/test/e2e/test.js index 0cfcd18..77253e2 100644 --- a/test/e2e/test.js +++ b/test/e2e/test.js @@ -13,8 +13,8 @@ const Kafka = require('no-kafka') const should = require('should') const logger = require('../../src/common/logger') const { fields, testTopics, groupsTopics } = require('../common/testData') -const { init, clearES } = require('../common/init-es') -const { getESRecord, getESGroupRecord } = require('../common/testHelper') +const { checkEmpty, clearData } = require('../common/init-es') +const { getESRecord, getESGroupRecord, getExpectValue } = require('../common/testHelper') describe('UBahn - Elasticsearch Data Processor E2E Test', () => { let app @@ -88,7 +88,10 @@ describe('UBahn - Elasticsearch Data Processor E2E Test', () => { } before(async () => { - await init(true) + const isESEmpty = await checkEmpty() + if (!isESEmpty) { + throw Error('Can not run e2e test when elastic search already have data') + } // inject logger with log collector logger.info = (message) => { @@ -134,7 +137,7 @@ describe('UBahn - Elasticsearch Data Processor E2E Test', () => { // ignore } - await clearES() + await clearData() }) beforeEach(() => { @@ -208,7 +211,8 @@ describe('UBahn - Elasticsearch Data Processor E2E Test', () => { if (testTopics[op][i].payload.resource === 'user') { should.equal(ret.handle, testTopics[op][i].payload.handle) } else { - should.deepEqual(ret, _.omit(testTopics[op][i].payload, ['resource', 'originalTopic'])) + // should.deepEqual(ret, _.omit(testTopics[op][i].payload, ['resource', 'originalTopic'])) + should.deepEqual(ret, getExpectValue(testTopics[op][i].payload, testTopics[op])) } } }) @@ -227,7 +231,7 @@ describe('UBahn - Elasticsearch Data Processor E2E Test', () => { }) } - if (op === 'Create') { + if (op === 'Create' && !topResources[_.lowerFirst(resource)]) { it(`failure - process create ${resource} with duplicate id`, async () => { await sendMessage(testTopics[op][i]) await waitJob() diff --git a/test/unit/prepare.js b/test/unit/prepare.js index 91ae9fb..0a25c27 100644 --- a/test/unit/prepare.js +++ b/test/unit/prepare.js @@ -41,12 +41,8 @@ prepare(function (done) { .query(true) .reply((uri, body) => { const id = _.last(_.split(uri, '/')).split('?')[0] - if (content[id]) { - content[id] = body - return [200] - } else { - return [404] - } + content[id] = body + return [200] }) .delete(() => true) .query(true) diff --git a/test/unit/test.js b/test/unit/test.js index fef2f11..4f8d8fa 100644 --- a/test/unit/test.js +++ b/test/unit/test.js @@ -113,7 +113,7 @@ describe('UBahn - Elasticsearch Data Processor Unit Test', () => { }) } - if (op === 'Create') { + if (op === 'Create' && !topResources[_.lowerFirst(resource)]) { it(`failure - process create ${resource} with duplicate id`, async () => { try { await service.processCreate(testTopics[op][i], 'transaction_11111')