From cc5ba1d6b5b058b68f333c5948354e3d096693f4 Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Fri, 11 Sep 2020 11:58:51 +0530 Subject: [PATCH 1/5] #45 - Rename search endpoint to skill search --- docs/UBahn_API.postman_collection.json | 84 ++++++++++++++++---------- docs/swagger.yaml | 18 +++--- src/modules/search/route.js | 8 +-- 3 files changed, 66 insertions(+), 44 deletions(-) diff --git a/docs/UBahn_API.postman_collection.json b/docs/UBahn_API.postman_collection.json index e86ffe3..d63ad7e 100644 --- a/docs/UBahn_API.postman_collection.json +++ b/docs/UBahn_API.postman_collection.json @@ -1,20 +1,20 @@ { "info": { - "_postman_id": "2e9d0d1e-22b4-4b5c-af7d-8e5d407de79b", + "_postman_id": "6d99ffc9-3c8b-4c0a-a006-5f1d3d23801d", "name": "UBahn_API", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" }, "item": [ { - "name": "search", + "name": "skillSearch", "item": [ { - "name": "{{HOST}}/search/userAttributes", + "name": "{{HOST}}/skill-search/userAttributes", "event": [ { "listen": "test", "script": { - "id": "256b27ab-e629-429f-ad93-372eaaa46239", + "id": "769b9779-be21-4518-9276-f8b13c44c0d0", "exec": [ "var rsp = pm.response.json();", "if(rsp.id) pm.environment.set(\"userId\", rsp.id);" @@ -39,12 +39,12 @@ } ], "url": { - "raw": "{{HOST}}/search/userAttributes?attributeId=c44d4bee-1356-46d6-9f1f-991936dec297&attributeValue=support", + "raw": "{{HOST}}/skill-search/userAttributes?attributeId=c44d4bee-1356-46d6-9f1f-991936dec297&attributeValue=support", "host": [ "{{HOST}}" ], "path": [ - "search", + "skill-search", "userAttributes" ], "query": [ @@ -62,12 +62,12 @@ "response": [] }, { - "name": "{{HOST}}/search/userAchievements", + "name": "{{HOST}}/skill-search/userAchievements", "event": [ { "listen": "test", "script": { - "id": "d1951826-476c-4149-bfa7-2a853c30df4e", + "id": "b2b52a88-23c3-4547-89b5-0a0afa0dad19", "exec": [ "" ], @@ -91,12 +91,12 @@ } ], "url": { - "raw": "{{HOST}}/search/userAchievements?organizationId=36ed815b-3da1-49f1-a043-aaed0a4e81ad&keyword=Topcoder", + "raw": "{{HOST}}/skill-search/userAchievements?organizationId=36ed815b-3da1-49f1-a043-aaed0a4e81ad&keyword=Topcoder", "host": [ "{{HOST}}" ], "path": [ - "search", + "skill-search", "userAchievements" ], "query": [ @@ -114,12 +114,12 @@ "response": [] }, { - "name": "{{HOST}}/search/skills", + "name": "{{HOST}}/skill-search/skills", "event": [ { "listen": "test", "script": { - "id": "d56349bd-a797-47cf-b6bf-2ecfbd342815", + "id": "35cfaef0-78f5-4ff9-83f3-c2a705dc0e9d", "exec": [ "" ], @@ -143,12 +143,12 @@ } ], "url": { - "raw": "{{HOST}}/search/skills?organizationId=36ed815b-3da1-49f1-a043-aaed0a4e81ad&keyword=net", + "raw": "{{HOST}}/skill-search/skills?organizationId=36ed815b-3da1-49f1-a043-aaed0a4e81ad&keyword=net", "host": [ "{{HOST}}" ], "path": [ - "search", + "skill-search", "skills" ], "query": [ @@ -166,12 +166,12 @@ "response": [] }, { - "name": "{{HOST}}/search/users", + "name": "{{HOST}}/skill-search/users", "event": [ { "listen": "test", "script": { - "id": "75560f72-efeb-438e-ac29-24d7c6ab540e", + "id": "23733012-4565-480e-9c0c-66b7c2fd2933", "exec": [ "var rsp = pm.response.json();", "if(rsp.id) pm.environment.set(\"userId\", rsp.id);" @@ -203,12 +203,12 @@ } }, "url": { - "raw": "{{HOST}}/search/users", + "raw": "{{HOST}}/skill-search/users", "host": [ "{{HOST}}" ], "path": [ - "search", + "skill-search", "users" ] } @@ -216,6 +216,28 @@ "response": [] } ], + "event": [ + { + "listen": "prerequest", + "script": { + "id": "e1951d10-6818-4d2d-8d49-452f3eb31f74", + "type": "text/javascript", + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "id": "bb0d8108-d90a-4b16-ac59-7d0720a86c81", + "type": "text/javascript", + "exec": [ + "" + ] + } + } + ], "protocolProfileBehavior": {} }, { @@ -227,7 +249,7 @@ { "listen": "test", "script": { - "id": "5db9dc16-daad-4768-a3af-ea8983487bec", + "id": "a80d76a2-572f-47e8-bb5b-7594906e6f17", "exec": [ "var rsp = pm.response.json();", "if(rsp.id) pm.environment.set(\"userId\", rsp.id);" @@ -594,7 +616,7 @@ { "listen": "test", "script": { - "id": "8469c20d-3ef5-4868-af50-d603584ed8a7", + "id": "021253da-d913-43df-b490-28001f14b233", "exec": [ "var rsp = pm.response.json();", "if(rsp.id) pm.environment.set(\"roleId\", rsp.id);" @@ -829,7 +851,7 @@ { "listen": "test", "script": { - "id": "d4ca59b9-f350-4c91-9667-a5e749eed956", + "id": "4a70f27f-df1b-43cf-a0fa-a2ca58a5521a", "exec": [ "" ], @@ -1071,7 +1093,7 @@ { "listen": "test", "script": { - "id": "ff10985c-75c8-48ba-b522-2c8213b0c6cd", + "id": "9194de70-291f-4660-82d3-5883c23d12bb", "exec": [ "var rsp = pm.response.json();", "if(rsp.id) pm.environment.set(\"organizationId\", rsp.id);" @@ -1306,7 +1328,7 @@ { "listen": "test", "script": { - "id": "0a875ba8-b4dc-42cc-b8ba-deb560b73556", + "id": "f40cae85-9ad1-4c54-ad87-b592395027ac", "exec": [ "var rsp = pm.response.json();", "if(rsp.id) pm.environment.set(\"skillsProviderId\", rsp.id);" @@ -1541,7 +1563,7 @@ { "listen": "test", "script": { - "id": "909e2488-3801-4b57-90c5-6a6ae2e91f13", + "id": "a0153fe7-82ee-4a03-af43-19cf44fab4f0", "exec": [ "var rsp = pm.response.json();", "if(rsp.id) pm.environment.set(\"skillId\", rsp.id);" @@ -1777,7 +1799,7 @@ { "listen": "test", "script": { - "id": "e3241acb-646f-4ec8-83fb-8a3d63da4990", + "id": "b4d7a5cc-9e17-4d45-84c3-6dfbdcf09579", "exec": [ "" ], @@ -2025,7 +2047,7 @@ { "listen": "test", "script": { - "id": "ee9256b0-cdc2-4194-9da7-c0b363082385", + "id": "08a6d74a-268c-403b-8cb1-85bee3c88855", "exec": [ "" ], @@ -2273,7 +2295,7 @@ { "listen": "test", "script": { - "id": "3c959e64-93e9-480d-b92e-44eacb46c2b1", + "id": "680443d3-c817-4939-8461-445fbeea6b00", "exec": [ "var rsp = pm.response.json();", "if(rsp.id) pm.environment.set(\"achievementsProviderId\", rsp.id);" @@ -2508,7 +2530,7 @@ { "listen": "test", "script": { - "id": "9b4f70b8-3ca8-4c1c-a65a-8a8b37e2fa62", + "id": "4ccce827-71ed-4373-b14f-e103b334e73f", "exec": [ "" ], @@ -2757,7 +2779,7 @@ { "listen": "test", "script": { - "id": "fb796928-9848-47a3-be88-61a8fc9e60f7", + "id": "6670cb30-0276-4625-9688-8cb1aa78d8fb", "exec": [ "var rsp = pm.response.json();", "if (rsp.id) pm.environment.set(\"attributeGroupId\", rsp.id);" @@ -2998,7 +3020,7 @@ { "listen": "test", "script": { - "id": "eb028137-ee03-496c-a812-2966380bca42", + "id": "bad538df-7c33-4d34-a90e-9e60e279f40d", "exec": [ "var rsp = pm.response.json();", "if (rsp.id) pm.environment.set(\"attributeId\", rsp.id);" @@ -3238,7 +3260,7 @@ { "listen": "test", "script": { - "id": "06c18b73-b9e6-46a8-acfa-74fc7ae68ac5", + "id": "b91ea1ce-88a2-48fa-8e90-ed689232b5e5", "exec": [ "" ], diff --git a/docs/swagger.yaml b/docs/swagger.yaml index bbdfecf..d8f7ba7 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -37,7 +37,7 @@ tags: description: "Attributes registered in the system" - name: "Attribute Groups" description: "Attribute groups registered in the system" -- name: "Search" +- name: "Skill Search" description: "Search info registered in the system" schemes: - "https" @@ -3718,10 +3718,10 @@ paths: security: - Bearer: [] x-swagger-router-controller: "AttributeGroups" - /search/users: + /skill-search/users: post: tags: - - "Search" + - "Skill Search" description: "Search for users in the application. If no results, then empty\ \ array is returned. Multiple filters are\nsupported.\n\n**Security** - Note\ \ that for non-admin users, this endpoint will only return entities that\n\ @@ -3758,10 +3758,10 @@ paths: security: - Bearer: [] x-swagger-router-controller: "Search" - /search/userAttributes: + /skill-search/userAttributes: get: tags: - - "Search" + - "Skill Search" description: "Search for user attributes in the application. If no results, then empty\ \ array is returned. Multiple filters are\nsupported.\n\n**Security** - Note\ \ that for non-admin users, this endpoint will only return entities that\n\ @@ -3795,10 +3795,10 @@ paths: security: - Bearer: [] x-swagger-router-controller: "Search" - /search/userAchievements: + /skill-search/userAchievements: get: tags: - - "Search" + - "Skill Search" description: "Search for user achievements in the application. If no results, then empty\ \ array is returned. Multiple filters are\nsupported.\n\n**Security** - Note\ \ that for non-admin users, this endpoint will only return entities that\n\ @@ -3833,10 +3833,10 @@ paths: security: - Bearer: [] x-swagger-router-controller: "Search" - /search/skills: + /skill-search/skills: get: tags: - - "Search" + - "Skill Search" description: "Search for skills associated with an org in the application. If no results, then empty\ \ array is returned. Multiple filters are\nsupported.\n\n**Security** - Note\ \ that for non-admin users, this endpoint will only return entities that\n\ diff --git a/src/modules/search/route.js b/src/modules/search/route.js index 4eb0c08..3b43020 100644 --- a/src/modules/search/route.js +++ b/src/modules/search/route.js @@ -6,7 +6,7 @@ const Controller = require('./controller') const consts = require('../../consts') module.exports = { - '/search/users': { + '/skill-search/users': { post: { method: Controller.searchUsers, auth: 'jwt', @@ -14,7 +14,7 @@ module.exports = { scopes: ['read:user', 'all:user'] } }, - '/search/skills': { + '/skill-search/skills': { get: { method: Controller.searchSkills, auth: 'jwt', @@ -22,7 +22,7 @@ module.exports = { scopes: ['create:userAttribute', 'all:userAttribute'] } }, - '/search/userAttributes': { + '/skill-search/userAttributes': { get: { method: Controller.searchAttributeValues, auth: 'jwt', @@ -30,7 +30,7 @@ module.exports = { scopes: ['create:userAttribute', 'all:userAttribute'] } }, - '/search/userAchievements': { + '/skill-search/userAchievements': { get: { method: Controller.searchAchievementValues, auth: 'jwt', From 71eeb2c733636ecab4543a81356383b50d5d425d Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Sun, 20 Sep 2020 13:08:14 +0530 Subject: [PATCH 2/5] #47 - Read groups info from elasticsearch instead of group api --- README.md | 1 - config/default.js | 1 - src/common/es-helper.js | 13 ------------- src/common/group-api.js | 39 --------------------------------------- 4 files changed, 54 deletions(-) delete mode 100644 src/common/group-api.js diff --git a/README.md b/README.md index 827e445..ce89626 100755 --- a/README.md +++ b/README.md @@ -28,7 +28,6 @@ Configuration for the application is at config/default.js and config/production. - AUTH0_CLIENT_ID: Auth0 client id, used to get TC M2M token - AUTH0_CLIENT_SECRET: Auth0 client secret, used to get TC M2M token - AUTH0_PROXY_SERVER_URL: Proxy Auth0 URL, used to get TC M2M token -- GROUP_API_URL: Topcoder Group API URL - BUSAPI_URL: Topcoder Bus API URL - KAFKA_ERROR_TOPIC: The error topic at which bus api will publish any errors - KAFKA_MESSAGE_ORIGINATOR: The originator value for the kafka messages diff --git a/config/default.js b/config/default.js index 00b208f..b46f98b 100755 --- a/config/default.js +++ b/config/default.js @@ -26,7 +26,6 @@ module.exports = { AUTH0_CLIENT_SECRET: process.env.AUTH0_CLIENT_SECRET, AUTH0_PROXY_SERVER_URL: process.env.AUTH0_PROXY_SERVER_URL, - GROUP_API_URL: process.env.GROUP_API_URL || 'https://api.topcoder-dev.com/v5/groups', BUSAPI_URL: process.env.BUSAPI_URL || 'https://api.topcoder-dev.com/v5', KAFKA_ERROR_TOPIC: process.env.KAFKA_ERROR_TOPIC || 'common.error.reporting', diff --git a/src/common/es-helper.js b/src/common/es-helper.js index cffa69a..0931403 100644 --- a/src/common/es-helper.js +++ b/src/common/es-helper.js @@ -2,7 +2,6 @@ const config = require('config') const _ = require('lodash') const querystring = require('querystring') const logger = require('../common/logger') -const groupApi = require('./group-api') const appConst = require('../consts') const esClient = require('./es-client').getESClient() @@ -499,8 +498,6 @@ async function getFromElasticSearch (resource, ...args) { if (params.enrich && resource === 'user') { const user = await enrichUser(result) - const groups = await groupApi.getGroups(user.id) - user.groups = groups return user } else if (subUserDoc) { // find top sub doc by sub.id @@ -1305,11 +1302,6 @@ async function searchElasticSearch (resource, ...args) { if (resource === 'user' && params.enrich) { const users = docs.hits.hits.map(hit => hit._source) result = await enrichUsers(users) - // enrich groups - for (const user of users) { - const groups = await groupApi.getGroups(user.id) - user.groups = groups - } } else if (topUserSubDoc) { result = docs.hits.hits[0]._source[topUserSubDoc.userField] // for sub-resource query, it returns all sub-resource items in one user, @@ -1430,11 +1422,6 @@ async function searchUsers (authUser, filter, params) { logger.debug('Enrich users') const result = await enrichUsers(users) - // enrich groups - for (const user of users) { - const groups = await groupApi.getGroups(user.id) - user.groups = groups - } return { total: getTotalCount(docs.hits.total), diff --git a/src/common/group-api.js b/src/common/group-api.js deleted file mode 100644 index 47cada3..0000000 --- a/src/common/group-api.js +++ /dev/null @@ -1,39 +0,0 @@ -const config = require('config') -const _ = require('lodash') -const axios = require('axios') -const m2mAuth = require('tc-core-library-js').auth.m2m -const logger = require('./logger') - -const m2m = m2mAuth(_.pick(config, ['AUTH0_URL', 'AUTH0_AUDIENCE', 'TOKEN_CACHE_TIME', 'AUTH0_PROXY_SERVER_URL'])) - -/** - * Get M2M token. - * @returns {Promise} - */ -async function getM2Mtoken () { - return m2m.getMachineToken(config.AUTH0_CLIENT_ID, config.AUTH0_CLIENT_SECRET) -} - -async function getGroups (universalUID) { - const m2mToken = await getM2Mtoken() - - logger.debug(`The m2m token is ${m2mToken} for user ${universalUID} with group api endpoint ${config.GROUP_API_URL}`) - - try { - const resp = await axios({ - method: 'get', - params: { universalUID, membershipType: 'user' }, - url: config.GROUP_API_URL, - headers: { Authorization: `Bearer ${m2mToken}` } - }) - return resp.data - } catch (error) { - logger.error(error) - return [] - } -} - -module.exports = { - getM2Mtoken, - getGroups -} From b420f0aa82be1f8ab7dfcb6a5c68827dd1437f2c Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Mon, 21 Sep 2020 21:05:28 +0530 Subject: [PATCH 3/5] Measure search execution time --- src/modules/search/controller.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/modules/search/controller.js b/src/modules/search/controller.js index 5412aec..69c9e16 100644 --- a/src/modules/search/controller.js +++ b/src/modules/search/controller.js @@ -9,7 +9,9 @@ const service = require('./service') * Search for users. Returns enriched users */ async function searchUsers (req, res) { + console.time('searchusers') const result = await esHelper.searchUsers(req.auth, req.body, req.query) + console.timeEnd('searchusers') injectSearchMeta(req, res, result) res.send(result.result) } From 896b9e4fbf929ab00efcb602d6d2391f29db5674 Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Mon, 21 Sep 2020 21:22:05 +0530 Subject: [PATCH 4/5] More console time statements to measure perf --- src/common/es-helper.js | 10 ++++++++-- src/modules/search/controller.js | 4 ++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/common/es-helper.js b/src/common/es-helper.js index 0931403..0b395c5 100644 --- a/src/common/es-helper.js +++ b/src/common/es-helper.js @@ -1355,14 +1355,18 @@ async function searchUsers (authUser, filter, params) { const authUserOrganizationId = filter.organizationId const filterKey = Object.keys(userFilters) + console.time('resolveUserFilterFromDb') for (const key of filterKey) { const resolved = await resolveUserFilterFromDb(userFilters[key], authUser, authUserOrganizationId) resolvedUserFilters.push(resolved) } + console.timeEnd('resolveUserFilterFromDb') + console.time('resolveSortClauseFromDb') if (params.orderBy) { sortClause = sortClause.concat(await resolveSortClauseFromDb(params.orderBy, authUser, authUserOrganizationId)) } + console.timeEnd('resolveSortClauseFromDb') const esQuery = { index: queryDoc.index, @@ -1415,13 +1419,15 @@ async function searchUsers (authUser, filter, params) { }) logger.debug(`ES query for searching users: ${JSON.stringify(esQuery, null, 2)}`) - + console.time('mainesquery') const docs = await esClient.search(esQuery) + console.timeEnd('mainesquery') const users = docs.hits.hits.map(hit => hit._source) logger.debug('Enrich users') - + console.time('enrichUsers') const result = await enrichUsers(users) + console.timeEnd('enrichUsers') return { total: getTotalCount(docs.hits.total), diff --git a/src/modules/search/controller.js b/src/modules/search/controller.js index 69c9e16..2154064 100644 --- a/src/modules/search/controller.js +++ b/src/modules/search/controller.js @@ -9,9 +9,9 @@ const service = require('./service') * Search for users. Returns enriched users */ async function searchUsers (req, res) { - console.time('searchusers') + console.time('searchuserscontroller') const result = await esHelper.searchUsers(req.auth, req.body, req.query) - console.timeEnd('searchusers') + console.timeEnd('searchuserscontroller') injectSearchMeta(req, res, result) res.send(result.result) } From 685bd21031684611def84857022df2fefab0a17c Mon Sep 17 00:00:00 2001 From: Mithun Kamath Date: Mon, 28 Sep 2020 15:59:18 +0530 Subject: [PATCH 5/5] Find exact match of attribute group names --- src/modules/attributeGroup/service.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/attributeGroup/service.js b/src/modules/attributeGroup/service.js index f8b776f..b9f14c7 100644 --- a/src/modules/attributeGroup/service.js +++ b/src/modules/attributeGroup/service.js @@ -23,7 +23,7 @@ const methods = helper.getServiceMethods( const dbQueries = [] // filter by provider name if (query.name) { - dbQueries.push(`name like '%${query.name}%'`) + dbQueries.push(`name = '${query.name}'`) } if (query.organizationId) { dbQueries.push(`organizationId = '${query.organizationId}'`)