From ce7ea1af679c273454e84a4b00e7494d2d6962cd Mon Sep 17 00:00:00 2001 From: eisbilir Date: Fri, 30 Jul 2021 21:50:30 +0300 Subject: [PATCH 01/13] adding new universal notification service --- README.md | 4 + config/default.js | 22 +- connect/config.js | 2 +- connect/connectNotificationServer.js | 7 +- constants.js | 3 + docs/tc-notifications.postman_collection.json | 38 +++ .../tc-notifications.postman_environment.json | 19 ++ package-lock.json | 284 +++++++++++++----- src/app.js | 24 +- src/common/tcApiHelper.js | 121 +++++++- src/controllers/NotificationController.js | 6 +- src/hooks/hookBulkMessage.js | 252 ++++++++-------- src/hooks/index.js | 4 +- src/models/BulkMessageUserRefs.js | 32 +- .../broadcast/bulkNotificationHandler.js | 21 +- src/processors/index.js | 2 + .../universal/universalNotificationHandler.js | 20 ++ src/services/NotificationService.js | 15 +- src/services/UniversalNotificationService.js | 126 ++++++++ test/checkHooks.js | 4 +- 20 files changed, 740 insertions(+), 266 deletions(-) create mode 100644 docs/tc-notifications.postman_collection.json create mode 100644 docs/tc-notifications.postman_environment.json create mode 100644 src/processors/universal/universalNotificationHandler.js create mode 100644 src/services/UniversalNotificationService.js diff --git a/README.md b/README.md index affbd38..b08fb02 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,10 @@ The following parameters can be set in config files or in env variables: - `REPLY_EMAIL_PREFIX`: prefix of the genereated reply email address - `REPLY_EMAIL_DOMAIN`: email domain - `DEFAULT_REPLY_EMAIL`: default reply to email address, for example no-reply@topcoder.com +- **Slack api** + - `SLACK_URL`: slack api url to post messages + - `SLACK_BOT_TOKEN`: slack bot user OAuth token + - `SLACK_NOTIFY`: slack notification switch, set to 'true' to enable slack notifications. Note that the above two configuration are separate because the common notification server config will be deployed to a NPM package, the connect notification server will use that NPM package, diff --git a/config/default.js b/config/default.js index 9d426a7..912c909 100644 --- a/config/default.js +++ b/config/default.js @@ -47,7 +47,12 @@ module.exports = { AUTH0_CLIENT_ID: process.env.AUTH0_CLIENT_ID, AUTH0_CLIENT_SECRET: process.env.AUTH0_CLIENT_SECRET, AUTH0_PROXY_SERVER_URL: process.env.AUTH0_PROXY_SERVER_URL, - + // Slack configuration. + SLACK: { + URL: process.env.SLACK_URL || 'https://slack.com/api/chat.postMessage', + BOT_TOKEN: process.env.SLACK_BOT_TOKEN, + NOTIFY: process.env.SLACK_NOTIFY === 'true', + }, KAFKA_CONSUMER_RULESETS: { // key is Kafka topic name, value is array of ruleset which have key as // handler function name defined in src/processors/index.js @@ -115,13 +120,16 @@ module.exports = { // }, // }, // ], - // */ // issue - https://github.com/topcoder-platform/community-app/issues/4108 + // */ // issue - https://github.com/topcoder-platform/community-app/issues/4108 'admin.notification.broadcast': [{ - handleBulkNotification: {} - } - ] - //'notifications.community.challenge.created': ['handleChallengeCreated'], - //'notifications.community.challenge.phasewarning': ['handleChallengePhaseWarning'], + handleBulkNotification: {}, + }, + ], + 'notification.action.create': [{ + handleUniversalNotification: {}, + }], + // 'notifications.community.challenge.created': ['handleChallengeCreated'], + // 'notifications.community.challenge.phasewarning': ['handleChallengePhaseWarning'], }, // email notification service related variables diff --git a/connect/config.js b/connect/config.js index c54bc28..e5d985c 100644 --- a/connect/config.js +++ b/connect/config.js @@ -36,6 +36,6 @@ module.exports = { DEFAULT_REPLY_EMAIL: process.env.DEFAULT_REPLY_EMAIL, CONNECT_URL: process.env.CONNECT_URL || 'https://connect.topcoder-dev.com', - ACCOUNTS_APP_URL: process.env.ACCOUNTS_APP_URL || "https://accounts-auth0.topcoder-dev.com", + ACCOUNTS_APP_URL: process.env.ACCOUNTS_APP_URL || 'https://accounts-auth0.topcoder-dev.com', TC_CDN_URL: process.env.TC_CDN_URL, }; diff --git a/connect/connectNotificationServer.js b/connect/connectNotificationServer.js index 34674e3..63fe8dc 100644 --- a/connect/connectNotificationServer.js +++ b/connect/connectNotificationServer.js @@ -92,7 +92,7 @@ const getNotificationsForMentionedUser = (logger, eventConfig, content) => { _.forEach(notifications, (notification) => { const mentionedUser = _.find(users, { handle: notification.userHandle }); notification.userId = mentionedUser ? mentionedUser.userId.toString() : null; - if (!notification.userId && logger) {// such notifications would be discarded later after aggregation + if (!notification.userId && logger) { // such notifications would be discarded later after aggregation logger.info(`Unable to find user with handle ${notification.userHandle}`); } }); @@ -102,12 +102,11 @@ const getNotificationsForMentionedUser = (logger, eventConfig, content) => { logger.error(error); logger.info('Unable to send notification to mentioned user'); } - //resolves with empty notification which essentially means we are unable to send notification to mentioned user + // resolves with empty notification which essentially means we are unable to send notification to mentioned user return Promise.resolve([]); }); - } else { - return Promise.resolve([]); } + return Promise.resolve([]); }; /** diff --git a/constants.js b/constants.js index 4cadd8f..ba27c18 100644 --- a/constants.js +++ b/constants.js @@ -3,11 +3,14 @@ module.exports = { SEARCH_USERS_PAGE_SIZE: 5, SETTINGS_EMAIL_SERVICE_ID: 'email', + SETTINGS_WEB_SERVICE_ID: 'web', + SETTINGS_SLACK_SERVICE_ID: 'slack', ACTIVE_USER_STATUSES: ['ACTIVE'], BUS_API_EVENT: { EMAIL: { GENERAL: 'connect.notification.email.project.notifications.generic', + UNIVERSAL: 'external.action.email', }, }, }; diff --git a/docs/tc-notifications.postman_collection.json b/docs/tc-notifications.postman_collection.json new file mode 100644 index 0000000..fd6249a --- /dev/null +++ b/docs/tc-notifications.postman_collection.json @@ -0,0 +1,38 @@ +{ + "info": { + "_postman_id": "ad14efc8-1fed-4914-8273-330754500801", + "name": "TC-NOTIFICATIONS", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" + }, + "item": [ + { + "name": "list notifications", + "request": { + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "Bearer {{TOKEN}}", + "type": "text" + } + ], + "url": { + "raw": "{{URL}}/list?limit=100", + "host": [ + "{{URL}}" + ], + "path": [ + "list" + ], + "query": [ + { + "key": "limit", + "value": "100" + } + ] + } + }, + "response": [] + } + ] +} \ No newline at end of file diff --git a/docs/tc-notifications.postman_environment.json b/docs/tc-notifications.postman_environment.json new file mode 100644 index 0000000..d338b9e --- /dev/null +++ b/docs/tc-notifications.postman_environment.json @@ -0,0 +1,19 @@ +{ + "id": "9d9c9e1b-6004-4bbe-9a98-55ad3a5838d7", + "name": "tc-notifications", + "values": [ + { + "key": "URL", + "value": "http://localhost:3000/v5/notifications", + "enabled": true + }, + { + "key": "TOKEN", + "value": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlcyI6WyJUb3Bjb2RlciBVc2VyIiwiYWRtaW5pc3RyYXRvciJdLCJpc3MiOiJodHRwczovL2FwaS50b3Bjb2Rlci1kZXYuY29tIiwiaGFuZGxlIjoidGVzdDEiLCJleHAiOjIxNDc0ODM2NDgsInVzZXJJZCI6IjQwMTUyODU2IiwiaWF0IjoxNDYzMDc2MDg5LCJlbWFpbCI6InRlc3RAdG9wY29kZXIuY29tIiwianRpIjoiYjMzYjc3Y2QtYjUyZS00MGZlLTgzN2UtYmViOGUwYWU2YTRhIn0.PKv0QrMCPf0-ZPjv4PGWT7eXne54m7i9YX9eq-fceMU", + "enabled": true + } + ], + "_postman_variable_scope": "environment", + "_postman_exported_at": "2021-07-24T21:01:35.787Z", + "_postman_exported_using": "Postman/8.8.0" +} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 17a40e0..f617e1e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4,6 +4,11 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "@tootallnate/once": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", + "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==" + }, "@types/babel-types": { "version": "7.0.4", "resolved": "https://registry.npmjs.org/@types/babel-types/-/babel-types-7.0.4.tgz", @@ -539,11 +544,11 @@ "integrity": "sha512-32NDda82rhwD9/JBCCkB+MRYDp0oSvlo2IL6rQWA10PQi7tDUM3eqMSltXmY+Oyl/7N3P3qNtAlv7X0d9bI28w==" }, "axios": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.12.0.tgz", - "integrity": "sha1-uQewIhzDTsHJ+sGOx/B935V4W6Q=", + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.19.2.tgz", + "integrity": "sha512-fjgm5MvRHLhx+osE2xoekY70AhARk3a6hkN+3Io1jc00jtquGvxYlKlsFUhmUET0V5te6CcZI7lcv2Ym61mjHA==", "requires": { - "follow-redirects": "0.0.7" + "follow-redirects": "1.5.10" } }, "babel-runtime": { @@ -3114,12 +3119,21 @@ "dev": true }, "follow-redirects": { - "version": "0.0.7", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-0.0.7.tgz", - "integrity": "sha1-NLkLqyqRGqNHVx2pDyK9NuzYqRk=", + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz", + "integrity": "sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ==", "requires": { - "debug": "2.6.9", - "stream-consume": "0.1.1" + "debug": "3.1.0" + }, + "dependencies": { + "debug": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", + "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", + "requires": { + "ms": "2.0.0" + } + } } }, "for-in": { @@ -6265,8 +6279,7 @@ "lodash.clonedeep": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", - "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=", - "dev": true + "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=" }, "lodash.cond": { "version": "4.5.2", @@ -6847,6 +6860,21 @@ } } }, + "node-cache": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/node-cache/-/node-cache-5.1.2.tgz", + "integrity": "sha512-t1QzWwnk4sjLWaQAS8CHgOJ+RAfmHpxFWmc36IWTiWHQfs0w5JDMBS1b1ZxQteo0vVVuWJvIUKHDkkeK7vIGCg==", + "requires": { + "clone": "2.1.2" + }, + "dependencies": { + "clone": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz", + "integrity": "sha1-G39Ln1kfHo+DZwQBYANFoCiHQ18=" + } + } + }, "node-cron": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-1.2.1.tgz", @@ -9195,7 +9223,8 @@ "stream-consume": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/stream-consume/-/stream-consume-0.1.1.tgz", - "integrity": "sha512-tNa3hzgkjEP7XbCkbRXe1jpg+ievoa0O4SCFlMOYEscGSS4JJsckGL8swUyAa/ApGU3Ae4t6Honor4HhL+tRyg==" + "integrity": "sha512-tNa3hzgkjEP7XbCkbRXe1jpg+ievoa0O4SCFlMOYEscGSS4JJsckGL8swUyAa/ApGU3Ae4t6Honor4HhL+tRyg==", + "dev": true }, "stream-throttle": { "version": "0.1.3", @@ -9479,34 +9508,66 @@ } }, "tc-core-library-js": { - "version": "github:appirio-tech/tc-core-library-js#d16413db30b1eed21c0cf426e185bedb2329ddab", + "version": "github:appirio-tech/tc-core-library-js#0d8b4dfc6a1cb0aa10a7ea1b90ed58ba5f0e11b0", "requires": { "auth0-js": "9.6.1", - "axios": "0.12.0", + "axios": "0.19.2", "bunyan": "1.8.12", "jsonwebtoken": "8.5.1", - "jwks-rsa": "1.4.0", + "jwks-rsa": "1.12.3", "le_node": "1.7.1", "lodash": "4.17.10", "millisecond": "0.1.2", - "request": "2.88.0" + "request": "2.88.2" }, "dependencies": { + "@types/express-jwt": { + "version": "0.0.42", + "resolved": "https://registry.npmjs.org/@types/express-jwt/-/express-jwt-0.0.42.tgz", + "integrity": "sha512-WszgUddvM1t5dPpJ3LhWNH8kfNN8GPIBrAGxgIYXVCEGx6Bx4A036aAuf/r5WH9DIEdlmp7gHOYvSM6U87B0ag==", + "requires": { + "@types/express": "4.11.1", + "@types/express-unless": "0.0.32" + } + }, + "agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "requires": { + "debug": "4.3.2" + } + }, "ajv": { - "version": "6.10.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.0.tgz", - "integrity": "sha512-nffhOpkymDECQyR0mnsUtoCE8RlX38G0rYP+wgLWFyZuUyuuojSSvi/+euOiQBIn63whYwYVIIH1TvE3tu4OEg==", + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", "requires": { - "fast-deep-equal": "2.0.1", + "fast-deep-equal": "3.1.3", "fast-json-stable-stringify": "2.0.0", "json-schema-traverse": "0.4.1", "uri-js": "4.2.2" } }, "aws4": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", - "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==" + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.11.0.tgz", + "integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==" + }, + "debug": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", + "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", + "requires": { + "ms": "2.1.2" + }, + "dependencies": { + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } }, "extend": { "version": "3.0.2", @@ -9514,19 +9575,43 @@ "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" }, "fast-deep-equal": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", - "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=" + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" + }, + "follow-redirects": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.1.tgz", + "integrity": "sha512-HWqDgT7ZEkqRzBvc2s64vSZ/hfOceEol3ac/7tKwzuvEyWx3/4UegXh5oBOIotkGsObyk3xznnSRVADBgWSQVg==" }, "har-validator": { - "version": "5.1.3", - "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", - "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", + "version": "5.1.5", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.5.tgz", + "integrity": "sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==", "requires": { - "ajv": "6.10.0", + "ajv": "6.12.6", "har-schema": "2.0.0" } }, + "http-proxy-agent": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz", + "integrity": "sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==", + "requires": { + "@tootallnate/once": "1.1.2", + "agent-base": "6.0.2", + "debug": "4.3.2" + } + }, + "https-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", + "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", + "requires": { + "agent-base": "6.0.2", + "debug": "4.3.2" + } + }, "json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", @@ -9545,8 +9630,8 @@ "lodash.isplainobject": "4.0.6", "lodash.isstring": "4.0.1", "lodash.once": "4.1.1", - "ms": "2.1.1", - "semver": "5.7.0" + "ms": "2.1.3", + "semver": "5.7.1" } }, "jwa": { @@ -9560,16 +9645,30 @@ } }, "jwks-rsa": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/jwks-rsa/-/jwks-rsa-1.4.0.tgz", - "integrity": "sha512-6aUc+oTuqsLwIarfq3A0FqoD5LFSgveW5JO1uX2s0J8TJuOEcDc4NIMZAmVHO8tMHDw7CwOPzXF/9QhfOpOElA==", - "requires": { - "@types/express-jwt": "0.0.34", - "debug": "2.6.9", - "limiter": "1.1.3", - "lru-memoizer": "1.12.0", - "ms": "2.1.1", - "request": "2.88.0" + "version": "1.12.3", + "resolved": "https://registry.npmjs.org/jwks-rsa/-/jwks-rsa-1.12.3.tgz", + "integrity": "sha512-cFipFDeYYaO9FhhYJcZWX/IyZgc0+g316rcHnDpT2dNRNIE/lMOmWKKqp09TkJoYlNFzrEVODsR4GgXJMgWhnA==", + "requires": { + "@types/express-jwt": "0.0.42", + "axios": "0.21.1", + "debug": "4.3.2", + "http-proxy-agent": "4.0.1", + "https-proxy-agent": "5.0.0", + "jsonwebtoken": "8.5.1", + "limiter": "1.1.5", + "lru-memoizer": "2.1.4", + "ms": "2.1.3", + "proxy-from-env": "1.1.0" + }, + "dependencies": { + "axios": { + "version": "0.21.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz", + "integrity": "sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==", + "requires": { + "follow-redirects": "1.14.1" + } + } } }, "jws": { @@ -9581,86 +9680,110 @@ "safe-buffer": "5.1.1" } }, + "limiter": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/limiter/-/limiter-1.1.5.tgz", + "integrity": "sha512-FWWMIEOxz3GwUI4Ts/IvgVy6LPvoMPgjMdQ185nN6psJyBJ4yOpzqm695/h5umdLJg2vW3GR5iG11MAkR2AzJA==" + }, + "lru-memoizer": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/lru-memoizer/-/lru-memoizer-2.1.4.tgz", + "integrity": "sha512-IXAq50s4qwrOBrXJklY+KhgZF+5y98PDaNo0gi/v2KQBFLyWr+JyFvijZXkGKjQj/h9c0OwoE+JZbwUXce76hQ==", + "requires": { + "lodash.clonedeep": "4.5.0", + "lru-cache": "4.0.2" + } + }, "mime-db": { - "version": "1.38.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.38.0.tgz", - "integrity": "sha512-bqVioMFFzc2awcdJZIzR3HjZFX20QhilVS7hytkKrv7xFAn8bM1gzc/FOX2awLISvWe0PV8ptFKcon+wZ5qYkg==" + "version": "1.48.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.48.0.tgz", + "integrity": "sha512-FM3QwxV+TnZYQ2aRqhlKBMHxk10lTbMt3bBkMAp54ddrNeVSfcQYOOKuGuy3Ddrm38I04If834fOUSq1yzslJQ==" }, "mime-types": { - "version": "2.1.22", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.22.tgz", - "integrity": "sha512-aGl6TZGnhm/li6F7yx82bJiBZwgiEa4Hf6CNr8YO+r5UHr53tSTYZb102zyU50DOWWKeOv0uQLRL0/9EiKWCog==", + "version": "2.1.31", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.31.tgz", + "integrity": "sha512-XGZnNzm3QvgKxa8dpzyhFTHmpP3l5YNusmne07VUOXxou9CqUqYa/HBy124RqtVh/O2pECas/MOcsDgpilPOPg==", "requires": { - "mime-db": "1.38.0" + "mime-db": "1.48.0" } }, "ms": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", - "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "oauth-sign": { "version": "0.9.0", "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" }, + "proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, + "punycode": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" + }, "qs": { "version": "6.5.2", "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" }, "request": { - "version": "2.88.0", - "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", - "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", + "version": "2.88.2", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", + "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==", "requires": { "aws-sign2": "0.7.0", - "aws4": "1.8.0", + "aws4": "1.11.0", "caseless": "0.12.0", "combined-stream": "1.0.6", "extend": "3.0.2", "forever-agent": "0.6.1", "form-data": "2.3.2", - "har-validator": "5.1.3", + "har-validator": "5.1.5", "http-signature": "1.2.0", "is-typedarray": "1.0.0", "isstream": "0.1.2", "json-stringify-safe": "5.0.1", - "mime-types": "2.1.22", + "mime-types": "2.1.31", "oauth-sign": "0.9.0", "performance-now": "2.1.0", "qs": "6.5.2", - "safe-buffer": "5.1.2", - "tough-cookie": "2.4.3", + "safe-buffer": "5.2.1", + "tough-cookie": "2.5.0", "tunnel-agent": "0.6.0", - "uuid": "3.3.2" + "uuid": "3.4.0" }, "dependencies": { "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" } } }, "semver": { - "version": "5.7.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", - "integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==" + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" }, "tough-cookie": { - "version": "2.4.3", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", - "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", + "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", "requires": { "psl": "1.1.31", - "punycode": "1.4.1" + "punycode": "2.1.1" } }, "uuid": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" } } }, @@ -9796,6 +9919,14 @@ "integrity": "sha1-zu78cXp2xDFvEm0LnbqlXX598Bo=", "dev": true }, + "topcoder-healthcheck-dropin": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/topcoder-healthcheck-dropin/-/topcoder-healthcheck-dropin-1.0.3.tgz", + "integrity": "sha512-k8X84IC2NALu1v8cD3SZXY0MMZAMWw2uzHmjXDlgXwpS5xnXdwnVU+BpJWqg1uz1OuYDdeaAIPguqnhs7G6Y0A==", + "requires": { + "express": "4.16.3" + } + }, "topo": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/topo/-/topo-2.0.2.tgz", @@ -10032,6 +10163,11 @@ } } }, + "urijs": { + "version": "1.19.7", + "resolved": "https://registry.npmjs.org/urijs/-/urijs-1.19.7.tgz", + "integrity": "sha512-Id+IKjdU0Hx+7Zx717jwLPsPeUqz7rAtuVBRLLs+qn+J2nf9NGITWVCxcijgYxBqe83C7sqsQPs6H1pyz3x9gA==" + }, "urix": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/urix/-/urix-0.1.0.tgz", diff --git a/src/app.js b/src/app.js index 1d7d4bc..ff2a4b3 100644 --- a/src/app.js +++ b/src/app.js @@ -21,7 +21,7 @@ const healthcheck = require('topcoder-healthcheck-dropin'); // helps in health checking in case of unhandled rejection of promises const unhandledRejections = []; process.on('unhandledRejection', (reason, promise) => { - console.log('Unhandled Rejection at:', promise, 'reason:', reason); + logger.debug('Unhandled Rejection at:', promise, 'reason:', reason); // aborts the process to let the HA of the container to restart the task // process.abort(); unhandledRejections.push(promise); @@ -31,7 +31,7 @@ process.on('unhandledRejection', (reason, promise) => { // from the unhandledRejections array. We just remove the first element from the array as we only care // about the count every time an unhandled rejection promise is handled process.on('rejectionHandled', (promise) => { - console.log('Handled Rejection at:', promise); + logger.debug('Handled Rejection at:', promise); unhandledRejections.shift(); }); @@ -93,10 +93,10 @@ function startKafkaConsumer(handlers, notificationServiceHandlers) { }); }); - var latestSubscriptions = null; + let latestSubscriptions = null; const check = function () { - logger.debug("Checking health"); + logger.debug('Checking health'); if (unhandledRejections && unhandledRejections.length > 0) { logger.error('Found unhandled promises. Application is potentially in stalled state.'); return false; @@ -106,12 +106,12 @@ function startKafkaConsumer(handlers, notificationServiceHandlers) { return false; } let connected = true; - let currentSubscriptions = consumer.subscriptions; - for(var sIdx in currentSubscriptions) { + const currentSubscriptions = consumer.subscriptions; + for (const sIdx of currentSubscriptions) { // current subscription - let sub = currentSubscriptions[sIdx]; + const sub = currentSubscriptions[sIdx]; // previous subscription - let prevSub = latestSubscriptions ? latestSubscriptions[sIdx] : null; + const prevSub = latestSubscriptions ? latestSubscriptions[sIdx] : null; // levarage the `paused` field (https://github.com/oleksiyk/kafka/blob/master/lib/base_consumer.js#L66) to // determine if there was a possibility of an unhandled exception. If we find paused status for the same // topic in two consecutive health checks, we assume it was stuck because of unhandled error @@ -123,11 +123,11 @@ function startKafkaConsumer(handlers, notificationServiceHandlers) { // stores the latest subscription status in global variable latestSubscriptions = consumer.subscriptions; consumer.client.initialBrokers.forEach(conn => { - logger.debug(`url ${conn.server()} - connected=${conn.connected}`) - connected = conn.connected & connected + logger.debug(`url ${conn.server()} - connected=${conn.connected}`); + connected = conn.connected & connected; }); - return connected - } + return connected; + }; consumer .init() diff --git a/src/common/tcApiHelper.js b/src/common/tcApiHelper.js index ed67c59..0bd9ca5 100644 --- a/src/common/tcApiHelper.js +++ b/src/common/tcApiHelper.js @@ -108,12 +108,127 @@ function* sendMessageToBus(data) { }); } +/** + * Notify slack channel. + * @param {string} channel the slack channel name + * @param {string} text the message + */ +function* notifySlackChannel(channel, text) { + if (config.SLACK.NOTIFY) { + const token = config.SLACK.BOT_TOKEN; + const url = config.SLACK.URL; + const res = yield request + .post(url) + .set('Content-Type', 'application/json') + .set('Authorization', `Bearer ${token}`) + .send({ channel, text }) + .catch((err) => { + const errorDetails = _.get(err, 'message'); + throw new Error( + 'Error posting message to Slack API.' + + (errorDetails ? ' Server response: ' + errorDetails : '') + ); + }); + if (res.body.ok) { + logger.info(`Message posted successfully to channel: ${channel}`); + } else { + logger.error(`Error posting message to Slack API: ${JSON.stringify(res.body, null, 4)}`); + } + } else { + logger.info(`Slack message won't be sent to channel: ${channel}`); + } +} + +/** + * Check if notification is explicitly disabled for given notification type. + * @param {number} userId the user id + * @param {string} notificationType the notification type + * @param {string} serviceId the service id + * @returns {boolean} is notification enabled? + */ +function* checkNotificationSetting(userId, notificationType, serviceId) { + const settings = yield NotificationService.getSettings(userId); + if (settings.notifications[notificationType] + && settings.notifications[notificationType][serviceId] + && settings.notifications[notificationType][serviceId].enabled === 'no' + ) { + return false; + } + return true; +} + /** * Notify user via email. + * @param {Object} message the Kafka message payload + * @return {Object} notification details. + */ +function* notifyUserViaWeb(message) { + const notificationType = message.type; + const userId = message.details.userId; + // if web notification is explicitly disabled for current notification type do nothing + const allowed = yield checkNotificationSetting(userId, notificationType, constants.SETTINGS_WEB_SERVICE_ID); + if (!allowed) { + logger.verbose(`Notification '${notificationType}' won't be sent by '${constants.SETTINGS_WEB_SERVICE_ID}'` + + ` service to the userId '${userId}' due to his notification settings.`); + return; + } + return message.details; +} + +/** + * Notify user via email. + * @param {Object} message the Kafka message payload + */ +function* notifyUserViaEmail(message) { + const notificationType = message.type; + const topic = constants.BUS_API_EVENT.EMAIL.UNIVERSAL; + for (const recipient of message.details.recipients) { + const userId = recipient.userId; + // if email notification is explicitly disabled for current notification type do nothing + const allowed = yield checkNotificationSetting(userId, notificationType, constants.SETTINGS_EMAIL_SERVICE_ID); + if (!allowed) { + logger.verbose(`Notification '${notificationType}' won't be sent by '${constants.SETTINGS_EMAIL_SERVICE_ID}'` + + ` service to the userId '${userId}' due to his notification settings.`); + continue; + } + let userEmail; + // if dev mode for email is enabled then replace recipient email + if (config.ENABLE_DEV_MODE) { + userEmail = config.DEV_MODE_EMAIL; + } else { + userEmail = recipient.email; + if (!userEmail) { + logger.error(`Email not received for user: ${userId}`); + continue; + } + } + const recipients = [userEmail]; + const payload = { + from: message.details.from, + recipients, + cc: message.details.cc || [], + data: message.details.data || {}, + sendgrid_template_id: message.details.sendgridTemplateId, + version: message.details.version, + }; + // send email message to bus api. + yield sendMessageToBus({ + topic, + originator: 'tc-notifications', + timestamp: (new Date()).toISOString(), + 'mime-type': 'application/json', + payload, + }); + logger.info(`Successfully sent ${topic} event with body ${JSON.stringify(payload, null, 4)} to bus api`); + } +} + +/** + * Notify challenge user via email. * @param {Object} user the user * @param {Object} message the Kafka message JSON */ -function* notifyUserViaEmail(user, message) { +function* notifyChallengeUserViaEmail(user, message) { const notificationType = message.topic; const eventType = constants.BUS_API_EVENT.EMAIL.GENERAL; @@ -382,7 +497,11 @@ module.exports = { getUsersBySkills, getUsersByHandles, sendMessageToBus, + notifySlackChannel, + checkNotificationSetting, + notifyUserViaWeb, notifyUserViaEmail, + notifyChallengeUserViaEmail, getChallenge, notifyUsersOfMessage, getUsersInfoFromChallenge, diff --git a/src/controllers/NotificationController.js b/src/controllers/NotificationController.js index 5f91450..b885ab7 100644 --- a/src/controllers/NotificationController.js +++ b/src/controllers/NotificationController.js @@ -32,15 +32,15 @@ function* listNotifications(req, res) { * disabling v5 API feature temporarily for connect-app (backward compatibility) */ - //res.json(items); + // res.json(items); // TODO disable this and revert to original res.json({ items, offset: currentPage, limit: perPage, - totalCount: total - }) + totalCount: total, + }); } function* updateNotification(req, res) { diff --git a/src/hooks/hookBulkMessage.js b/src/hooks/hookBulkMessage.js index 70801c0..97d5b91 100644 --- a/src/hooks/hookBulkMessage.js +++ b/src/hooks/hookBulkMessage.js @@ -2,182 +2,180 @@ * Hook to insert broadcast notification into database for a user. */ -'use strict' +'use strict'; -const _ = require('lodash') -const logger = require('../common/logger') -const models = require('../models') -const api = require('../common/broadcastAPIHelper') +const _ = require('lodash'); +const logger = require('../common/logger'); +const models = require('../models'); +const api = require('../common/broadcastAPIHelper'); -const logPrefix = "BulkNotificationHook: " +const logPrefix = 'BulkNotificationHook: '; /** * CREATE NEW TABLES IF NOT EXISTS */ models.BulkMessages.sync().then((t) => { - models.BulkMessageUserRefs.sync() -}) + models.BulkMessageUserRefs.sync(); +}); /** - * Main function - * @param {Integer} userId + * Main function + * @param {Integer} userId */ async function checkBulkMessageForUser(userId) { - return new Promise(function (resolve, reject) { - models.BulkMessages.count().then(function (tBulkMessages) { - if (tBulkMessages > 0) { + return new Promise(function (resolve, reject) { + models.BulkMessages.count().then(function (tBulkMessages) { + if (tBulkMessages > 0) { // the condition can help to optimize the execution - models.BulkMessageUserRefs.count({ - where: { - user_id: userId - } - }).then(async function (tUserRefs) { - let result = true - if (tUserRefs < tBulkMessages) { - logger.info(`${logPrefix} Need to sync broadcast message for current user ${userId}`) - result = await syncBulkMessageForUser(userId) - } - resolve(result) // resolve here - }).catch((e) => { - reject(`${logPrefix} Failed to check total userRefs condition. Error: ${e}`) - }) - } else { - resolve(true) - } + models.BulkMessageUserRefs.count({ + where: { + user_id: userId, + }, + }).then(async function (tUserRefs) { + let result = true; + if (tUserRefs < tBulkMessages) { + logger.info(`${logPrefix} Need to sync broadcast message for current user ${userId}`); + result = await syncBulkMessageForUser(userId); + } + resolve(result); // resolve here }).catch((e) => { - logger.error(`${logPrefix} Failed to check total broadcast message condition. Error: `, e) - reject(e) - }) - }) + reject(`${logPrefix} Failed to check total userRefs condition. Error: ${e}`); + }); + } else { + resolve(true); + } + }).catch((e) => { + logger.error(`${logPrefix} Failed to check total broadcast message condition. Error: `, e); + reject(e); + }); + }); } /** - * Helper function - * @param {Integer} userId + * Helper function + * @param {Integer} userId */ async function syncBulkMessageForUser(userId) { - - return new Promise(function (resolve, reject) { + return new Promise(function (resolve, reject) { /** * Check if all bulk mesaages processed for current user or not */ - let q = "SELECT a.* FROM bulk_messages AS a " + - " LEFT OUTER JOIN (SELECT id as refid, bulk_message_id " + - " FROM bulk_message_user_refs AS bmur WHERE bmur.user_id=$1)" + - " AS b ON a.id=b.bulk_message_id WHERE b.refid IS NULL" - let memberInfo = [] - let userGroupInfo = [] - models.sequelize.query(q, { bind: [userId] }) + let q = 'SELECT a.* FROM bulk_messages AS a ' + + ' LEFT OUTER JOIN (SELECT id as refid, bulk_message_id ' + + ' FROM bulk_message_user_refs AS bmur WHERE bmur.user_id=$1)' + + ' AS b ON a.id=b.bulk_message_id WHERE b.refid IS NULL'; + let memberInfo = []; + let userGroupInfo = []; + models.sequelize.query(q, { bind: [userId] }) .then(async function (res) { - try { - memberInfo = await api.getMemberInfo(userId) - userGroupInfo = await api.getUserGroup(userId) - } catch (e) { - reject(`${logPrefix} Failed to get member/group info: ${e}`) - } - Promise.all(res[0].map((r) => isBroadCastMessageForUser(userId, r, memberInfo, userGroupInfo))) + try { + memberInfo = await api.getMemberInfo(userId); + userGroupInfo = await api.getUserGroup(userId); + } catch (e) { + reject(`${logPrefix} Failed to get member/group info: ${e}`); + } + Promise.all(res[0].map((r) => isBroadCastMessageForUser(userId, r, memberInfo, userGroupInfo))) .then((results) => { - Promise.all(results.map((o) => { - if (o.result) { - return createNotificationForUser(userId, o.record) - } else { - return insertUserRefs(userId, o.record.id, null) - } - })).then((results) => { - resolve(results) - }).catch((e) => { - reject(e) - }) + Promise.all(results.map((o) => { + if (o.result) { + return createNotificationForUser(userId, o.record); + } else { + return insertUserRefs(userId, o.record.id, null); + } + })).then((results) => { + resolve(results); + }).catch((e) => { + reject(e); + }); }).catch((e) => { - reject(e) - }) + reject(e); + }); }).catch((e) => { - reject(`${logPrefix} Failed to check bulk message condition: error - ${e}`) - }) - }) + reject(`${logPrefix} Failed to check bulk message condition: error - ${e}`); + }); + }); } /** - * Helper function - * Check if current user in broadcast recipent group - * @param {Integer} userId - * @param {Object} bulkMessage + * Helper function + * Check if current user in broadcast recipent group + * @param {Integer} userId + * @param {Object} bulkMessage * @param {Object} memberInfo * @param {Object} userGroupInfo * - * @retun promise + * @retun promise */ async function isBroadCastMessageForUser(userId, bulkMessage, memberInfo, userGroupInfo) { - return api.checkBroadcastMessageForUser(userId, bulkMessage, memberInfo, userGroupInfo) + return api.checkBroadcastMessageForUser(userId, bulkMessage, memberInfo, userGroupInfo); } /** * Helper function - Insert in bulkMessage user reference table - * - * @param {Integer} userId - * @param {Integer} bulkMessageId + * + * @param {Integer} userId + * @param {Integer} bulkMessageId * @param {Object} notificationObj */ async function insertUserRefs(userId, bulkMessageId, notificationObj) { - let notificationId = null - if (notificationObj) { - notificationId = notificationObj.id - } - try { - const r = await models.BulkMessageUserRefs.create({ - bulk_message_id: bulkMessageId, - user_id: userId, - notification_id: notificationId, - }) - logger.info(`${logPrefix} Inserted userRef record for bulk message id ${r.id} for current user ${userId}`) - return r - } catch (e) { - logger.error(`${logPrefix} Failed to insert userRef record for user: ${userId}, error: ${e}`) - if (notificationId && notificationObj) { - try { - await notificationObj.destroy() - logger.info(`Deleted/reverted duplicate/ref-transaction failed, broadcast notification ${notificationId} for user: ${userId}`) - } catch (error) { - logger.error(`Error in deleting duplicate notification record, ${error}`) - } - - } - throw new Error(`insertUserRefs() : ${e}`) + let notificationId = null; + if (notificationObj) { + notificationId = notificationObj.id; + } + try { + const r = await models.BulkMessageUserRefs.create({ + bulk_message_id: bulkMessageId, + user_id: userId, + notification_id: notificationId, + }); + logger.info(`${logPrefix} Inserted userRef record for bulk message id ${r.id} for current user ${userId}`); + return r; + } catch (e) { + logger.error(`${logPrefix} Failed to insert userRef record for user: ${userId}, error: ${e}`); + if (notificationId && notificationObj) { + try { + await notificationObj.destroy(); + logger.info(`Deleted/reverted duplicate/ref-transaction failed, broadcast notification ${notificationId} for user: ${userId}`); + } catch (error) { + logger.error(`Error in deleting duplicate notification record, ${error}`); + } } + throw new Error(`insertUserRefs() : ${e}`); + } } /** - * Helper function - * @param {Integer} userId - * @param {Object} bulkMessage + * Helper function + * @param {Integer} userId + * @param {Object} bulkMessage */ async function createNotificationForUser(userId, bulkMessage) { - try { - const n = await models.Notification.create({ - userId: userId, - type: bulkMessage.type, - contents: { - id: bulkMessage.id, /** broadcast message id */ - message: bulkMessage.message, /** broadcast message */ - group: 'broadcast', - title: 'Broadcast Message', - }, - read: false, - seen: false, - version: null, - }) - logger.info(`${logPrefix} Inserted notification record ${n.id} for current user ${userId}`) + try { + const n = await models.Notification.create({ + userId, + type: bulkMessage.type, + contents: { + id: bulkMessage.id, /** broadcast message id */ + message: bulkMessage.message, /** broadcast message */ + group: 'broadcast', + title: 'Broadcast Message', + }, + read: false, + seen: false, + version: null, + }); + logger.info(`${logPrefix} Inserted notification record ${n.id} for current user ${userId}`); // TODO need to be in transaction so that rollback will be possible - const result = await insertUserRefs(userId, bulkMessage.id, n) - return result - } catch (e) { - logger.error(`${logPrefix} insert broadcast notification error: ${e} `) - throw new Error(`createNotificationForUser() : ${e}`) - } + const result = await insertUserRefs(userId, bulkMessage.id, n); + return result; + } catch (e) { + logger.error(`${logPrefix} insert broadcast notification error: ${e} `); + throw new Error(`createNotificationForUser() : ${e}`); + } } // Exports module.exports = { - checkBulkMessageForUser, -}; \ No newline at end of file + checkBulkMessageForUser, +}; diff --git a/src/hooks/index.js b/src/hooks/index.js index 5dcc1a6..ff68fed 100644 --- a/src/hooks/index.js +++ b/src/hooks/index.js @@ -3,13 +3,13 @@ */ /** - * Hook implementation + * Hook implementation * * @author TCSCODER * @version 1.0 */ -const hookBulkMessage = require("./hookBulkMessage") +const hookBulkMessage = require('./hookBulkMessage'); module.exports = { diff --git a/src/models/BulkMessageUserRefs.js b/src/models/BulkMessageUserRefs.js index 7f3baf3..db97f1d 100644 --- a/src/models/BulkMessageUserRefs.js +++ b/src/models/BulkMessageUserRefs.js @@ -11,24 +11,24 @@ module.exports = (sequelize, DataTypes) => sequelize.define('bulk_message_user_refs', { - id: { type: DataTypes.BIGINT, primaryKey: true, autoIncrement: true }, - bulk_message_id: { - type: DataTypes.BIGINT, - allowNull: false, - references: { - model: 'bulk_messages', - key: 'id' - } + id: { type: DataTypes.BIGINT, primaryKey: true, autoIncrement: true }, + bulk_message_id: { + type: DataTypes.BIGINT, + allowNull: false, + references: { + model: 'bulk_messages', + key: 'id', }, - notification_id: { - type: DataTypes.BIGINT, - allowNull: true, - references: { - model: 'Notifications', - key: 'id' - } + }, + notification_id: { + type: DataTypes.BIGINT, + allowNull: true, + references: { + model: 'Notifications', + key: 'id', }, - user_id: { type: DataTypes.BIGINT, allowNull: false } + }, + user_id: { type: DataTypes.BIGINT, allowNull: false }, }, {}); // sequelize will generate and manage createdAt, updatedAt fields diff --git a/src/processors/broadcast/bulkNotificationHandler.js b/src/processors/broadcast/bulkNotificationHandler.js index 93d1e04..f663d52 100644 --- a/src/processors/broadcast/bulkNotificationHandler.js +++ b/src/processors/broadcast/bulkNotificationHandler.js @@ -1,10 +1,10 @@ /** * Bulk notification handler. */ -const joi = require('joi') -const co = require('co') -const models = require('../../models') -const logger = require('../../common/logger') +const joi = require('joi'); +const co = require('co'); +const models = require('../../models'); +const logger = require('../../common/logger'); /** * Handle Kafka JSON message of broadcast. @@ -14,22 +14,23 @@ const logger = require('../../common/logger') * * @return {Promise} promise resolved to notifications */ +// eslint-disable-next-line no-unused-vars const handle = (message, ruleSets) => co(function* () { try { const bm = yield models.BulkMessages.create({ type: message.topic, message: message.payload.message, recipients: message.payload.recipients, - }) - logger.info("Broadcast message recieved and inserted in db with id:", bm.id) + }); + logger.info('Broadcast message recieved and inserted in db with id:', bm.id); } catch (e) { - logger.error(`Broadcast processor failed in db operation. Error: ${e}`) + logger.error(`Broadcast processor failed in db operation. Error: ${e}`); } - return [] // this point of time, send empty notification object + return []; // this point of time, send empty notification object }); /** - * validate kafka payload + * validate kafka payload */ handle.schema = { message: joi.object().keys({ @@ -49,4 +50,4 @@ module.exports = { handle, }; -logger.buildService(module.exports); \ No newline at end of file +logger.buildService(module.exports); diff --git a/src/processors/index.js b/src/processors/index.js index a0243be..a4b22b3 100644 --- a/src/processors/index.js +++ b/src/processors/index.js @@ -9,6 +9,7 @@ const ChallengeHandler = require('./challenge/ChallengeHandler'); const AutoPilotHandler = require('./challenge/AutoPilotHandler'); const SubmissionHandler = require('./challenge/SubmissionHandler'); const BulkNotificationHandler = require('./broadcast/bulkNotificationHandler'); +const UniversalNotificationHandler = require('./universal/universalNotificationHandler'); // Exports module.exports = { @@ -18,4 +19,5 @@ module.exports = { handleAutoPilot: AutoPilotHandler.handle, handleSubmission: SubmissionHandler.handle, handleBulkNotification: BulkNotificationHandler.handle, + handleUniversalNotification: UniversalNotificationHandler.handle, }; diff --git a/src/processors/universal/universalNotificationHandler.js b/src/processors/universal/universalNotificationHandler.js new file mode 100644 index 0000000..4b0b5e3 --- /dev/null +++ b/src/processors/universal/universalNotificationHandler.js @@ -0,0 +1,20 @@ +/** + * Universal notification handler. + */ + const co = require('co'); + const service = require('../../services/UniversalNotificationService'); + + /** + * Handle Kafka JSON message of notifications requested. + * + * @param {Object} message the Kafka JSON message + * + * @return {Promise} promise resolved to notifications + */ + const handle = (message) => co(function* () { + return yield service.handle(message); + }); + + module.exports = { + handle, + }; diff --git a/src/services/NotificationService.js b/src/services/NotificationService.js index 6fa81f8..2461306 100644 --- a/src/services/NotificationService.js +++ b/src/services/NotificationService.js @@ -61,8 +61,8 @@ getSettings.schema = { function* saveNotificationSetting(entry, userId) { const setting = yield models.NotificationSetting.findOne({ where: { - userId, topic: entry.topic, serviceId: entry.serviceId, name: entry.name - } + userId, topic: entry.topic, serviceId: entry.serviceId, name: entry.name, + }, }); if (setting) { setting.value = entry.value; @@ -86,8 +86,8 @@ function* saveNotificationSetting(entry, userId) { function* saveServiceSetting(entry, userId) { const setting = yield models.ServiceSettings.findOne({ where: { - userId, serviceId: entry.serviceId, name: entry.name - } + userId, serviceId: entry.serviceId, name: entry.name, + }, }); if (setting) { setting.value = entry.value; @@ -192,9 +192,10 @@ function* listNotifications(query, userId) { const filter = { where: { userId, - }, offset, limit, order: [['createdAt', 'DESC']] + }, offset, limit, order: [['createdAt', 'DESC']], }; + // eslint-disable-next-line default-case switch (query.platform) { case 'connect': filter.where.type = { $like: 'connect.notification.%' }; @@ -206,9 +207,9 @@ function* listNotifications(query, userId) { if (config.ENABLE_HOOK_BULK_NOTIFICATION) { try { - yield hooks.hookBulkMessage.checkBulkMessageForUser(userId) + yield hooks.hookBulkMessage.checkBulkMessageForUser(userId); } catch (e) { - logger.error(`Error in calling bulk notification hook: ${e}`) + logger.error(`Error in calling bulk notification hook: ${e}`); } } diff --git a/src/services/UniversalNotificationService.js b/src/services/UniversalNotificationService.js new file mode 100644 index 0000000..3f41d86 --- /dev/null +++ b/src/services/UniversalNotificationService.js @@ -0,0 +1,126 @@ +/** + * Universal notification handler service. + */ + +'use strict'; + +const joi = require('joi'); +const logger = require('../common/logger'); +const tcApiHelper = require('../common/tcApiHelper'); +const constants = require('../../constants'); + +const emailSchema = joi.object().keys({ + serviceId: joi.string().valid(constants.SETTINGS_EMAIL_SERVICE_ID).required(), + type: joi.string().required(), + details: joi.object().keys({ + from: joi.string().email().required(), + recipients: joi.array().items( + joi.object().keys({ + userId: joi.number().integer().required(), + email: joi.string().email().required(), + }).required() + ).min(1).required(), + cc: joi.array().items( + joi.object().keys({ + userId: joi.number().integer(), + email: joi.string().email().required(), + }).required() + ), + data: joi.object().keys({ + subject: joi.string(), + body: joi.string(), + }).unknown(), + sendgridTemplateId: joi.string().required(), + version: joi.string().required(), + }).required(), +}).required(); + +const slackSchema = joi.object().keys({ + serviceId: joi.string().valid(constants.SETTINGS_SLACK_SERVICE_ID).required(), + type: joi.string().required(), + details: joi.object().keys({ + channel: joi.string().required(), + text: joi.string().required(), + }).required(), +}).required(); + +const webSchema = joi.object().keys({ + serviceId: joi.string().valid(constants.SETTINGS_WEB_SERVICE_ID).required(), + type: joi.string().required(), + details: joi.object().keys({ + userId: joi.number().integer().required(), + contents: joi.object(), + version: joi.number().integer().required(), + }).required(), +}).required(); + +function validator(data, schema) { + const validationResult = schema.validate(data); + if (validationResult.error) { + logger.error(validationResult.error.message); + return false; + } + return true; +} + +/** + * Handle notification message + * @param {Object} message the Kafka message + * @returns {Array} the notifications + */ +function* handle(message) { + const notifications = []; + for (const data of message.payload) { + try { + switch (data.serviceId) { + case constants.SETTINGS_EMAIL_SERVICE_ID: + if (validator(data, emailSchema)) { + yield tcApiHelper.notifyUserViaEmail(data); + } + break; + case constants.SETTINGS_SLACK_SERVICE_ID: + if (validator(data, slackSchema)) { + yield tcApiHelper.notifySlackChannel(data.details.channel, data.details.text); + } + break; + case constants.SETTINGS_WEB_SERVICE_ID: + if (validator(data, webSchema)) { + const notification = yield tcApiHelper.notifyUserViaWeb(data); + if (notification) { + notifications.push(notification); + } + } + break; + default: + break; + } + } catch (err) { + logger.logFullError(err); + } + } + return notifications; +} + +handle.schema = { + message: joi.object().keys({ + topic: joi.string().required(), + originator: joi.string().required(), + timestamp: joi.date().required(), + 'mime-type': joi.string().required(), + payload: joi.array().items( + joi.object().keys({ + serviceId: joi.string().valid( + constants.SETTINGS_EMAIL_SERVICE_ID, + constants.SETTINGS_SLACK_SERVICE_ID, + constants.SETTINGS_WEB_SERVICE_ID).required(), + }).unknown() + ).min(1).required(), + }).required(), +}; + +// Exports +module.exports = { + handle, +}; + +logger.buildService(module.exports); diff --git a/test/checkHooks.js b/test/checkHooks.js index 6371283..6f4844d 100644 --- a/test/checkHooks.js +++ b/test/checkHooks.js @@ -1,3 +1,3 @@ -const bulkhook = require("../src/hooks/hookBulkMessage") +const bulkhook = require('../src/hooks/hookBulkMessage'); -bulkhook.checkBulkMessageForUser(123) \ No newline at end of file +bulkhook.checkBulkMessageForUser(123); From 93e4665f30c440bdce2f530b0199b89031ac68e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C4=B0=C5=9Fbilir?= Date: Thu, 5 Aug 2021 05:26:28 +0300 Subject: [PATCH 02/13] update: improve universal notifications payload --- config/default.js | 5 +- src/common/tcApiHelper.js | 144 ++++++++++++++++--- src/services/UniversalNotificationService.js | 129 ++++++++++++++--- 3 files changed, 241 insertions(+), 37 deletions(-) diff --git a/config/default.js b/config/default.js index 912c909..1ee9a67 100644 --- a/config/default.js +++ b/config/default.js @@ -28,9 +28,10 @@ module.exports = { KAFKA_CLIENT_CERT_KEY: process.env.KAFKA_CLIENT_CERT_KEY ? process.env.KAFKA_CLIENT_CERT_KEY.replace('\\n', '\n') : null, - TC_API_V3_BASE_URL: process.env.TC_API_V3_BASE_URL || '', + TC_API_V3_BASE_URL: process.env.TC_API_V3_BASE_URL || 'http://api.topcoder-dev.com/v3', TC_API_V4_BASE_URL: process.env.TC_API_V4_BASE_URL || '', TC_API_V5_BASE_URL: process.env.TC_API_V5_BASE_URL || '', + TC_API_V5_USERS_URL: process.env.TC_API_V5_USERS_URL || 'https://api.topcoder-dev.com/v5/users', API_CONTEXT_PATH: process.env.API_CONTEXT_PATH || '/v5/notifications', TC_API_BASE_URL: process.env.TC_API_BASE_URL || '', @@ -135,7 +136,7 @@ module.exports = { // email notification service related variables ENV: process.env.ENV, ENABLE_EMAILS: process.env.ENABLE_EMAILS ? Boolean(process.env.ENABLE_EMAILS) : false, - ENABLE_DEV_MODE: process.env.ENABLE_DEV_MODE ? Boolean(process.env.ENABLE_DEV_MODE) : true, + ENABLE_DEV_MODE: process.env.ENABLE_DEV_MODE === 'true', DEV_MODE_EMAIL: process.env.DEV_MODE_EMAIL, DEFAULT_REPLY_EMAIL: process.env.DEFAULT_REPLY_EMAIL, ENABLE_HOOK_BULK_NOTIFICATION: process.env.ENABLE_HOOK_BULK_NOTIFICATION || false, diff --git a/src/common/tcApiHelper.js b/src/common/tcApiHelper.js index 0bd9ca5..346896f 100644 --- a/src/common/tcApiHelper.js +++ b/src/common/tcApiHelper.js @@ -88,6 +88,103 @@ function* getUsersByHandles(handles) { return yield searchUsersByQuery(query); } +/** + * Get users by handles or userIds. + * @param {Array} handles the objects that has user handles. + * @param {Array} userIds the objects that has userIds. + * @returns {Array} the matched users + */ +function* getUsersByHandlesAndUserIds(handles, userIds) { + if ((!handles || handles.length === 0) && (!userIds || userIds.length === 0)) { + return []; + } + const handlesQuery = _.map(handles, h => `handleLower:${h.handle.toLowerCase()}`); + const userIdsQuery = _.map(userIds, u => `userId:${u.userId}`); + const query = _.concat(handlesQuery, userIdsQuery).join(URI.encodeQuery(' OR ', 'utf8')); + try { + return yield searchUsersByQuery(query); + } catch (err) { + const error = new Error(err.response.text); + error.status = err.status; + throw error; + } +} + +/** + * Search users by query string. + * @param {String} query the query string + * @returns {Array} the matched users + */ +function* searchUsersByEmailQuery(query) { + const token = yield getM2MToken(); + const res = yield request + .get(`${ + config.TC_API_V3_BASE_URL + }/users?filter=${ + query + }&fields=id,email,handle`) + .set('Authorization', `Bearer ${token}`); + if (!_.get(res, 'body.result.success')) { + throw new Error(`Failed to search users by query: ${query}`); + } + const records = _.get(res, 'body.result.content') || []; + + logger.verbose(`Searched users: ${JSON.stringify(records, null, 4)}`); + return records; +} + +/** + * Get users by emails. + * @param {Array} emails the objects that has user emails. + * @returns {Array} the matched users + */ +function* getUsersByEmails(emails) { + if (!emails || emails.length === 0) { + return []; + } + const users = []; + try { + for (const email of emails) { + const query = `email%3D${email.email}`; + const result = yield searchUsersByEmailQuery(query); + users.push(...result); + } + return users; + } catch (err) { + const error = new Error(err.response.text); + error.status = err.status; + throw error; + } +} + +/** + * Get users by uuid. + * @param {Array} ids the objects that has user uuids. + * @returns {Array} the matched users + */ +function* getUsersByUserUUIDs(ids) { + if (!ids || ids.length === 0) { + return []; + } + const users = []; + const token = yield getM2MToken(); + try { + for (const id of ids) { + const res = yield request + .get(`${config.TC_API_V5_USERS_URL}/${id}`) + .set('Authorization', `Bearer ${token}`); + const user = res.body; + logger.verbose(`Searched users: ${JSON.stringify(user, null, 4)}`); + users.push(user); + } + return users; + } catch (err) { + const error = new Error(err.response.text); + error.status = err.status; + throw error; + } +} + /** * Send message to bus. * @param {Object} data the data to send @@ -158,21 +255,30 @@ function* checkNotificationSetting(userId, notificationType, serviceId) { } /** - * Notify user via email. + * Notify user via web. * @param {Object} message the Kafka message payload - * @return {Object} notification details. + * @return {Array} notification details. */ function* notifyUserViaWeb(message) { const notificationType = message.type; - const userId = message.details.userId; - // if web notification is explicitly disabled for current notification type do nothing - const allowed = yield checkNotificationSetting(userId, notificationType, constants.SETTINGS_WEB_SERVICE_ID); - if (!allowed) { - logger.verbose(`Notification '${notificationType}' won't be sent by '${constants.SETTINGS_WEB_SERVICE_ID}'` + const notifications = []; + for (const recipient of message.details.recipients) { + const userId = recipient.userId; + if (_.isUndefined(userId)) { + logger.error(`userId not received for user: ${JSON.stringify(recipient, null, 4)}`); + continue; + } + // if web notification is explicitly disabled for current notification type do nothing + const allowed = yield checkNotificationSetting(userId, notificationType, constants.SETTINGS_WEB_SERVICE_ID); + if (!allowed) { + logger.verbose(`Notification '${notificationType}' won't be sent by '${constants.SETTINGS_WEB_SERVICE_ID}'` + ` service to the userId '${userId}' due to his notification settings.`); - return; + continue; + } + notifications.push(_.assign({}, _.pick(message.details, ['contents', 'version']), { userId })); } - return message.details; + + return notifications; } /** @@ -184,13 +290,6 @@ function* notifyUserViaEmail(message) { const topic = constants.BUS_API_EVENT.EMAIL.UNIVERSAL; for (const recipient of message.details.recipients) { const userId = recipient.userId; - // if email notification is explicitly disabled for current notification type do nothing - const allowed = yield checkNotificationSetting(userId, notificationType, constants.SETTINGS_EMAIL_SERVICE_ID); - if (!allowed) { - logger.verbose(`Notification '${notificationType}' won't be sent by '${constants.SETTINGS_EMAIL_SERVICE_ID}'` - + ` service to the userId '${userId}' due to his notification settings.`); - continue; - } let userEmail; // if dev mode for email is enabled then replace recipient email if (config.ENABLE_DEV_MODE) { @@ -202,6 +301,16 @@ function* notifyUserViaEmail(message) { continue; } } + // skip checking notification setting if userId is not found. + if (!_.isUndefined(userId)) { + // if email notification is explicitly disabled for current notification type do nothing + const allowed = yield checkNotificationSetting(userId, notificationType, constants.SETTINGS_EMAIL_SERVICE_ID); + if (!allowed) { + logger.verbose(`Notification '${notificationType}' won't be sent by '${constants.SETTINGS_EMAIL_SERVICE_ID}'` + + ` service to the userId '${userId}' due to his notification settings.`); + continue; + } + } const recipients = [userEmail]; const payload = { from: message.details.from, @@ -496,6 +605,9 @@ module.exports = { getM2MToken, getUsersBySkills, getUsersByHandles, + getUsersByHandlesAndUserIds, + getUsersByEmails, + getUsersByUserUUIDs, sendMessageToBus, notifySlackChannel, checkNotificationSetting, diff --git a/src/services/UniversalNotificationService.js b/src/services/UniversalNotificationService.js index 3f41d86..eb79ded 100644 --- a/src/services/UniversalNotificationService.js +++ b/src/services/UniversalNotificationService.js @@ -3,7 +3,7 @@ */ 'use strict'; - +const _ = require('lodash'); const joi = require('joi'); const logger = require('../common/logger'); const tcApiHelper = require('../common/tcApiHelper'); @@ -16,15 +16,19 @@ const emailSchema = joi.object().keys({ from: joi.string().email().required(), recipients: joi.array().items( joi.object().keys({ - userId: joi.number().integer().required(), - email: joi.string().email().required(), - }).required() + userId: joi.number().integer(), + userUUID: joi.string().uuid(), + email: joi.string().email(), + handle: joi.string(), + }).min(1).required() ).min(1).required(), cc: joi.array().items( joi.object().keys({ userId: joi.number().integer(), - email: joi.string().email().required(), - }).required() + userUUID: joi.string().uuid(), + email: joi.string().email(), + handle: joi.string(), + }).min(1).required() ), data: joi.object().keys({ subject: joi.string(), @@ -48,7 +52,14 @@ const webSchema = joi.object().keys({ serviceId: joi.string().valid(constants.SETTINGS_WEB_SERVICE_ID).required(), type: joi.string().required(), details: joi.object().keys({ - userId: joi.number().integer().required(), + recipients: joi.array().items( + joi.object().keys({ + userId: joi.number().integer(), + userUUID: joi.string().uuid(), + email: joi.string().email(), + handle: joi.string(), + }).min(1).required() + ).min(1).required(), contents: joi.object(), version: joi.number().integer().required(), }).required(), @@ -63,6 +74,82 @@ function validator(data, schema) { return true; } +function* completeMissingFields(details, findEmail, findUserId) { + const getFieldsByUserId = []; + const getFieldsByHandle = []; + const getFieldsByUserUUID = []; + const getFieldsByEmail = []; + function findMissingFields(data, email, userId) { + for (const recipient of data) { + if (_.isUndefined(recipient.email) && email) { + if (!_.isUndefined(recipient.userId)) { + getFieldsByUserId.push(recipient); + } else if (!_.isUndefined(recipient.handle)) { + getFieldsByHandle.push(recipient); + } else { + getFieldsByUserUUID.push(recipient); + } + } else if (_.isUndefined(recipient.userId) && userId) { + if (!_.isUndefined(recipient.handle)) { + getFieldsByHandle.push(recipient); + } else if (!_.isUndefined(recipient.email)) { + getFieldsByEmail.push(recipient); + } else { + getFieldsByUserUUID.push(recipient); + } + } + } + } + + findMissingFields(details.recipients, findEmail, findUserId); + if (_.isArray(details.cc) && !_.isEmpty(details.cc)) { + findMissingFields(details.cc, findEmail, false); + } + const foundUsersByHandleOrId = yield tcApiHelper.getUsersByHandlesAndUserIds(getFieldsByHandle, getFieldsByUserId); + if (!_.isEmpty(foundUsersByHandleOrId)) { + for (const user of [...getFieldsByUserId, ...getFieldsByHandle]) { + const found = _.find(foundUsersByHandleOrId, !_.isUndefined(user.handle) + ? ['handle', user.handle] : ['userId', user.userId]) || {}; + if (!_.isUndefined(found.email) && _.isUndefined(user.email)) { + _.assign(user, { email: found.email }); + } + if (!_.isUndefined(found.userId) && _.isUndefined(user.userId)) { + _.assign(user, { userId: found.userId }); + } + } + } + const foundUsersByEmail = yield tcApiHelper.getUsersByEmails(getFieldsByEmail); + if (!_.isEmpty(foundUsersByEmail)) { + for (const user of getFieldsByEmail) { + const found = _.find(foundUsersByEmail, ['email', user.email]) || {}; + if (!_.isUndefined(found.id)) { + _.assign(user, { userId: found.id }); + } + } + } + const foundUsersByUUID = yield tcApiHelper.getUsersByUserUUIDs(getFieldsByUserUUID); + if (!_.isEmpty(foundUsersByUUID)) { + for (const user of getFieldsByUserUUID) { + const found = _.find(foundUsersByUUID, ['id', user.userUUID]) || {}; + if (!_.isUndefined(found.externalProfiles) && !_.isEmpty(found.externalProfiles)) { + _.assign(user, { userId: _.toInteger(_.get(found.externalProfiles[0], 'externalId')) }); + } + } + if (findEmail) { + const usersHaveId = _.filter(getFieldsByUserUUID, u => !_.isUndefined(u.userId)); + const foundUsersById = yield tcApiHelper.getUsersByHandlesAndUserIds([], usersHaveId); + if (!_.isEmpty(foundUsersById)) { + for (const user of getFieldsByUserUUID) { + const found = _.find(foundUsersById, ['userId', user.userId]) || {}; + if (!_.isUndefined(found.email)) { + _.assign(user, { email: found.email }); + } + } + } + } + } +} + /** * Handle notification message * @param {Object} message the Kafka message @@ -70,11 +157,12 @@ function validator(data, schema) { */ function* handle(message) { const notifications = []; - for (const data of message.payload) { + for (const data of message.payload.notifications) { try { switch (data.serviceId) { case constants.SETTINGS_EMAIL_SERVICE_ID: if (validator(data, emailSchema)) { + yield completeMissingFields(data.details, true, true); yield tcApiHelper.notifyUserViaEmail(data); } break; @@ -85,9 +173,10 @@ function* handle(message) { break; case constants.SETTINGS_WEB_SERVICE_ID: if (validator(data, webSchema)) { - const notification = yield tcApiHelper.notifyUserViaWeb(data); - if (notification) { - notifications.push(notification); + yield completeMissingFields(data.details, false, true); + const _notifications = yield tcApiHelper.notifyUserViaWeb(data); + if (_notifications) { + notifications.push(..._notifications); } } break; @@ -107,14 +196,16 @@ handle.schema = { originator: joi.string().required(), timestamp: joi.date().required(), 'mime-type': joi.string().required(), - payload: joi.array().items( - joi.object().keys({ - serviceId: joi.string().valid( - constants.SETTINGS_EMAIL_SERVICE_ID, - constants.SETTINGS_SLACK_SERVICE_ID, - constants.SETTINGS_WEB_SERVICE_ID).required(), - }).unknown() - ).min(1).required(), + payload: joi.object().keys({ + notifications: joi.array().items( + joi.object().keys({ + serviceId: joi.string().valid( + constants.SETTINGS_EMAIL_SERVICE_ID, + constants.SETTINGS_SLACK_SERVICE_ID, + constants.SETTINGS_WEB_SERVICE_ID).required(), + }).unknown() + ).min(1).required(), + }).required(), }).required(), }; From 1cb6e253ea6c5edc6306a1f484ff5b0ed157516e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C4=B0=C5=9Fbilir?= Date: Thu, 5 Aug 2021 14:58:33 +0300 Subject: [PATCH 03/13] fix: find users by uuid, allow emply cc, corect cc format --- config/default.js | 3 +-- src/common/tcApiHelper.js | 9 ++++--- src/services/UniversalNotificationService.js | 27 ++++++++++++++++---- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/config/default.js b/config/default.js index 1ee9a67..4916bab 100644 --- a/config/default.js +++ b/config/default.js @@ -30,8 +30,7 @@ module.exports = { TC_API_V3_BASE_URL: process.env.TC_API_V3_BASE_URL || 'http://api.topcoder-dev.com/v3', TC_API_V4_BASE_URL: process.env.TC_API_V4_BASE_URL || '', - TC_API_V5_BASE_URL: process.env.TC_API_V5_BASE_URL || '', - TC_API_V5_USERS_URL: process.env.TC_API_V5_USERS_URL || 'https://api.topcoder-dev.com/v5/users', + TC_API_V5_BASE_URL: process.env.TC_API_V5_BASE_URL || 'https://api.topcoder-dev.com/v5', API_CONTEXT_PATH: process.env.API_CONTEXT_PATH || '/v5/notifications', TC_API_BASE_URL: process.env.TC_API_BASE_URL || '', diff --git a/src/common/tcApiHelper.js b/src/common/tcApiHelper.js index 346896f..8a030b7 100644 --- a/src/common/tcApiHelper.js +++ b/src/common/tcApiHelper.js @@ -162,7 +162,7 @@ function* getUsersByEmails(emails) { * @param {Array} ids the objects that has user uuids. * @returns {Array} the matched users */ -function* getUsersByUserUUIDs(ids) { +function* getUsersByUserUUIDs(ids, enrich) { if (!ids || ids.length === 0) { return []; } @@ -171,7 +171,7 @@ function* getUsersByUserUUIDs(ids) { try { for (const id of ids) { const res = yield request - .get(`${config.TC_API_V5_USERS_URL}/${id}`) + .get(`${config.TC_API_V5_BASE_URL}/users/${id.userUUID}${enrich ? '?enrich=true' : ''}`) .set('Authorization', `Bearer ${token}`); const user = res.body; logger.verbose(`Searched users: ${JSON.stringify(user, null, 4)}`); @@ -288,6 +288,7 @@ function* notifyUserViaWeb(message) { function* notifyUserViaEmail(message) { const notificationType = message.type; const topic = constants.BUS_API_EVENT.EMAIL.UNIVERSAL; + const cc = _.map(_.filter(message.details.cc, c => !_.isUndefined(c.email)), 'email'); for (const recipient of message.details.recipients) { const userId = recipient.userId; let userEmail; @@ -297,7 +298,7 @@ function* notifyUserViaEmail(message) { } else { userEmail = recipient.email; if (!userEmail) { - logger.error(`Email not received for user: ${userId}`); + logger.error(`Email not received for user: ${JSON.stringify(recipient, null, 4)}`); continue; } } @@ -315,7 +316,7 @@ function* notifyUserViaEmail(message) { const payload = { from: message.details.from, recipients, - cc: message.details.cc || [], + cc, data: message.details.data || {}, sendgrid_template_id: message.details.sendgridTemplateId, version: message.details.version, diff --git a/src/services/UniversalNotificationService.js b/src/services/UniversalNotificationService.js index eb79ded..c03f896 100644 --- a/src/services/UniversalNotificationService.js +++ b/src/services/UniversalNotificationService.js @@ -28,7 +28,7 @@ const emailSchema = joi.object().keys({ userUUID: joi.string().uuid(), email: joi.string().email(), handle: joi.string(), - }).min(1).required() + }).min(1) ), data: joi.object().keys({ subject: joi.string(), @@ -74,6 +74,15 @@ function validator(data, schema) { return true; } + +/** + * Complete missing user fields of given payload details + * This function mutates the given details object. + * @param {Object} details the object which has recipients array + * @param {Boolean} findEmail true if emails are needed + * @param {Boolean} findUserId true if userIds are needed + * @returns {undefined} + */ function* completeMissingFields(details, findEmail, findUserId) { const getFieldsByUserId = []; const getFieldsByHandle = []; @@ -127,20 +136,26 @@ function* completeMissingFields(details, findEmail, findUserId) { } } } - const foundUsersByUUID = yield tcApiHelper.getUsersByUserUUIDs(getFieldsByUserUUID); + const foundUsersByUUID = yield tcApiHelper.getUsersByUserUUIDs(getFieldsByUserUUID, true); if (!_.isEmpty(foundUsersByUUID)) { for (const user of getFieldsByUserUUID) { const found = _.find(foundUsersByUUID, ['id', user.userUUID]) || {}; if (!_.isUndefined(found.externalProfiles) && !_.isEmpty(found.externalProfiles)) { _.assign(user, { userId: _.toInteger(_.get(found.externalProfiles[0], 'externalId')) }); } + if (!_.isUndefined(found.handle) && _.isUndefined(user.handle)) { + _.assign(user, { handle: found.handle }); + } } + if (findEmail) { const usersHaveId = _.filter(getFieldsByUserUUID, u => !_.isUndefined(u.userId)); - const foundUsersById = yield tcApiHelper.getUsersByHandlesAndUserIds([], usersHaveId); - if (!_.isEmpty(foundUsersById)) { + const usersHaveHandle = _.filter(getFieldsByUserUUID, u => _.isUndefined(u.userId) && !_.isUndefined(u.handle)); + const foundUser = yield tcApiHelper.getUsersByHandlesAndUserIds(usersHaveHandle, usersHaveId); + if (!_.isEmpty(foundUser)) { for (const user of getFieldsByUserUUID) { - const found = _.find(foundUsersById, ['userId', user.userId]) || {}; + const found = _.find(foundUser, !_.isUndefined(user.handle) + ? ['handle', user.handle] : ['userId', user.userId]) || {}; if (!_.isUndefined(found.email)) { _.assign(user, { email: found.email }); } @@ -162,6 +177,7 @@ function* handle(message) { switch (data.serviceId) { case constants.SETTINGS_EMAIL_SERVICE_ID: if (validator(data, emailSchema)) { + // find missing emails and userIds yield completeMissingFields(data.details, true, true); yield tcApiHelper.notifyUserViaEmail(data); } @@ -173,6 +189,7 @@ function* handle(message) { break; case constants.SETTINGS_WEB_SERVICE_ID: if (validator(data, webSchema)) { + // find missing userIds yield completeMissingFields(data.details, false, true); const _notifications = yield tcApiHelper.notifyUserViaWeb(data); if (_notifications) { From c321bd26309c5b6167f7bc36dd61c5eec9ef5742 Mon Sep 17 00:00:00 2001 From: Maksym Mykhailenko Date: Fri, 6 Aug 2021 12:53:16 +0300 Subject: [PATCH 04/13] revert package-lock changes --- package-lock.json | 284 ++++++++++++---------------------------------- 1 file changed, 74 insertions(+), 210 deletions(-) diff --git a/package-lock.json b/package-lock.json index f617e1e..17a40e0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4,11 +4,6 @@ "lockfileVersion": 1, "requires": true, "dependencies": { - "@tootallnate/once": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", - "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==" - }, "@types/babel-types": { "version": "7.0.4", "resolved": "https://registry.npmjs.org/@types/babel-types/-/babel-types-7.0.4.tgz", @@ -544,11 +539,11 @@ "integrity": "sha512-32NDda82rhwD9/JBCCkB+MRYDp0oSvlo2IL6rQWA10PQi7tDUM3eqMSltXmY+Oyl/7N3P3qNtAlv7X0d9bI28w==" }, "axios": { - "version": "0.19.2", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.19.2.tgz", - "integrity": "sha512-fjgm5MvRHLhx+osE2xoekY70AhARk3a6hkN+3Io1jc00jtquGvxYlKlsFUhmUET0V5te6CcZI7lcv2Ym61mjHA==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.12.0.tgz", + "integrity": "sha1-uQewIhzDTsHJ+sGOx/B935V4W6Q=", "requires": { - "follow-redirects": "1.5.10" + "follow-redirects": "0.0.7" } }, "babel-runtime": { @@ -3119,21 +3114,12 @@ "dev": true }, "follow-redirects": { - "version": "1.5.10", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz", - "integrity": "sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ==", + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-0.0.7.tgz", + "integrity": "sha1-NLkLqyqRGqNHVx2pDyK9NuzYqRk=", "requires": { - "debug": "3.1.0" - }, - "dependencies": { - "debug": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", - "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", - "requires": { - "ms": "2.0.0" - } - } + "debug": "2.6.9", + "stream-consume": "0.1.1" } }, "for-in": { @@ -6279,7 +6265,8 @@ "lodash.clonedeep": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", - "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=" + "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=", + "dev": true }, "lodash.cond": { "version": "4.5.2", @@ -6860,21 +6847,6 @@ } } }, - "node-cache": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/node-cache/-/node-cache-5.1.2.tgz", - "integrity": "sha512-t1QzWwnk4sjLWaQAS8CHgOJ+RAfmHpxFWmc36IWTiWHQfs0w5JDMBS1b1ZxQteo0vVVuWJvIUKHDkkeK7vIGCg==", - "requires": { - "clone": "2.1.2" - }, - "dependencies": { - "clone": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz", - "integrity": "sha1-G39Ln1kfHo+DZwQBYANFoCiHQ18=" - } - } - }, "node-cron": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-1.2.1.tgz", @@ -9223,8 +9195,7 @@ "stream-consume": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/stream-consume/-/stream-consume-0.1.1.tgz", - "integrity": "sha512-tNa3hzgkjEP7XbCkbRXe1jpg+ievoa0O4SCFlMOYEscGSS4JJsckGL8swUyAa/ApGU3Ae4t6Honor4HhL+tRyg==", - "dev": true + "integrity": "sha512-tNa3hzgkjEP7XbCkbRXe1jpg+ievoa0O4SCFlMOYEscGSS4JJsckGL8swUyAa/ApGU3Ae4t6Honor4HhL+tRyg==" }, "stream-throttle": { "version": "0.1.3", @@ -9508,66 +9479,34 @@ } }, "tc-core-library-js": { - "version": "github:appirio-tech/tc-core-library-js#0d8b4dfc6a1cb0aa10a7ea1b90ed58ba5f0e11b0", + "version": "github:appirio-tech/tc-core-library-js#d16413db30b1eed21c0cf426e185bedb2329ddab", "requires": { "auth0-js": "9.6.1", - "axios": "0.19.2", + "axios": "0.12.0", "bunyan": "1.8.12", "jsonwebtoken": "8.5.1", - "jwks-rsa": "1.12.3", + "jwks-rsa": "1.4.0", "le_node": "1.7.1", "lodash": "4.17.10", "millisecond": "0.1.2", - "request": "2.88.2" + "request": "2.88.0" }, "dependencies": { - "@types/express-jwt": { - "version": "0.0.42", - "resolved": "https://registry.npmjs.org/@types/express-jwt/-/express-jwt-0.0.42.tgz", - "integrity": "sha512-WszgUddvM1t5dPpJ3LhWNH8kfNN8GPIBrAGxgIYXVCEGx6Bx4A036aAuf/r5WH9DIEdlmp7gHOYvSM6U87B0ag==", - "requires": { - "@types/express": "4.11.1", - "@types/express-unless": "0.0.32" - } - }, - "agent-base": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", - "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", - "requires": { - "debug": "4.3.2" - } - }, "ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "version": "6.10.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.0.tgz", + "integrity": "sha512-nffhOpkymDECQyR0mnsUtoCE8RlX38G0rYP+wgLWFyZuUyuuojSSvi/+euOiQBIn63whYwYVIIH1TvE3tu4OEg==", "requires": { - "fast-deep-equal": "3.1.3", + "fast-deep-equal": "2.0.1", "fast-json-stable-stringify": "2.0.0", "json-schema-traverse": "0.4.1", "uri-js": "4.2.2" } }, "aws4": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.11.0.tgz", - "integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==" - }, - "debug": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", - "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", - "requires": { - "ms": "2.1.2" - }, - "dependencies": { - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - } - } + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", + "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==" }, "extend": { "version": "3.0.2", @@ -9575,43 +9514,19 @@ "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" }, "fast-deep-equal": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" - }, - "follow-redirects": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.1.tgz", - "integrity": "sha512-HWqDgT7ZEkqRzBvc2s64vSZ/hfOceEol3ac/7tKwzuvEyWx3/4UegXh5oBOIotkGsObyk3xznnSRVADBgWSQVg==" + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", + "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=" }, "har-validator": { - "version": "5.1.5", - "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.5.tgz", - "integrity": "sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==", + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", + "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", "requires": { - "ajv": "6.12.6", + "ajv": "6.10.0", "har-schema": "2.0.0" } }, - "http-proxy-agent": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz", - "integrity": "sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==", - "requires": { - "@tootallnate/once": "1.1.2", - "agent-base": "6.0.2", - "debug": "4.3.2" - } - }, - "https-proxy-agent": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", - "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", - "requires": { - "agent-base": "6.0.2", - "debug": "4.3.2" - } - }, "json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", @@ -9630,8 +9545,8 @@ "lodash.isplainobject": "4.0.6", "lodash.isstring": "4.0.1", "lodash.once": "4.1.1", - "ms": "2.1.3", - "semver": "5.7.1" + "ms": "2.1.1", + "semver": "5.7.0" } }, "jwa": { @@ -9645,30 +9560,16 @@ } }, "jwks-rsa": { - "version": "1.12.3", - "resolved": "https://registry.npmjs.org/jwks-rsa/-/jwks-rsa-1.12.3.tgz", - "integrity": "sha512-cFipFDeYYaO9FhhYJcZWX/IyZgc0+g316rcHnDpT2dNRNIE/lMOmWKKqp09TkJoYlNFzrEVODsR4GgXJMgWhnA==", - "requires": { - "@types/express-jwt": "0.0.42", - "axios": "0.21.1", - "debug": "4.3.2", - "http-proxy-agent": "4.0.1", - "https-proxy-agent": "5.0.0", - "jsonwebtoken": "8.5.1", - "limiter": "1.1.5", - "lru-memoizer": "2.1.4", - "ms": "2.1.3", - "proxy-from-env": "1.1.0" - }, - "dependencies": { - "axios": { - "version": "0.21.1", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz", - "integrity": "sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==", - "requires": { - "follow-redirects": "1.14.1" - } - } + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/jwks-rsa/-/jwks-rsa-1.4.0.tgz", + "integrity": "sha512-6aUc+oTuqsLwIarfq3A0FqoD5LFSgveW5JO1uX2s0J8TJuOEcDc4NIMZAmVHO8tMHDw7CwOPzXF/9QhfOpOElA==", + "requires": { + "@types/express-jwt": "0.0.34", + "debug": "2.6.9", + "limiter": "1.1.3", + "lru-memoizer": "1.12.0", + "ms": "2.1.1", + "request": "2.88.0" } }, "jws": { @@ -9680,110 +9581,86 @@ "safe-buffer": "5.1.1" } }, - "limiter": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/limiter/-/limiter-1.1.5.tgz", - "integrity": "sha512-FWWMIEOxz3GwUI4Ts/IvgVy6LPvoMPgjMdQ185nN6psJyBJ4yOpzqm695/h5umdLJg2vW3GR5iG11MAkR2AzJA==" - }, - "lru-memoizer": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/lru-memoizer/-/lru-memoizer-2.1.4.tgz", - "integrity": "sha512-IXAq50s4qwrOBrXJklY+KhgZF+5y98PDaNo0gi/v2KQBFLyWr+JyFvijZXkGKjQj/h9c0OwoE+JZbwUXce76hQ==", - "requires": { - "lodash.clonedeep": "4.5.0", - "lru-cache": "4.0.2" - } - }, "mime-db": { - "version": "1.48.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.48.0.tgz", - "integrity": "sha512-FM3QwxV+TnZYQ2aRqhlKBMHxk10lTbMt3bBkMAp54ddrNeVSfcQYOOKuGuy3Ddrm38I04If834fOUSq1yzslJQ==" + "version": "1.38.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.38.0.tgz", + "integrity": "sha512-bqVioMFFzc2awcdJZIzR3HjZFX20QhilVS7hytkKrv7xFAn8bM1gzc/FOX2awLISvWe0PV8ptFKcon+wZ5qYkg==" }, "mime-types": { - "version": "2.1.31", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.31.tgz", - "integrity": "sha512-XGZnNzm3QvgKxa8dpzyhFTHmpP3l5YNusmne07VUOXxou9CqUqYa/HBy124RqtVh/O2pECas/MOcsDgpilPOPg==", + "version": "2.1.22", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.22.tgz", + "integrity": "sha512-aGl6TZGnhm/li6F7yx82bJiBZwgiEa4Hf6CNr8YO+r5UHr53tSTYZb102zyU50DOWWKeOv0uQLRL0/9EiKWCog==", "requires": { - "mime-db": "1.48.0" + "mime-db": "1.38.0" } }, "ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", + "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" }, "oauth-sign": { "version": "0.9.0", "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" }, - "proxy-from-env": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", - "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" - }, - "punycode": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" - }, "qs": { "version": "6.5.2", "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" }, "request": { - "version": "2.88.2", - "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", - "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==", + "version": "2.88.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", + "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", "requires": { "aws-sign2": "0.7.0", - "aws4": "1.11.0", + "aws4": "1.8.0", "caseless": "0.12.0", "combined-stream": "1.0.6", "extend": "3.0.2", "forever-agent": "0.6.1", "form-data": "2.3.2", - "har-validator": "5.1.5", + "har-validator": "5.1.3", "http-signature": "1.2.0", "is-typedarray": "1.0.0", "isstream": "0.1.2", "json-stringify-safe": "5.0.1", - "mime-types": "2.1.31", + "mime-types": "2.1.22", "oauth-sign": "0.9.0", "performance-now": "2.1.0", "qs": "6.5.2", - "safe-buffer": "5.2.1", - "tough-cookie": "2.5.0", + "safe-buffer": "5.1.2", + "tough-cookie": "2.4.3", "tunnel-agent": "0.6.0", - "uuid": "3.4.0" + "uuid": "3.3.2" }, "dependencies": { "safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } }, "semver": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", - "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" + "version": "5.7.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", + "integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==" }, "tough-cookie": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", - "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", + "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", "requires": { "psl": "1.1.31", - "punycode": "2.1.1" + "punycode": "1.4.1" } }, "uuid": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" } } }, @@ -9919,14 +9796,6 @@ "integrity": "sha1-zu78cXp2xDFvEm0LnbqlXX598Bo=", "dev": true }, - "topcoder-healthcheck-dropin": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/topcoder-healthcheck-dropin/-/topcoder-healthcheck-dropin-1.0.3.tgz", - "integrity": "sha512-k8X84IC2NALu1v8cD3SZXY0MMZAMWw2uzHmjXDlgXwpS5xnXdwnVU+BpJWqg1uz1OuYDdeaAIPguqnhs7G6Y0A==", - "requires": { - "express": "4.16.3" - } - }, "topo": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/topo/-/topo-2.0.2.tgz", @@ -10163,11 +10032,6 @@ } } }, - "urijs": { - "version": "1.19.7", - "resolved": "https://registry.npmjs.org/urijs/-/urijs-1.19.7.tgz", - "integrity": "sha512-Id+IKjdU0Hx+7Zx717jwLPsPeUqz7rAtuVBRLLs+qn+J2nf9NGITWVCxcijgYxBqe83C7sqsQPs6H1pyz3x9gA==" - }, "urix": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/urix/-/urix-0.1.0.tgz", From f448419013854b81d22acdf1dee3ccbf1205a8b8 Mon Sep 17 00:00:00 2001 From: Maksym Mykhailenko Date: Fri, 6 Aug 2021 13:59:56 +0300 Subject: [PATCH 05/13] revert some lint fixes for universal notifications --- connect/connectNotificationServer.js | 7 ++++--- src/app.js | 24 ++++++++++++------------ 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/connect/connectNotificationServer.js b/connect/connectNotificationServer.js index 63fe8dc..34674e3 100644 --- a/connect/connectNotificationServer.js +++ b/connect/connectNotificationServer.js @@ -92,7 +92,7 @@ const getNotificationsForMentionedUser = (logger, eventConfig, content) => { _.forEach(notifications, (notification) => { const mentionedUser = _.find(users, { handle: notification.userHandle }); notification.userId = mentionedUser ? mentionedUser.userId.toString() : null; - if (!notification.userId && logger) { // such notifications would be discarded later after aggregation + if (!notification.userId && logger) {// such notifications would be discarded later after aggregation logger.info(`Unable to find user with handle ${notification.userHandle}`); } }); @@ -102,11 +102,12 @@ const getNotificationsForMentionedUser = (logger, eventConfig, content) => { logger.error(error); logger.info('Unable to send notification to mentioned user'); } - // resolves with empty notification which essentially means we are unable to send notification to mentioned user + //resolves with empty notification which essentially means we are unable to send notification to mentioned user return Promise.resolve([]); }); + } else { + return Promise.resolve([]); } - return Promise.resolve([]); }; /** diff --git a/src/app.js b/src/app.js index ff2a4b3..1d7d4bc 100644 --- a/src/app.js +++ b/src/app.js @@ -21,7 +21,7 @@ const healthcheck = require('topcoder-healthcheck-dropin'); // helps in health checking in case of unhandled rejection of promises const unhandledRejections = []; process.on('unhandledRejection', (reason, promise) => { - logger.debug('Unhandled Rejection at:', promise, 'reason:', reason); + console.log('Unhandled Rejection at:', promise, 'reason:', reason); // aborts the process to let the HA of the container to restart the task // process.abort(); unhandledRejections.push(promise); @@ -31,7 +31,7 @@ process.on('unhandledRejection', (reason, promise) => { // from the unhandledRejections array. We just remove the first element from the array as we only care // about the count every time an unhandled rejection promise is handled process.on('rejectionHandled', (promise) => { - logger.debug('Handled Rejection at:', promise); + console.log('Handled Rejection at:', promise); unhandledRejections.shift(); }); @@ -93,10 +93,10 @@ function startKafkaConsumer(handlers, notificationServiceHandlers) { }); }); - let latestSubscriptions = null; + var latestSubscriptions = null; const check = function () { - logger.debug('Checking health'); + logger.debug("Checking health"); if (unhandledRejections && unhandledRejections.length > 0) { logger.error('Found unhandled promises. Application is potentially in stalled state.'); return false; @@ -106,12 +106,12 @@ function startKafkaConsumer(handlers, notificationServiceHandlers) { return false; } let connected = true; - const currentSubscriptions = consumer.subscriptions; - for (const sIdx of currentSubscriptions) { + let currentSubscriptions = consumer.subscriptions; + for(var sIdx in currentSubscriptions) { // current subscription - const sub = currentSubscriptions[sIdx]; + let sub = currentSubscriptions[sIdx]; // previous subscription - const prevSub = latestSubscriptions ? latestSubscriptions[sIdx] : null; + let prevSub = latestSubscriptions ? latestSubscriptions[sIdx] : null; // levarage the `paused` field (https://github.com/oleksiyk/kafka/blob/master/lib/base_consumer.js#L66) to // determine if there was a possibility of an unhandled exception. If we find paused status for the same // topic in two consecutive health checks, we assume it was stuck because of unhandled error @@ -123,11 +123,11 @@ function startKafkaConsumer(handlers, notificationServiceHandlers) { // stores the latest subscription status in global variable latestSubscriptions = consumer.subscriptions; consumer.client.initialBrokers.forEach(conn => { - logger.debug(`url ${conn.server()} - connected=${conn.connected}`); - connected = conn.connected & connected; + logger.debug(`url ${conn.server()} - connected=${conn.connected}`) + connected = conn.connected & connected }); - return connected; - }; + return connected + } consumer .init() From 568d0313f01c07d0905b01d806b14eee1cae8c05 Mon Sep 17 00:00:00 2001 From: Maksym Mykhailenko Date: Fri, 6 Aug 2021 14:14:41 +0300 Subject: [PATCH 06/13] rename universal notification kafka message --- config/default.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/default.js b/config/default.js index 4916bab..1f772e7 100644 --- a/config/default.js +++ b/config/default.js @@ -125,7 +125,7 @@ module.exports = { handleBulkNotification: {}, }, ], - 'notification.action.create': [{ + 'notifications.action.create': [{ handleUniversalNotification: {}, }], // 'notifications.community.challenge.created': ['handleChallengeCreated'], From 1dbd799ffff3a5fffcb534c2ae7ebd560cd45b8f Mon Sep 17 00:00:00 2001 From: Maksym Mykhailenko Date: Fri, 6 Aug 2021 16:34:02 +0300 Subject: [PATCH 07/13] fix universal notification validation rule allow empty body and subject for emails --- src/services/UniversalNotificationService.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/services/UniversalNotificationService.js b/src/services/UniversalNotificationService.js index c03f896..1cef956 100644 --- a/src/services/UniversalNotificationService.js +++ b/src/services/UniversalNotificationService.js @@ -31,8 +31,9 @@ const emailSchema = joi.object().keys({ }).min(1) ), data: joi.object().keys({ - subject: joi.string(), - body: joi.string(), + subject: joi.string().allow(''), + body: joi.string().allow(''), + // any other fields are allowed inside `data` too }).unknown(), sendgridTemplateId: joi.string().required(), version: joi.string().required(), From 28e5cac2606a3848c333fad7e0ab0e088e0968d1 Mon Sep 17 00:00:00 2001 From: Maksym Mykhailenko Date: Sat, 7 Aug 2021 14:24:37 +0300 Subject: [PATCH 08/13] fix error handling for universal notifications --- src/common/tcApiHelper.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/tcApiHelper.js b/src/common/tcApiHelper.js index 8a030b7..b1b526a 100644 --- a/src/common/tcApiHelper.js +++ b/src/common/tcApiHelper.js @@ -179,7 +179,7 @@ function* getUsersByUserUUIDs(ids, enrich) { } return users; } catch (err) { - const error = new Error(err.response.text); + const error = new Error(_.get(err, 'response.text', err.toString())); error.status = err.status; throw error; } From f97e369e077abbc193a23c4231317cdbe0e8b48d Mon Sep 17 00:00:00 2001 From: Maksym Mykhailenko Date: Wed, 11 Aug 2021 12:49:52 +0300 Subject: [PATCH 09/13] allow "blocks" for slack universal notifications --- src/services/UniversalNotificationService.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/services/UniversalNotificationService.js b/src/services/UniversalNotificationService.js index 1cef956..e873a2b 100644 --- a/src/services/UniversalNotificationService.js +++ b/src/services/UniversalNotificationService.js @@ -46,6 +46,7 @@ const slackSchema = joi.object().keys({ details: joi.object().keys({ channel: joi.string().required(), text: joi.string().required(), + blocks: joi.array().items(joi.object()), }).required(), }).required(); From f17891f4237510facc8a6cab9407476cdac1b789 Mon Sep 17 00:00:00 2001 From: Maksym Mykhailenko Date: Wed, 11 Aug 2021 16:51:09 +0300 Subject: [PATCH 10/13] fix universal web notifications "type" --- src/common/tcApiHelper.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/common/tcApiHelper.js b/src/common/tcApiHelper.js index b1b526a..aa00d65 100644 --- a/src/common/tcApiHelper.js +++ b/src/common/tcApiHelper.js @@ -275,7 +275,14 @@ function* notifyUserViaWeb(message) { + ` service to the userId '${userId}' due to his notification settings.`); continue; } - notifications.push(_.assign({}, _.pick(message.details, ['contents', 'version']), { userId })); + notifications.push(_.assign( + {}, + _.pick(message.details, ['contents', 'version']), + { + userId, + type: message.type, + } + )); } return notifications; From 911fe340623917b8d08abf2b89de49d8bf2a4536 Mon Sep 17 00:00:00 2001 From: Maksym Mykhailenko Date: Wed, 11 Aug 2021 18:55:25 +0300 Subject: [PATCH 11/13] support "taas" platform --- src/services/NotificationService.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/services/NotificationService.js b/src/services/NotificationService.js index 2461306..c6501cd 100644 --- a/src/services/NotificationService.js +++ b/src/services/NotificationService.js @@ -200,8 +200,16 @@ function* listNotifications(query, userId) { case 'connect': filter.where.type = { $like: 'connect.notification.%' }; break; + case 'taas': + filter.where.type = { $like: 'taas.notification.%' }; + break; case 'community': - filter.where.type = { $notLike: 'connect.notification.%' }; + filter.where.type = { + $and: [ + { $notLike: 'connect.notification.%' }, + { $notLike: 'taas.notification.%' }, + ], + }; break; } From 11498052bdfc31a19c8b6912eb5bb6a086d49dff Mon Sep 17 00:00:00 2001 From: Maksym Mykhailenko Date: Fri, 13 Aug 2021 16:46:41 +0300 Subject: [PATCH 12/13] support rich message Slack fromat --- src/common/tcApiHelper.js | 7 ++++--- src/services/UniversalNotificationService.js | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/common/tcApiHelper.js b/src/common/tcApiHelper.js index aa00d65..7cf6654 100644 --- a/src/common/tcApiHelper.js +++ b/src/common/tcApiHelper.js @@ -208,9 +208,10 @@ function* sendMessageToBus(data) { /** * Notify slack channel. * @param {string} channel the slack channel name - * @param {string} text the message + * @param {string} text the message + * @param {string} blocks rich formatted message as per https://api.slack.com/block-kit */ -function* notifySlackChannel(channel, text) { +function* notifySlackChannel(channel, text, blocks) { if (config.SLACK.NOTIFY) { const token = config.SLACK.BOT_TOKEN; const url = config.SLACK.URL; @@ -218,7 +219,7 @@ function* notifySlackChannel(channel, text) { .post(url) .set('Content-Type', 'application/json') .set('Authorization', `Bearer ${token}`) - .send({ channel, text }) + .send({ channel, text, blocks }) .catch((err) => { const errorDetails = _.get(err, 'message'); throw new Error( diff --git a/src/services/UniversalNotificationService.js b/src/services/UniversalNotificationService.js index e873a2b..f4927e6 100644 --- a/src/services/UniversalNotificationService.js +++ b/src/services/UniversalNotificationService.js @@ -186,7 +186,7 @@ function* handle(message) { break; case constants.SETTINGS_SLACK_SERVICE_ID: if (validator(data, slackSchema)) { - yield tcApiHelper.notifySlackChannel(data.details.channel, data.details.text); + yield tcApiHelper.notifySlackChannel(data.details.channel, data.details.text, data.details.blocks); } break; case constants.SETTINGS_WEB_SERVICE_ID: From 0b2e9ba0359bbb76bf4ffd14a12773c8457007e0 Mon Sep 17 00:00:00 2001 From: Maksym Mykhailenko Date: Fri, 13 Aug 2021 16:49:51 +0300 Subject: [PATCH 13/13] local setup docker-compose with verification guide --- local/Verification.md | 95 +++++++++++++++++++ local/docker-compose.yml | 56 +++++++++++ local/generic-tc-service/Dockerfile | 15 +++ local/generic-tc-service/docker-entrypoint.sh | 10 ++ 4 files changed, 176 insertions(+) create mode 100644 local/Verification.md create mode 100644 local/docker-compose.yml create mode 100644 local/generic-tc-service/Dockerfile create mode 100755 local/generic-tc-service/docker-entrypoint.sh diff --git a/local/Verification.md b/local/Verification.md new file mode 100644 index 0000000..cb85dae --- /dev/null +++ b/local/Verification.md @@ -0,0 +1,95 @@ +# Deployment & Verification Guide + +> This guide is prepared for Linux OS, npm install command fails for Windows. +> Please switch to Node.js version 8.x before proceed. + +1. I prepared a docker-compose file for local deployment. It will start the following services: + * postgres + * zookeeper + * kafka + * bus-api +2. Run following commands to start dependency services: + ```bash + cd local + docker-compose build --no-cache + docker-compose up -d + ``` +3. After services have started successfully, set environment variables for notifications api: + ```sh + # you need to provide the values for the next variables + export AUTH0_CLIENT_ID + export AUTH0_CLIENT_SECRET + export SLACK_BOT_TOKEN + + # for other variables can uses these ones + export AUTH0_URL='https://topcoder-dev.auth0.com/oauth/token' + export AUTH0_AUDIENCE='https://m2m.topcoder-dev.com/' + export AUTH0_AUDIENCE_UBAHN='https://u-bahn.topcoder.com' + export AUTH_SECRET='mysecret' + export VALID_ISSUERS='["https://api.topcoder-dev.com", "https://api.topcoder.com", "https://topcoder-dev.auth0.com/", "https://auth.topcoder-dev.com/"]' + export DATABASE_URL='postgres://postgres:postgres@localhost:5432/postgres' + export PORT=4000 + export TC_API_V5_BASE_URL='http://localhost:8002/v5' + export KAFKA_URL='localhost:9092' + export KAFKA_GROUP_ID='tc-notifications' + export SLACK_NOTIFY='true' + export ENABLE_DEV_MODE='false' + ``` + +4. Run command `npm install` +5. Run command `npm run reset:db` to initialize tables. +6. Run command `npm run startConsumer` to start kafka consumer. +7. Wait to see following logs: + ```bash + 2021-07-24T15:11:01.512Z INFO no-kafka-client Joined group tc-notifications generationId 1 as no-kafka-client-2689c63f-9850-448a-a3f0-11d2ec8e49ce + 2021-07-24T15:11:01.512Z INFO no-kafka-client Elected as group leader + 2021-07-24T15:11:01.583Z DEBUG no-kafka-client Subscribed to notifications.autopilot.events:0 offset 0 leader localhost:9092 + 2021-07-24T15:11:01.583Z DEBUG no-kafka-client Subscribed to challenge.notification.events:0 offset 0 leader localhost:9092 + 2021-07-24T15:11:01.584Z DEBUG no-kafka-client Subscribed to notification.action.create:0 offset 0 leader localhost:9092 + 2021-07-24T15:11:01.584Z DEBUG no-kafka-client Subscribed to admin.notification.broadcast:0 offset 0 leader localhost:9092 + ``` +8. Now, we will prepare terminals to be used later for verifying api. + * 1 terminal to watch kafka topic for email notifications. + * 1 terminal to query Notifications table for web notifications. + * Slack app, web or mobile application. +9. Open a new terminal for watching kafka topic `external.action.email` + ```bash + docker exec notification-kafka /opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic external.action.email + ``` +10. **Alternative-1:** Open a new terminal for postgres. It will be used to check if notifications are getting created. + ```bash + docker exec -it notification-postgres psql -U postgres + \c postgres + SELECT * FROM "Notifications"; + ``` + **Alternative-2:** Start notifications api and use postman to list notifications. Open a new terminal and change PORT to 3000 + ```bash + . ./environment.sh + export PORT=3000 + npm run startAPI + ``` + Open postman and use following collection and environment: + * collection: docs/tc-notifications.postman_collection.json + * environment: docs/tc-notifications.postman_environment.json + + Existing postman collections are outdated and need to be converted to version 2.0.0. So, I had to create a new collection. + +11. To verify slack messages, you can use following workspace and user credentials. Token for posting messages to slack was already shared inside environment variables and has been set at step 3. + `https://tc-notifications.slack.com/` + username: `tc.notification.slack@gmail.com` + password: `@Topcoder123` + + +12. Now we can start testing. Open a new terminal for producing kafka messages for the topic `notification.action.create` + ```bash + docker exec -it notification-kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic notification.action.create + ``` +13. You can copy and paste the following messages to the terminal we opened for producing kafka messages. + send messages successfully + ```json + {"topic":"notification.action.create","originator":"tc-direct","timestamp":"2018-02-16T00:00:00","mime-type":"application\/json","payload":[{"serviceId":"email","type":"taas.notification.request-submitted","details":{"from":"example@example.com","recipients":[{"userId":123,"email":"test1@test.com"},{"userId":456,"email":"test2@test.com"}],"cc":[{"userId":789,"email":"test3@test.com"},{"userId":987,"email":"test4@test.com"}],"data":{"subject":"...","body":"...","field1":"...","field2":"...","filedN":"..."},"sendgridTemplateId":"...","version":"v3"}},{"serviceId":"slack","type":"taas.notification.request-submitted","details":{"channel":"general","text":"test message"}},{"serviceId":"web","type":"taas.notification.request-submitted","details":{"userId":40152856,"contents":{},"version":1}}]} + ``` + email: fail (invalid email), web: success, slack: success + ```json + {"topic":"notification.action.create","originator":"tc-direct","timestamp":"2018-02-16T00:00:00","mime-type":"application\/json","payload":[{"serviceId":"email","type":"taas.notification.request-submitted","details":{"from":"example.com","recipients":[{"userId":123,"email":"test1@test.com"},{"userId":456,"email":"test2@test.com"}],"cc":[{"userId":789,"email":"test3@test.com"},{"userId":987,"email":"test4@test.com"}],"data":{"subject":"...","body":"...","field1":"...","field2":"...","filedN":"..."},"sendgridTemplateId":"...","version":"v3"}},{"serviceId":"slack","type":"taas.notification.request-submitted","details":{"channel":"random","text":"test message"}},{"serviceId":"web","type":"taas.notification.request-submitted","details":{"userId":40152856,"contents":{},"version":1}}]} + ``` \ No newline at end of file diff --git a/local/docker-compose.yml b/local/docker-compose.yml new file mode 100644 index 0000000..9ed82a2 --- /dev/null +++ b/local/docker-compose.yml @@ -0,0 +1,56 @@ +version: "3" +services: + postgres: + container_name: notification-postgres + image: postgres:11.8 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + ports: + - 5432:5432 + + zookeeper: + image: wurstmeister/zookeeper + container_name: notification-zookeeper + ports: + - 2181:2181 + + kafka: + image: wurstmeister/kafka + container_name: notification-kafka + depends_on: + - zookeeper + ports: + - 9092:9092 + environment: + KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka:9093,OUTSIDE://localhost:9092 + KAFKA_LISTENERS: INSIDE://kafka:9093,OUTSIDE://0.0.0.0:9092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_CREATE_TOPICS: "challenge.notification.events:1:1,notifications.autopilot.events:1:1,admin.notification.broadcast:1:1,notifications.action.create:1:1,external.action.email:1:1" + + tc-bus-api: + container_name: tc-bus-api + build: + context: ./generic-tc-service + args: + NODE_VERSION: 8.11.3 + GIT_URL: https://github.com/topcoder-platform/tc-bus-api + GIT_BRANCH: dev + BYPASS_TOKEN_VALIDATION: 1 + command: start + ports: + - 8002:8002 + depends_on: + - kafka + environment: + - PORT=8002 + - KAFKA_URL=kafka:9093 + - JWT_TOKEN_SECRET=secret + - VALID_ISSUERS="[\"https:\/\/topcoder-newauth.auth0.com\/\",\"https:\/\/api.topcoder-dev.com\",\"https:\/\/topcoder-dev.auth0.com\/\"]" + - AUTH0_URL + - AUTH0_AUDIENCE + - AUTH0_CLIENT_ID + - AUTH0_CLIENT_SECRET + - AUTH0_PROXY_SERVER_URL diff --git a/local/generic-tc-service/Dockerfile b/local/generic-tc-service/Dockerfile new file mode 100644 index 0000000..2d9c886 --- /dev/null +++ b/local/generic-tc-service/Dockerfile @@ -0,0 +1,15 @@ +ARG NODE_VERSION=12.16.3 + +FROM node:$NODE_VERSION +ARG GIT_URL +ARG GIT_BRANCH +ARG BYPASS_TOKEN_VALIDATION + +RUN git clone $GIT_URL /opt/app +WORKDIR /opt/app +RUN git checkout -b node-branch origin/$GIT_BRANCH + +RUN npm install +RUN if [ $BYPASS_TOKEN_VALIDATION -eq 1 ]; then sed -i '/decodedToken = jwt.decode/a \ callback(undefined, decodedToken.payload); return;' node_modules/tc-core-library-js/lib/auth/verifier.js; fi +COPY docker-entrypoint.sh /opt/ +ENTRYPOINT ["/opt/docker-entrypoint.sh"] \ No newline at end of file diff --git a/local/generic-tc-service/docker-entrypoint.sh b/local/generic-tc-service/docker-entrypoint.sh new file mode 100755 index 0000000..2c70231 --- /dev/null +++ b/local/generic-tc-service/docker-entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +HOST_DOMAIN="host.docker.internal" +ping -q -c1 $HOST_DOMAIN > /dev/null 2>&1 +if [ $? -ne 0 ]; then + HOST_IP=$(ip route | awk 'NR==1 {print $3}') + echo -e "$HOST_IP\t$HOST_DOMAIN" >> /etc/hosts +fi + +cd /opt/app/ && npm run $1 \ No newline at end of file