[PROD] Universal Notifications

README.md

@@ -103,6 +103,10 @@ The following parameters can be set in config files or in env variables:
   - `REPLY_EMAIL_PREFIX`: prefix of the genereated reply email address
   - `REPLY_EMAIL_DOMAIN`: email domain
   - `DEFAULT_REPLY_EMAIL`: default reply to email address, for example no-reply@topcoder.com
+- **Slack api**
+  - `SLACK_URL`: slack api url to post messages
+  - `SLACK_BOT_TOKEN`: slack bot user OAuth token
+  - `SLACK_NOTIFY`: slack notification switch, set to 'true' to enable slack notifications.
 
 Note that the above two configuration are separate because the common notification server config
 will be deployed to a NPM package, the connect notification server will use that NPM package,

config/default.js

@@ -28,9 +28,9 @@ module.exports = {
   KAFKA_CLIENT_CERT_KEY: process.env.KAFKA_CLIENT_CERT_KEY ?
     process.env.KAFKA_CLIENT_CERT_KEY.replace('\\n', '\n') : null,
 
-  TC_API_V3_BASE_URL: process.env.TC_API_V3_BASE_URL || '',
+  TC_API_V3_BASE_URL: process.env.TC_API_V3_BASE_URL || 'http://api.topcoder-dev.com/v3',
   TC_API_V4_BASE_URL: process.env.TC_API_V4_BASE_URL || '',
-  TC_API_V5_BASE_URL: process.env.TC_API_V5_BASE_URL || '',
+  TC_API_V5_BASE_URL: process.env.TC_API_V5_BASE_URL || 'https://api.topcoder-dev.com/v5',
   API_CONTEXT_PATH: process.env.API_CONTEXT_PATH || '/v5/notifications',
   TC_API_BASE_URL: process.env.TC_API_BASE_URL || '',
 
@@ -47,7 +47,12 @@ module.exports = {
   AUTH0_CLIENT_ID: process.env.AUTH0_CLIENT_ID,
   AUTH0_CLIENT_SECRET: process.env.AUTH0_CLIENT_SECRET,
   AUTH0_PROXY_SERVER_URL: process.env.AUTH0_PROXY_SERVER_URL,
-
+  // Slack configuration.
+  SLACK: {
+    URL: process.env.SLACK_URL || 'https://slack.com/api/chat.postMessage',
+    BOT_TOKEN: process.env.SLACK_BOT_TOKEN,
+    NOTIFY: process.env.SLACK_NOTIFY === 'true',
+  },
   KAFKA_CONSUMER_RULESETS: {
     // key is Kafka topic name, value is array of ruleset which have key as
     // handler function name defined in src/processors/index.js
@@ -115,19 +120,22 @@ module.exports = {
     //    },
     //  },
     // ],
-    // */ // issue - https://github.com/topcoder-platform/community-app/issues/4108 
+    // */ // issue - https://github.com/topcoder-platform/community-app/issues/4108
     'admin.notification.broadcast': [{
-      handleBulkNotification: {}
-    }
-    ]
-    //'notifications.community.challenge.created': ['handleChallengeCreated'],
-    //'notifications.community.challenge.phasewarning': ['handleChallengePhaseWarning'],
+      handleBulkNotification: {},
+    },
+    ],
+    'notifications.action.create': [{
+      handleUniversalNotification: {},
+    }],
+    // 'notifications.community.challenge.created': ['handleChallengeCreated'],
+    // 'notifications.community.challenge.phasewarning': ['handleChallengePhaseWarning'],
   },
 
   // email notification service related variables
   ENV: process.env.ENV,
   ENABLE_EMAILS: process.env.ENABLE_EMAILS ? Boolean(process.env.ENABLE_EMAILS) : false,
-  ENABLE_DEV_MODE: process.env.ENABLE_DEV_MODE ? Boolean(process.env.ENABLE_DEV_MODE) : true,
+  ENABLE_DEV_MODE: process.env.ENABLE_DEV_MODE === 'true',
   DEV_MODE_EMAIL: process.env.DEV_MODE_EMAIL,
   DEFAULT_REPLY_EMAIL: process.env.DEFAULT_REPLY_EMAIL,
   ENABLE_HOOK_BULK_NOTIFICATION: process.env.ENABLE_HOOK_BULK_NOTIFICATION || false,

connect/config.js

@@ -36,6 +36,6 @@ module.exports = {
   DEFAULT_REPLY_EMAIL: process.env.DEFAULT_REPLY_EMAIL,
 
   CONNECT_URL: process.env.CONNECT_URL || 'https://connect.topcoder-dev.com',
-  ACCOUNTS_APP_URL: process.env.ACCOUNTS_APP_URL || "https://accounts-auth0.topcoder-dev.com",
+  ACCOUNTS_APP_URL: process.env.ACCOUNTS_APP_URL || 'https://accounts-auth0.topcoder-dev.com',
   TC_CDN_URL: process.env.TC_CDN_URL,
 };

constants.js

@@ -3,11 +3,14 @@ module.exports = {
   SEARCH_USERS_PAGE_SIZE: 5,
 
   SETTINGS_EMAIL_SERVICE_ID: 'email',
+  SETTINGS_WEB_SERVICE_ID: 'web',
+  SETTINGS_SLACK_SERVICE_ID: 'slack',
   ACTIVE_USER_STATUSES: ['ACTIVE'],
 
   BUS_API_EVENT: {
     EMAIL: {
       GENERAL: 'connect.notification.email.project.notifications.generic',
+      UNIVERSAL: 'external.action.email',
     },
   },
 };

docs/tc-notifications.postman_collection.json

@@ -0,0 +1,38 @@
+{
+	"info": {
+		"_postman_id": "ad14efc8-1fed-4914-8273-330754500801",
+		"name": "TC-NOTIFICATIONS",
+		"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
+	},
+	"item": [
+		{
+			"name": "list notifications",
+			"request": {
+				"method": "GET",
+				"header": [
+					{
+						"key": "Authorization",
+						"value": "Bearer {{TOKEN}}",
+						"type": "text"
+					}
+				],
+				"url": {
+					"raw": "{{URL}}/list?limit=100",
+					"host": [
+						"{{URL}}"
+					],
+					"path": [
+						"list"
+					],
+					"query": [
+						{
+							"key": "limit",
+							"value": "100"
+						}
+					]
+				}
+			},
+			"response": []
+		}
+	]
+}

docs/tc-notifications.postman_environment.json

@@ -0,0 +1,19 @@
+{
+	"id": "9d9c9e1b-6004-4bbe-9a98-55ad3a5838d7",
+	"name": "tc-notifications",
+	"values": [
+		{
+			"key": "URL",
+			"value": "http://localhost:3000/v5/notifications",
+			"enabled": true
+		},
+		{
+			"key": "TOKEN",
+			"value": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlcyI6WyJUb3Bjb2RlciBVc2VyIiwiYWRtaW5pc3RyYXRvciJdLCJpc3MiOiJodHRwczovL2FwaS50b3Bjb2Rlci1kZXYuY29tIiwiaGFuZGxlIjoidGVzdDEiLCJleHAiOjIxNDc0ODM2NDgsInVzZXJJZCI6IjQwMTUyODU2IiwiaWF0IjoxNDYzMDc2MDg5LCJlbWFpbCI6InRlc3RAdG9wY29kZXIuY29tIiwianRpIjoiYjMzYjc3Y2QtYjUyZS00MGZlLTgzN2UtYmViOGUwYWU2YTRhIn0.PKv0QrMCPf0-ZPjv4PGWT7eXne54m7i9YX9eq-fceMU",
+			"enabled": true
+		}
+	],
+	"_postman_variable_scope": "environment",
+	"_postman_exported_at": "2021-07-24T21:01:35.787Z",
+	"_postman_exported_using": "Postman/8.8.0"
+}

local/Verification.md

@@ -0,0 +1,95 @@
+# Deployment & Verification Guide
+
+> This guide is prepared for Linux OS, npm install command fails for Windows.
+> Please switch to Node.js version 8.x before proceed.
+
+1. I prepared a docker-compose file for local deployment. It will start the following services:
+   * postgres
+   * zookeeper
+   * kafka
+   * bus-api
+2. Run following commands to start dependency services:
+   ```bash
+   cd local
+   docker-compose build --no-cache
+   docker-compose up -d
+   ```
+3. After services have started successfully, set environment variables for notifications api:
+   ```sh
+    # you need to provide the values for the next variables
+    export AUTH0_CLIENT_ID
+    export AUTH0_CLIENT_SECRET
+    export SLACK_BOT_TOKEN
+
+    # for other variables can uses these ones
+    export AUTH0_URL='https://topcoder-dev.auth0.com/oauth/token'
+    export AUTH0_AUDIENCE='https://m2m.topcoder-dev.com/'
+    export AUTH0_AUDIENCE_UBAHN='https://u-bahn.topcoder.com'
+    export AUTH_SECRET='mysecret'
+    export VALID_ISSUERS='["https://api.topcoder-dev.com", "https://api.topcoder.com", "https://topcoder-dev.auth0.com/", "https://auth.topcoder-dev.com/"]'
+    export DATABASE_URL='postgres://postgres:postgres@localhost:5432/postgres'
+    export PORT=4000
+    export TC_API_V5_BASE_URL='http://localhost:8002/v5'
+    export KAFKA_URL='localhost:9092'
+    export KAFKA_GROUP_ID='tc-notifications'
+    export SLACK_NOTIFY='true'
+    export ENABLE_DEV_MODE='false'
+    ```
+
+4. Run command `npm install`
+5. Run command `npm run reset:db` to initialize tables.
+6. Run command `npm run startConsumer` to start kafka consumer.
+7. Wait to see following logs:
+   ```bash
+   2021-07-24T15:11:01.512Z INFO no-kafka-client Joined group tc-notifications generationId 1 as no-kafka-client-2689c63f-9850-448a-a3f0-11d2ec8e49ce
+   2021-07-24T15:11:01.512Z INFO no-kafka-client Elected as group leader
+   2021-07-24T15:11:01.583Z DEBUG no-kafka-client Subscribed to notifications.autopilot.events:0 offset 0 leader localhost:9092
+   2021-07-24T15:11:01.583Z DEBUG no-kafka-client Subscribed to challenge.notification.events:0 offset 0 leader localhost:9092
+   2021-07-24T15:11:01.584Z DEBUG no-kafka-client Subscribed to notification.action.create:0 offset 0 leader localhost:9092
+   2021-07-24T15:11:01.584Z DEBUG no-kafka-client Subscribed to admin.notification.broadcast:0 offset 0 leader localhost:9092
+   ```
+8. Now, we will prepare terminals to be used later for verifying api.
+    * 1 terminal to watch kafka topic for email notifications.
+    * 1 terminal to query Notifications table for web notifications.
+    * Slack app, web or mobile application.
+9.  Open a new terminal for watching kafka topic `external.action.email`
+    ```bash
+    docker exec notification-kafka /opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic external.action.email
+    ```
+10. **Alternative-1:**  Open a new terminal for postgres. It will be used to check if notifications are getting created.
+    ```bash
+    docker exec -it notification-postgres psql -U postgres
+    \c postgres
+    SELECT * FROM "Notifications";
+    ```
+    **Alternative-2:** Start notifications api and use postman to list notifications. Open a new terminal and change PORT to 3000
+    ```bash
+    . ./environment.sh
+    export PORT=3000
+    npm run startAPI
+    ```
+    Open postman and use following collection and environment:
+    * collection: docs/tc-notifications.postman_collection.json
+    * environment: docs/tc-notifications.postman_environment.json
+
+    Existing postman collections are outdated and need to be converted to version 2.0.0. So, I had to create a new collection.
+
+11. To verify slack messages, you can use following workspace and user credentials. Token for posting messages to slack was already shared inside environment variables and has been set at step 3.
+    `https://tc-notifications.slack.com/`
+    username: `tc.notification.slack@gmail.com`
+    password: `@Topcoder123`
+
+
+12. Now we can start testing. Open a new terminal for producing kafka messages for the topic `notification.action.create`
+    ```bash
+    docker exec -it notification-kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic notification.action.create
+    ```
+13. You can copy and paste the following messages to the terminal we opened for producing kafka messages.
+    send messages successfully
+    ```json
+    {"topic":"notification.action.create","originator":"tc-direct","timestamp":"2018-02-16T00:00:00","mime-type":"application\/json","payload":[{"serviceId":"email","type":"taas.notification.request-submitted","details":{"from":"example@example.com","recipients":[{"userId":123,"email":"test1@test.com"},{"userId":456,"email":"test2@test.com"}],"cc":[{"userId":789,"email":"test3@test.com"},{"userId":987,"email":"test4@test.com"}],"data":{"subject":"...","body":"...","field1":"...","field2":"...","filedN":"..."},"sendgridTemplateId":"...","version":"v3"}},{"serviceId":"slack","type":"taas.notification.request-submitted","details":{"channel":"general","text":"test message"}},{"serviceId":"web","type":"taas.notification.request-submitted","details":{"userId":40152856,"contents":{},"version":1}}]}
+    ```
+    email: fail (invalid email), web: success, slack: success
+    ```json
+    {"topic":"notification.action.create","originator":"tc-direct","timestamp":"2018-02-16T00:00:00","mime-type":"application\/json","payload":[{"serviceId":"email","type":"taas.notification.request-submitted","details":{"from":"example.com","recipients":[{"userId":123,"email":"test1@test.com"},{"userId":456,"email":"test2@test.com"}],"cc":[{"userId":789,"email":"test3@test.com"},{"userId":987,"email":"test4@test.com"}],"data":{"subject":"...","body":"...","field1":"...","field2":"...","filedN":"..."},"sendgridTemplateId":"...","version":"v3"}},{"serviceId":"slack","type":"taas.notification.request-submitted","details":{"channel":"random","text":"test message"}},{"serviceId":"web","type":"taas.notification.request-submitted","details":{"userId":40152856,"contents":{},"version":1}}]}
+    ```

local/docker-compose.yml

@@ -0,0 +1,56 @@
+version: "3"
+services:
+  postgres:
+    container_name: notification-postgres
+    image: postgres:11.8
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+    ports:
+      - 5432:5432
+
+  zookeeper:
+    image: wurstmeister/zookeeper
+    container_name: notification-zookeeper
+    ports:
+      - 2181:2181
+
+  kafka:
+    image: wurstmeister/kafka
+    container_name: notification-kafka
+    depends_on:
+      - zookeeper
+    ports:
+      - 9092:9092
+    environment:
+      KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka:9093,OUTSIDE://localhost:9092
+      KAFKA_LISTENERS: INSIDE://kafka:9093,OUTSIDE://0.0.0.0:9092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_CREATE_TOPICS: "challenge.notification.events:1:1,notifications.autopilot.events:1:1,admin.notification.broadcast:1:1,notifications.action.create:1:1,external.action.email:1:1"
+
+  tc-bus-api:
+    container_name: tc-bus-api
+    build:
+      context: ./generic-tc-service
+      args:
+        NODE_VERSION: 8.11.3
+        GIT_URL: https://github.com/topcoder-platform/tc-bus-api
+        GIT_BRANCH: dev
+        BYPASS_TOKEN_VALIDATION: 1
+    command: start
+    ports:
+      - 8002:8002
+    depends_on:
+      - kafka
+    environment:
+      - PORT=8002
+      - KAFKA_URL=kafka:9093
+      - JWT_TOKEN_SECRET=secret
+      - VALID_ISSUERS="[\"https:\/\/topcoder-newauth.auth0.com\/\",\"https:\/\/api.topcoder-dev.com\",\"https:\/\/topcoder-dev.auth0.com\/\"]"
+      - AUTH0_URL
+      - AUTH0_AUDIENCE
+      - AUTH0_CLIENT_ID
+      - AUTH0_CLIENT_SECRET
+      - AUTH0_PROXY_SERVER_URL

local/generic-tc-service/Dockerfile

@@ -0,0 +1,15 @@
+ARG NODE_VERSION=12.16.3
+
+FROM node:$NODE_VERSION
+ARG GIT_URL
+ARG GIT_BRANCH
+ARG BYPASS_TOKEN_VALIDATION
+
+RUN git clone $GIT_URL /opt/app
+WORKDIR /opt/app
+RUN git checkout -b node-branch origin/$GIT_BRANCH
+
+RUN npm install
+RUN if [ $BYPASS_TOKEN_VALIDATION -eq 1 ]; then sed -i '/decodedToken = jwt.decode/a \      callback(undefined, decodedToken.payload); return;' node_modules/tc-core-library-js/lib/auth/verifier.js; fi
+COPY docker-entrypoint.sh /opt/
+ENTRYPOINT ["/opt/docker-entrypoint.sh"]

local/generic-tc-service/docker-entrypoint.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+HOST_DOMAIN="host.docker.internal"
+ping -q -c1 $HOST_DOMAIN > /dev/null 2>&1
+if [ $? -ne 0 ]; then
+  HOST_IP=$(ip route | awk 'NR==1 {print $3}')
+  echo -e "$HOST_IP\t$HOST_DOMAIN" >> /etc/hosts
+fi
+
+cd /opt/app/ && npm run $1