tc-email-service v1.0

.circleci/config.yml

@@ -0,0 +1,86 @@
+version: 2
+
+jobs:
+  # Build & Deploy against development backend rer
+  "build-dev-deploy-test":
+    docker:
+      - image: docker:17.06.1-ce-git
+    steps:
+      # Initialization.
+      - checkout
+      - setup_remote_docker
+      - run:
+          name: Installation of build dependencies.
+          command: apk add --no-cache bash
+
+      # Restoration of node_modules from cache.
+      - restore_cache:
+          key: docker-tc-email-service-{{ checksum "package-lock.json" }}
+
+      # Build of Docker image.
+      - run:
+          name: Build of Docker image
+          command: ./build.sh DEV
+
+      # Caching node modules.
+      - save_cache:
+          key: docker-tc-email-service-{{ checksum "package-lock.json" }}
+          paths:
+            - node_modules
+
+      # Deployment.
+      - run:
+          name: Installing AWS client
+          command: |
+            apk add --no-cache jq py-pip sudo
+            sudo pip install awscli --upgrade
+
+      - deploy:
+          command: ./deploy.sh DEV $CIRCLE_SHA1
+
+  "build-prod":
+    docker:
+      - image: docker:17.06.1-ce-git
+    steps:
+      # Initialization.
+      - checkout
+      - setup_remote_docker
+      - run:
+          name: Installation of build dependencies.
+          command: apk add --no-cache bash
+
+      # Restoration of node_modules from cache.
+      - restore_cache:
+          key: docker-tc-email-service-{{ checksum "package-lock.json" }}
+
+      # Build of Docker image.
+      - run:
+          name: Build of Docker image
+          command: ./build.sh PROD
+
+      # Caching node modules.
+      - save_cache:
+          key: docker-tc-email-service-{{ checksum "package-lock.json" }}
+          paths:
+            - node_modules
+
+      # Deployment.
+      - run:
+          name: Installing AWS client
+          command: |
+            apk add --no-cache jq py-pip sudo
+            sudo pip install awscli --upgrade
+
+      - deploy:
+          command: ./deploy.sh PROD $CIRCLE_SHA1
+
+workflows:
+  version: 2
+  build:
+    jobs:
+      # Development builds are executed on "develop" branch only.
+      - "build-dev-deploy-test":
+          filters:
+            branches:
+              only: "dev"
+

.gitignore

@@ -0,0 +1,2 @@
+node_modules
+.env

Dockerfile

@@ -0,0 +1,13 @@
+# Builds production version of Community App inside Docker container,
+# and runs it against the specified Topcoder backend (development or
+# production) when container is executed.
+
+FROM node:8.2.1
+LABEL app="tc email" version="1.0"
+
+WORKDIR /opt/app
+COPY . .
+RUN npm install
+RUN npm install dotenv --save
+#RUN npm test
+CMD ["npm", "start"]

README.md

@@ -1,2 +1,101 @@
-# tc-email-service
-General purpose email service
+# TOPCODER EMAIL SERIES - EMAIL SERVER
+
+
+## Dependencies
+- nodejs https://nodejs.org/en/ (v8+)
+- Heroku Toolbelt https://toolbelt.heroku.com
+- git
+- PostgreSQL 9.5
+
+
+## Configuration
+Configuration for the notification server is at `config/default.js`.
+The following parameters can be set in config files or in env variables:
+- LOG_LEVEL: the log level
+- PORT: the notification server port
+- authSecret: TC auth secret
+- authDomain: TC auth domain
+- validIssuers: TC auth valid issuers
+- jwksUri: TC auth JWKS URI
+- DATABASE_URL: URI to PostgreSQL database
+- DATABASE_OPTIONS: database connection options
+- KAFKA_URL: comma separated Kafka hosts
+- KAFKA_TOPIC_IGNORE_PREFIX: ignore this prefix for topics in the Kafka
+- KAFKA_GROUP_ID: Kafka consumer group id
+- KAFKA_CLIENT_CERT: Kafka connection certificate, optional;
+    if not provided, then SSL connection is not used, direct insecure connection is used;
+    if provided, it can be either path to certificate file or certificate content
+- KAFKA_CLIENT_CERT_KEY: Kafka connection private key, optional;
+    if not provided, then SSL connection is not used, direct insecure connection is used;
+    if provided, it can be either path to private key file or private key content
+- TEMPLATE_MAP: the map between topic and SendGrid template id
+
+
+Configuration for the connect notification server is at `connect/config.js`.
+The following parameters can be set in config files or in env variables:
+- TEMPLATE_MAP: the map between topic and SendGrid template id
+- SUBJECT_MAP: the map between topic and SendGrid email subject
+- EMAIL_FROM: from email to use to send email with SendGrid
+- SENDGRID_API_KEY: SendGrid API key
+
+
+Note that the above two configuration are separate because the common notification server config
+will be deployed to a NPM package, the connect notification server will use that NPM package,
+the connection notification server should only use API exposed by the index.js.
+
+## TC API Admin Token
+
+An admin token is needed to access TC API. This is already configured Postman notification
+server API environment TC_ADMIN_TOKEN variable.
+In case it expires, you may get a new token in this way:
+
+- use Chrome to browse connect.topcoder-dev.com
+- open developer tools, click the Network tab
+- log in with suser1 / Topcoder123, or mess / appirio123
+- once logged in, open some project, for example https://connect.topcoder-dev.com/projects/1936 and in the network inspector
+  look for the call to the project api and get the token from the auth header, see
+  http://pokit.org/get/img/68cdd34f3d205d6d9bd8bddb07bdc216.jpg
+
+
+## Local deployment
+- for local development environment you can set variables as following:
+  - `authSecret`, `authDomain`, `validIssuers` can get from [tc-project-service config](https://github.com/topcoder-platform/tc-project-service/blob/dev/config/default.json)
+  - `PORT=4001` because **connect-app** call this port by default
+  - `jwksUri` - any
+  - `KAFKA_TOPIC_IGNORE_PREFIX=joan-26673.` (with point at the end)
+  - `KAFKA_URL`, `KAFKA_CLIENT_CERT` and `KAFKA_CLIENT_CERT_KEY` get from [tc-bus-api readme](https://github.com/topcoder-platform/tc-bus-api/tree/dev)
+- start local PostgreSQL db, create an empty database, update the config/default.js DATABASE_URL param to point to the db
+- install dependencies `npm i`
+- run code lint check `npm run lint`
+- start connect notification server `npm start`
+- the app is running at `http://localhost:4001`, it also starts Kafka consumer to listen for events, send emails using SendGrid and save emails data to database
+
+
+## Heroku deployment
+
+- git init
+- git add .
+- git commit -m 'message'
+- heroku login
+- heroku create [application-name] // choose a name, or leave it empty to use generated one
+- heroku addons:create heroku-postgresql:hobby-dev
+- note that you may need to wait for several minutes before the PostgreSQL database is ready
+- optionally, to set some environment variables in heroku, run command like:
+  `heroku config:set KAFKA_CLIENT_CERT=path/to/certificate/file`
+  `heroku config:set KAFKA_CLIENT_CERT_KEY=path/to/private/key/file`
+  `heroku config:set KAFKA_GROUP_ID=some-group`
+  etc.
+- git push heroku master // push code to Heroku
+
+
+## Verification
+
+- start the app following above sections
+- Import `docs/tc-email-server-api-local-env.postman_environment.json` and `docs/tc-email-server-api.postman_collection.json` to Postman
+- in Postman, using the email server API collection and environment to run the tests
+
+
+## Swagger
+
+Swagger API definition is provided at `docs/swagger_api.yaml`,
+you may check it at `http://editor.swagger.io`.

build.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+set -eo pipefail
+
+# Builds Docker image of Community App application.
+# This script expects a single argument: NODE_ENV, which must be either
+# "development" or "production".
+
+NODE_ENV=$1
+
+ENV=$1
+AWS_REGION=$(eval "echo \$${ENV}_AWS_REGION")
+AWS_ACCESS_KEY_ID=$(eval "echo \$${ENV}_AWS_ACCESS_KEY_ID")
+AWS_SECRET_ACCESS_KEY=$(eval "echo \$${ENV}_AWS_SECRET_ACCESS_KEY")
+AWS_ACCOUNT_ID=$(eval "echo \$${ENV}_AWS_ACCOUNT_ID")
+AWS_REPOSITORY=$(eval "echo \$${ENV}_AWS_REPOSITORY")
+
+#App variables
+
+AUTHDOMAIN=$(eval "echo \$${ENV}_AUTHDOMAIN")
+AUTHSECRET=$(eval "echo \$${ENV}_AUTHSECRET")
+VALIDISSUERS=$(eval "echo \$${ENV}_VALIDISSUERS")
+
+KAFKA_CLIENT_CERT=$(eval "echo \$${ENV}_KAFKA_CLIENT_CERT")
+KAFKA_CLIENT_CERT_KEY=$(eval "echo \$${ENV}_KAFKA_CLIENT_CERT_KEY")
+KAFKA_URL=$(eval "echo \$${ENV}_KAFKA_URL")
+SENDGRID_API_KEY=$(eval "echo \$${ENV}_SENDGRID_API_KEY")
+
+
+DB_DATABASE=$(eval "echo \$${ENV}_DB_DATABASE")
+DB_HOST=$(eval "echo \$${ENV}_DB_HOST")
+DB_PASSWORD=$(eval "echo \$${ENV}_DB_PASSWORD")
+DB_PORT=$(eval "echo \$${ENV}_DB_PORT")
+DB_USER=$(eval "echo \$${ENV}_DB_USER")
+DATABASE_URL=postgres://$DB_USER:$DB_PASSWORD@$DB_HOST:$DB_PORT/$DB_DATABASE;
+
+
+KAFKA_GROUP_ID=$(eval "echo \$${ENV}_KAFKA_GROUP_ID")
+EMAIL_FROM=$(eval "echo \$${ENV}_EMAIL_FROM")
+LOG_LEVEL=$(eval "echo \$${ENV}_LOG_LEVEL")
+NODE_ENV=$(eval "echo \$${ENV}_NODE_ENV")
+NODE_PORT=$(eval "echo \$${ENV}_NODE_PORT")
+JWKSURI=$(eval "echo \$${ENV}_JWKSURI")
+TEMPLATE_MAP=$(eval "echo \$${ENV}_TEMPLATE_MAP")
+
+
+TAG=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/tc-email-service:$CIRCLE_SHA1
+
+docker build -t $TAG .
+
+# Copies "node_modules" from the created image, if necessary for caching.
+docker create --name app $TAG
+
+if [ -d node_modules ]
+then
+  # If "node_modules" directory already exists, we should compare
+  # "package-lock.json" from the code and from the container to decide,
+  # whether we need to re-cache, and thus to copy "node_modules" from
+  # the Docker container.
+  mv package-lock.json old-package-lock.json
+  docker cp app:/opt/app/package-lock.json package-lock.json
+ # docker cp .env app:/opt/app/
+  set +eo pipefail
+  UPDATE_CACHE=$(cmp package-lock.json old-package-lock.json)
+  set -eo pipefail
+else
+  # If "node_modules" does not exist, then cache must be created.
+  UPDATE_CACHE=1
+fi
+
+if [ "$UPDATE_CACHE" == 1 ]
+then
+  docker cp app:/opt/app/node_modules .
+fi
+
+

config/default.js

@@ -0,0 +1,39 @@
+/**
+ * The configuration file.
+ */
+module.exports = {
+  LOG_LEVEL: process.env.LOG_LEVEL,
+  PORT: process.env.PORT,
+  authSecret: process.env.authSecret,
+  authDomain: process.env.authDomain,
+  jwksUri: process.env.jwksUri,
+  DATABASE_URL: process.env.DATABASE_URL,
+  DATABASE_OPTIONS: {
+    dialect: 'postgres',
+    dialectOptions: {
+      ssl: process.env.DATABASE_SSL != null,
+    },
+    pool: {
+      max: 5,
+      min: 0,
+      idle: 10000,
+    },
+  },
+  DISABLE_LOGGING: process.env.DISABLE_LOGGING || 'false',
+
+  validIssuers: process.env.validIssuers ? process.env.validIssuers.replace(/\\"/g, '') : null,
+  KAFKA_URL: process.env.KAFKA_URL,
+  KAFKA_TOPIC_IGNORE_PREFIX: process.env.KAFKA_TOPIC_IGNORE_PREFIX,
+  KAFKA_GROUP_ID: process.env.KAFKA_GROUP_ID,
+  KAFKA_CLIENT_CERT: process.env.KAFKA_CLIENT_CERT ? process.env.KAFKA_CLIENT_CERT.replace('\\n', '\n') : null,
+  KAFKA_CLIENT_CERT_KEY: process.env.KAFKA_CLIENT_CERT_KEY ?
+    process.env.KAFKA_CLIENT_CERT_KEY.replace('\\n', '\n') : null,
+
+  // mapping from event type to sendgrid email template id
+  TEMPLATE_MAP: process.env.TEMPLATE_MAP,
+  SENDGRID_API_KEY: process.env.SENDGRID_API_KEY || '',
+  EMAIL_FROM: process.env.EMAIL_FROM || 'no-reply@topcoder.com',
+  EMAIL_MAX_ERRORS: process.env.EMAIL_MAX_ERRORS || 2,
+  EMAIL_PAUSE_TIME: process.env.EMAIL_PAUSE_TIME || 30,
+  EMAIL_RETRY_SCHEDULE: process.env.EMAIL_RETRY_SCHEDULE || '0 */2 * * *',
+};

config/development.js

@@ -0,0 +1,7 @@
+/**
+ * The development configuration file. These config will override default config.
+ */
+module.exports = {
+  LOG_LEVEL: process.env.LOG_LEVEL || 'debug',
+  PORT: process.env.PORT || 4000,
+};

config/production.js

@@ -0,0 +1,8 @@
+/**
+ * The production configuration file. These config will override default config.
+ */
+module.exports = {
+  LOG_LEVEL: process.env.LOG_LEVEL || 'error',
+  PORT: process.env.PORT || 4000,
+  DISABLE_LOGGING: process.env.DISABLE_LOGGING || 'true',
+};

config/test.js

@@ -0,0 +1,7 @@
+/**
+ * The test configuration file. These config will override default config.
+ */
+module.exports = {
+  LOG_LEVEL: process.env.LOG_LEVEL || 'debug',
+  PORT: process.env.PORT || 4000,
+};

connect/connectEmailServer.js

@@ -0,0 +1,48 @@
+/**
+ * This is TopCoder connect email server.
+ */
+'use strict';
+
+global.Promise = require('bluebird');
+
+const _ = require('lodash');
+const config = require('config');
+const emailServer = require('../index');
+const service = require('./service');
+
+// set configuration for the server, see ../config/default.js for available config parameters
+// setConfig should be called before initDatabase and start functions
+emailServer.setConfig({ LOG_LEVEL: 'debug' });
+
+// add topic handlers,
+// handler is used build a notification list for a message of a topic,
+// it is defined as: function(topic, message, callback),
+// the topic is topic name,
+// the message is JSON event message,
+// the callback is function(error, templateId), where templateId is the used SendGrid template id
+const handler = (topic, message, callback) => {
+  const templateId = config.TEMPLATE_MAP[topic];
+  if (templateId === undefined) {
+    return callback(null, { success: false, error: `Template not found for topic ${topic}` });
+  }
+
+  service.sendEmail(templateId, message.recipients, message.data).then(() => {
+    callback(null, { success: true });
+  }).catch((err) => {
+    callback(null, { success: false, error: err });
+  });
+};
+
+// init all events
+_.keys(config.TEMPLATE_MAP).forEach((eventType) => {
+  emailServer.addTopicHandler(eventType, handler);
+});
+
+// init database, it will clear and re-create all tables
+emailServer
+  .initDatabase()
+  .then(() => emailServer.start())
+  .catch((e) => console.log(e)); // eslint-disable-line no-console
+
+// if no need to init database, then directly start the server:
+// emailServer.start();