Skip to content

Commit 3c36b8a

Browse files
author
sachin-maheshwari
authored
Merge pull request #2 from topcoder-platform/dev
Merging with master without deployment
2 parents bcf4792 + 78fca5b commit 3c36b8a

33 files changed

+5091
-2
lines changed

‎.circleci/config.yml

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
version: 2
2+
3+
jobs:
4+
# Build & Deploy against development backend rer
5+
"build-dev-deploy-test":
6+
docker:
7+
- image: docker:17.06.1-ce-git
8+
steps:
9+
# Initialization.
10+
- checkout
11+
- setup_remote_docker
12+
- run:
13+
name: Installation of build dependencies.
14+
command: apk add --no-cache bash
15+
16+
# Restoration of node_modules from cache.
17+
- restore_cache:
18+
key: docker-tc-email-service-{{ checksum "package-lock.json" }}
19+
20+
# Build of Docker image.
21+
- run:
22+
name: Build of Docker image
23+
command: ./build.sh DEV
24+
25+
# Caching node modules.
26+
- save_cache:
27+
key: docker-tc-email-service-{{ checksum "package-lock.json" }}
28+
paths:
29+
- node_modules
30+
31+
# Deployment.
32+
- run:
33+
name: Installing AWS client
34+
command: |
35+
apk add --no-cache jq py-pip sudo
36+
sudo pip install awscli --upgrade
37+
38+
- deploy:
39+
command: ./deploy.sh DEV $CIRCLE_SHA1
40+
41+
"build-prod":
42+
docker:
43+
- image: docker:17.06.1-ce-git
44+
steps:
45+
# Initialization.
46+
- checkout
47+
- setup_remote_docker
48+
- run:
49+
name: Installation of build dependencies.
50+
command: apk add --no-cache bash
51+
52+
# Restoration of node_modules from cache.
53+
- restore_cache:
54+
key: docker-tc-email-service-{{ checksum "package-lock.json" }}
55+
56+
# Build of Docker image.
57+
- run:
58+
name: Build of Docker image
59+
command: ./build.sh PROD
60+
61+
# Caching node modules.
62+
- save_cache:
63+
key: docker-tc-email-service-{{ checksum "package-lock.json" }}
64+
paths:
65+
- node_modules
66+
67+
# Deployment.
68+
- run:
69+
name: Installing AWS client
70+
command: |
71+
apk add --no-cache jq py-pip sudo
72+
sudo pip install awscli --upgrade
73+
74+
- deploy:
75+
command: ./deploy.sh PROD $CIRCLE_SHA1
76+
77+
workflows:
78+
version: 2
79+
build:
80+
jobs:
81+
# Development builds are executed on "develop" branch only.
82+
- "build-dev-deploy-test":
83+
filters:
84+
branches:
85+
only: "dev"
86+

‎.eslintrc

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
"parserOptions": {
2+
"ecmaVersion": 6
3+
}

‎.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
node_modules
2+
.env

‎Dockerfile

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# Builds production version of Community App inside Docker container,
2+
# and runs it against the specified Topcoder backend (development or
3+
# production) when container is executed.
4+
5+
FROM node:8.2.1
6+
LABEL app="tc email" version="1.0"
7+
8+
WORKDIR /opt/app
9+
COPY . .
10+
RUN npm install
11+
RUN npm install dotenv --save
12+
RUN npm run lint
13+
CMD ["npm", "start"]

‎README.md

Lines changed: 101 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,101 @@
1-
# tc-email-service
2-
General purpose email service
1+
# TOPCODER EMAIL SERIES - EMAIL SERVER
2+
3+
4+
## Dependencies
5+
- nodejs https://nodejs.org/en/ (v8+)
6+
- Heroku Toolbelt https://toolbelt.heroku.com
7+
- git
8+
- PostgreSQL 9.5
9+
10+
11+
## Configuration
12+
Configuration for the notification server is at `config/default.js`.
13+
The following parameters can be set in config files or in env variables:
14+
- LOG_LEVEL: the log level
15+
- PORT: the notification server port
16+
- authSecret: TC auth secret
17+
- authDomain: TC auth domain
18+
- validIssuers: TC auth valid issuers
19+
- jwksUri: TC auth JWKS URI
20+
- DATABASE_URL: URI to PostgreSQL database
21+
- DATABASE_OPTIONS: database connection options
22+
- KAFKA_URL: comma separated Kafka hosts
23+
- KAFKA_TOPIC_IGNORE_PREFIX: ignore this prefix for topics in the Kafka
24+
- KAFKA_GROUP_ID: Kafka consumer group id
25+
- KAFKA_CLIENT_CERT: Kafka connection certificate, optional;
26+
if not provided, then SSL connection is not used, direct insecure connection is used;
27+
if provided, it can be either path to certificate file or certificate content
28+
- KAFKA_CLIENT_CERT_KEY: Kafka connection private key, optional;
29+
if not provided, then SSL connection is not used, direct insecure connection is used;
30+
if provided, it can be either path to private key file or private key content
31+
- TEMPLATE_MAP: the map between topic and SendGrid template id
32+
33+
34+
Configuration for the connect notification server is at `connect/config.js`.
35+
The following parameters can be set in config files or in env variables:
36+
- TEMPLATE_MAP: the map between topic and SendGrid template id
37+
- SUBJECT_MAP: the map between topic and SendGrid email subject
38+
- EMAIL_FROM: from email to use to send email with SendGrid
39+
- SENDGRID_API_KEY: SendGrid API key
40+
41+
42+
Note that the above two configuration are separate because the common notification server config
43+
will be deployed to a NPM package, the connect notification server will use that NPM package,
44+
the connection notification server should only use API exposed by the index.js.
45+
46+
## TC API Admin Token
47+
48+
An admin token is needed to access TC API. This is already configured Postman notification
49+
server API environment TC_ADMIN_TOKEN variable.
50+
In case it expires, you may get a new token in this way:
51+
52+
- use Chrome to browse connect.topcoder-dev.com
53+
- open developer tools, click the Network tab
54+
- log in with suser1 / Topcoder123, or mess / appirio123
55+
- once logged in, open some project, for example https://connect.topcoder-dev.com/projects/1936 and in the network inspector
56+
look for the call to the project api and get the token from the auth header, see
57+
http://pokit.org/get/img/68cdd34f3d205d6d9bd8bddb07bdc216.jpg
58+
59+
60+
## Local deployment
61+
- for local development environment you can set variables as following:
62+
- `authSecret`, `authDomain`, `validIssuers` can get from [tc-project-service config](https://github.com/topcoder-platform/tc-project-service/blob/dev/config/default.json)
63+
- `PORT=4001` because **connect-app** call this port by default
64+
- `jwksUri` - any
65+
- `KAFKA_TOPIC_IGNORE_PREFIX=joan-26673.` (with point at the end)
66+
- `KAFKA_URL`, `KAFKA_CLIENT_CERT` and `KAFKA_CLIENT_CERT_KEY` get from [tc-bus-api readme](https://github.com/topcoder-platform/tc-bus-api/tree/dev)
67+
- start local PostgreSQL db, create an empty database, update the config/default.js DATABASE_URL param to point to the db
68+
- install dependencies `npm i`
69+
- run code lint check `npm run lint`
70+
- start connect notification server `npm start`
71+
- the app is running at `http://localhost:4001`, it also starts Kafka consumer to listen for events, send emails using SendGrid and save emails data to database
72+
73+
74+
## Heroku deployment
75+
76+
- git init
77+
- git add .
78+
- git commit -m 'message'
79+
- heroku login
80+
- heroku create [application-name] // choose a name, or leave it empty to use generated one
81+
- heroku addons:create heroku-postgresql:hobby-dev
82+
- note that you may need to wait for several minutes before the PostgreSQL database is ready
83+
- optionally, to set some environment variables in heroku, run command like:
84+
`heroku config:set KAFKA_CLIENT_CERT=path/to/certificate/file`
85+
`heroku config:set KAFKA_CLIENT_CERT_KEY=path/to/private/key/file`
86+
`heroku config:set KAFKA_GROUP_ID=some-group`
87+
etc.
88+
- git push heroku master // push code to Heroku
89+
90+
91+
## Verification
92+
93+
- start the app following above sections
94+
- Import `docs/tc-email-server-api-local-env.postman_environment.json` and `docs/tc-email-server-api.postman_collection.json` to Postman
95+
- in Postman, using the email server API collection and environment to run the tests
96+
97+
98+
## Swagger
99+
100+
Swagger API definition is provided at `docs/swagger_api.yaml`,
101+
you may check it at `http://editor.swagger.io`.

‎build.sh

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
#!/bin/bash
2+
set -eo pipefail
3+
4+
# Builds Docker image of Community App application.
5+
# This script expects a single argument: NODE_ENV, which must be either
6+
# "development" or "production".
7+
8+
NODE_ENV=$1
9+
10+
ENV=$1
11+
AWS_REGION=$(eval "echo \$${ENV}_AWS_REGION")
12+
AWS_ACCESS_KEY_ID=$(eval "echo \$${ENV}_AWS_ACCESS_KEY_ID")
13+
AWS_SECRET_ACCESS_KEY=$(eval "echo \$${ENV}_AWS_SECRET_ACCESS_KEY")
14+
AWS_ACCOUNT_ID=$(eval "echo \$${ENV}_AWS_ACCOUNT_ID")
15+
AWS_REPOSITORY=$(eval "echo \$${ENV}_AWS_REPOSITORY")
16+
17+
#App variables
18+
19+
AUTHDOMAIN=$(eval "echo \$${ENV}_AUTHDOMAIN")
20+
AUTHSECRET=$(eval "echo \$${ENV}_AUTHSECRET")
21+
VALIDISSUERS=$(eval "echo \$${ENV}_VALIDISSUERS")
22+
23+
KAFKA_CLIENT_CERT=$(eval "echo \$${ENV}_KAFKA_CLIENT_CERT")
24+
KAFKA_CLIENT_CERT_KEY=$(eval "echo \$${ENV}_KAFKA_CLIENT_CERT_KEY")
25+
KAFKA_URL=$(eval "echo \$${ENV}_KAFKA_URL")
26+
SENDGRID_API_KEY=$(eval "echo \$${ENV}_SENDGRID_API_KEY")
27+
28+
29+
DB_DATABASE=$(eval "echo \$${ENV}_DB_DATABASE")
30+
DB_HOST=$(eval "echo \$${ENV}_DB_HOST")
31+
DB_PASSWORD=$(eval "echo \$${ENV}_DB_PASSWORD")
32+
DB_PORT=$(eval "echo \$${ENV}_DB_PORT")
33+
DB_USER=$(eval "echo \$${ENV}_DB_USER")
34+
DATABASE_URL=postgres://$DB_USER:$DB_PASSWORD@$DB_HOST:$DB_PORT/$DB_DATABASE;
35+
36+
37+
KAFKA_GROUP_ID=$(eval "echo \$${ENV}_KAFKA_GROUP_ID")
38+
EMAIL_FROM=$(eval "echo \$${ENV}_EMAIL_FROM")
39+
LOG_LEVEL=$(eval "echo \$${ENV}_LOG_LEVEL")
40+
NODE_ENV=$(eval "echo \$${ENV}_NODE_ENV")
41+
NODE_PORT=$(eval "echo \$${ENV}_NODE_PORT")
42+
JWKSURI=$(eval "echo \$${ENV}_JWKSURI")
43+
TEMPLATE_MAP=$(eval "echo \$${ENV}_TEMPLATE_MAP")
44+
45+
46+
TAG=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/tc-email-service:$CIRCLE_SHA1
47+
48+
docker build -t $TAG .
49+
50+
# Copies "node_modules" from the created image, if necessary for caching.
51+
docker create --name app $TAG
52+
53+
if [ -d node_modules ]
54+
then
55+
# If "node_modules" directory already exists, we should compare
56+
# "package-lock.json" from the code and from the container to decide,
57+
# whether we need to re-cache, and thus to copy "node_modules" from
58+
# the Docker container.
59+
mv package-lock.json old-package-lock.json
60+
docker cp app:/opt/app/package-lock.json package-lock.json
61+
# docker cp .env app:/opt/app/
62+
set +eo pipefail
63+
UPDATE_CACHE=$(cmp package-lock.json old-package-lock.json)
64+
set -eo pipefail
65+
else
66+
# If "node_modules" does not exist, then cache must be created.
67+
UPDATE_CACHE=1
68+
fi
69+
70+
if [ "$UPDATE_CACHE" == 1 ]
71+
then
72+
docker cp app:/opt/app/node_modules .
73+
fi
74+
75+

‎config/default.js

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
/**
2+
* The configuration file.
3+
*/
4+
module.exports = {
5+
LOG_LEVEL: process.env.LOG_LEVEL,
6+
PORT: process.env.PORT,
7+
authSecret: process.env.authSecret,
8+
authDomain: process.env.authDomain,
9+
jwksUri: process.env.jwksUri,
10+
DATABASE_URL: process.env.DATABASE_URL,
11+
DATABASE_OPTIONS: {
12+
dialect: 'postgres',
13+
dialectOptions: {
14+
ssl: process.env.DATABASE_SSL != null,
15+
},
16+
pool: {
17+
max: 5,
18+
min: 0,
19+
idle: 10000,
20+
},
21+
},
22+
DISABLE_LOGGING: process.env.DISABLE_LOGGING || 'false',
23+
24+
validIssuers: process.env.validIssuers ? process.env.validIssuers.replace(/\\"/g, '') : null,
25+
KAFKA_URL: process.env.KAFKA_URL,
26+
KAFKA_TOPIC_IGNORE_PREFIX: process.env.KAFKA_TOPIC_IGNORE_PREFIX,
27+
KAFKA_GROUP_ID: process.env.KAFKA_GROUP_ID,
28+
KAFKA_CLIENT_CERT: process.env.KAFKA_CLIENT_CERT ? process.env.KAFKA_CLIENT_CERT.replace('\\n', '\n') : null,
29+
KAFKA_CLIENT_CERT_KEY: process.env.KAFKA_CLIENT_CERT_KEY ?
30+
process.env.KAFKA_CLIENT_CERT_KEY.replace('\\n', '\n') : null,
31+
32+
// mapping from event type to sendgrid email template id
33+
TEMPLATE_MAP: process.env.TEMPLATE_MAP,
34+
SENDGRID_API_KEY: process.env.SENDGRID_API_KEY || '',
35+
EMAIL_FROM: process.env.EMAIL_FROM || 'no-reply@topcoder.com',
36+
37+
// temporary not in use feature
38+
EMAIL_MAX_ERRORS: process.env.EMAIL_MAX_ERRORS || 2,
39+
EMAIL_PAUSE_TIME: process.env.EMAIL_PAUSE_TIME || 30,
40+
41+
//in every 2 minutes will retry for failed status
42+
EMAIL_RETRY_SCHEDULE: process.env.EMAIL_RETRY_SCHEDULE || '0 */2 * * * *',
43+
};

‎config/development.js

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
/**
2+
* The development configuration file. These config will override default config.
3+
*/
4+
module.exports = {
5+
LOG_LEVEL: process.env.LOG_LEVEL || 'debug',
6+
PORT: process.env.PORT || 4000,
7+
};

‎config/production.js

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
/**
2+
* The production configuration file. These config will override default config.
3+
*/
4+
module.exports = {
5+
LOG_LEVEL: process.env.LOG_LEVEL || 'error',
6+
PORT: process.env.PORT || 4000,
7+
DISABLE_LOGGING: process.env.DISABLE_LOGGING || 'true',
8+
};

‎config/test.js

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
/**
2+
* The test configuration file. These config will override default config.
3+
*/
4+
module.exports = {
5+
LOG_LEVEL: process.env.LOG_LEVEL || 'debug',
6+
PORT: process.env.PORT || 4000,
7+
};

0 commit comments

Comments
 (0)