sendgrid v3 Api, response logging and lib vulnerability fixes.

README.md

@@ -13,14 +13,11 @@ Configuration for the notification server is at `config/default.js`.
 The following parameters can be set in config files or in env variables:
 - LOG_LEVEL: the log level
 - PORT: the notification server port
-- authSecret: TC auth secret
-- authDomain: TC auth domain
-- validIssuers: TC auth valid issuers
-- jwksUri: TC auth JWKS URI
+- AUTH_SECRET: TC auth secret
+- VALID_ISSUERS: TC auth valid issuers
 - DATABASE_URL: URI to PostgreSQL database
 - DATABASE_OPTIONS: database connection options
 - KAFKA_URL: comma separated Kafka hosts
-- KAFKA_TOPIC_IGNORE_PREFIX: ignore this prefix for topics in the Kafka
 - KAFKA_GROUP_ID: Kafka consumer group id
 - KAFKA_CLIENT_CERT: Kafka connection certificate, optional;
     if not provided, then SSL connection is not used, direct insecure connection is used;
@@ -59,10 +56,8 @@ In case it expires, you may get a new token in this way:
 
 ## Local deployment
 - for local development environment you can set variables as following:
-  - `authSecret`, `authDomain`, `validIssuers` can get from [tc-project-service config](https://github.com/topcoder-platform/tc-project-service/blob/dev/config/default.json)
+  - `AUTH_SECRET`, `VALID_ISSUERS` can get from [tc-project-service config](https://github.com/topcoder-platform/tc-project-service/blob/dev/config/default.json)
   - `PORT=4001` because **connect-app** call this port by default
-  - `jwksUri` - any
-  - `KAFKA_TOPIC_IGNORE_PREFIX=joan-26673.` (with point at the end)
   - `KAFKA_URL`, `KAFKA_CLIENT_CERT` and `KAFKA_CLIENT_CERT_KEY` get from [tc-bus-api readme](https://github.com/topcoder-platform/tc-bus-api/tree/dev)
 - start local PostgreSQL db, create an empty database, update the config/default.js DATABASE_URL param to point to the db
 - install dependencies `npm i`

config/default.js

@@ -21,7 +21,6 @@ module.exports = {
 
   VALID_ISSUERS: process.env.VALID_ISSUERS,
   KAFKA_URL: process.env.KAFKA_URL,
-  KAFKA_TOPIC_IGNORE_PREFIX: process.env.KAFKA_TOPIC_IGNORE_PREFIX,
   KAFKA_GROUP_ID: process.env.KAFKA_GROUP_ID,
   KAFKA_CLIENT_CERT: process.env.KAFKA_CLIENT_CERT ? process.env.KAFKA_CLIENT_CERT.replace('\\n', '\n') : null,
   KAFKA_CLIENT_CERT_KEY: process.env.KAFKA_CLIENT_CERT_KEY ?
@@ -38,6 +37,8 @@ module.exports = {
 
   //in every 2 minutes will retry for failed status
   EMAIL_RETRY_SCHEDULE: process.env.EMAIL_RETRY_SCHEDULE || '0 */2 * * * *',
+  //wont't retry failed emails older than this time (msec)
+  EMAIL_RETRY_MAX_AGE: process.env.EMAIL_RETRY_MAX_AGE || 1000*60*60*24,
 
   API_CONTEXT_PATH: process.env.API_CONTEXT_PATH || '/v5/email',
 

connect/service.js

@@ -18,18 +18,31 @@ const sendEmail = (templateId, message) => { // send email
   const to = message.recipients;
   const cc =  message.cc ? message.cc : [];
   const bcc = message.bcc ? message.bcc : [];
-
-  return  sgMail.send({
-    to,
-    templateId,
-    substitutions,
-    from,
-    substitutionWrappers: ['{{', '}}'],
-    replyTo,
-    categories,
-    cc,
-    bcc,
-  });
+
+  if (message.version && message.version=="v3"){
+    return  sgMail.send({
+      to,
+      templateId,
+      dynamicTemplateData: substitutions,
+      from,
+      replyTo,
+      categories,
+      cc,
+      bcc,
+    });
+  } else{
+    return  sgMail.send({
+      to,
+      templateId,
+      substitutions,
+      substitutionWrappers: ['{{', '}}'],
+      from,
+      replyTo,
+      categories,
+      cc,
+      bcc,
+    });
+  }
 }
 module.exports = {
   sendEmail,

deploy.sh

@@ -62,6 +62,7 @@ TEMPLATE_MAP=$(eval "echo \$${ENV}_TEMPLATE_MAP")
 EMAIL_MAX_ERRORS=$(eval "echo \$${ENV}_EMAIL_MAX_ERRORS")
 EMAIL_PAUSE_TIME=$(eval "echo \$${ENV}_EMAIL_PAUSE_TIME")
 EMAIL_RETRY_SCHEDULE=$(eval "echo \"\$${ENV}_EMAIL_RETRY_SCHEDULE\"")
+EMAIL_RETRY_MAX_AGE=$(eval "echo \"\$${ENV}_EMAIL_RETRY_MAX_AGE\"")
 DISABLE_LOGGING=$(eval "echo \$${ENV}_DISABLE_LOGGING")
 
 API_CONTEXT_PATH=$(eval "echo \$${ENV}_API_CONTEXT_PATH")
@@ -177,6 +178,10 @@ make_task_def(){
           "name": "EMAIL_RETRY_SCHEDULE",
           "value": "%s"
         },
+        {
+          "name": "EMAIL_RETRY_MAX_AGE",
+          "value": "%s"
+        },
         {
           "name": "DISABLE_LOGGING",
           "value": "%s"
@@ -205,7 +210,7 @@ make_task_def(){
   ]
   }'
 
-  task_def=$(printf "$task_template" $family $AWS_ACCOUNT_ID $AWS_ECS_CONTAINER_NAME $AWS_ACCOUNT_ID $AWS_REGION $AWS_REPOSITORY $TAG $ENV $AUTH_SECRET $DATABASE_URL $EMAIL_FROM "$KAFKA_CLIENT_CERT" "$KAFKA_CLIENT_CERT_KEY" $KAFKA_GROUP_ID $KAFKA_URL $LOG_LEVEL $PORT $SENDGRID_API_KEY "$TEMPLATE_MAP" "$VALID_ISSUERS" $EMAIL_MAX_ERRORS $EMAIL_PAUSE_TIME "$EMAIL_RETRY_SCHEDULE" "$DISABLE_LOGGING" "$API_CONTEXT_PATH" $PORT $PORT $AWS_ECS_CLUSTER $AWS_REGION $ENV)
+  task_def=$(printf "$task_template" $family $AWS_ACCOUNT_ID $AWS_ECS_CONTAINER_NAME $AWS_ACCOUNT_ID $AWS_REGION $AWS_REPOSITORY $TAG $ENV $AUTH_SECRET $DATABASE_URL $EMAIL_FROM "$KAFKA_CLIENT_CERT" "$KAFKA_CLIENT_CERT_KEY" $KAFKA_GROUP_ID $KAFKA_URL $LOG_LEVEL $PORT $SENDGRID_API_KEY "$TEMPLATE_MAP" "$VALID_ISSUERS" $EMAIL_MAX_ERRORS $EMAIL_PAUSE_TIME "$EMAIL_RETRY_SCHEDULE" $EMAIL_RETRY_MAX_AGE "$DISABLE_LOGGING" "$API_CONTEXT_PATH" $PORT $PORT $AWS_ECS_CLUSTER $AWS_REGION $ENV)
 
 }
 

docs/swagger_api.yaml

@@ -15,46 +15,20 @@ securityDefinitions:
     description: JWT Authentication. Provide API Key in the form 'Bearer <token>'.
 
 paths:
-  /templates/{name}:
+  /health:
     get:
       description:
-        get email template placholders name
+        health check endpoint
       produces:
         - application/json
-      security:
-        - jwt: []
-      parameters:
-        - name: name
-          in: path
-          description: The Kafka topic name
-          required: true
-          type: string
       responses:
         200:
           description: OK
           schema:
             type: object
             properties:
-              items:
-                type: array
-                items:
-                  type: string
-        400:
-          description: "Bad request error."
-          schema:
-            $ref: "#/definitions/Error"
-       404:
-         description: "Template is not found error."
-         schema:
-            $ref: "#/definitions/Error"
-        401:
-          description: "Authentication failed."
-          schema:
-            $ref: "#/definitions/Error"
-        500:
-          description: "Internal server error."
-          schema:
-            $ref: "#/definitions/Error"
+              health:
+                type: string
 definitions:
   Error:
     properties: