Merge pull request #186 from topcoder-platform/dev

Production Release: To support Connect 2.4.6

Author: vikasrohit

.circleci/config.yml

@@ -76,7 +76,7 @@ workflows:
             - test
           filters:
             branches:
-              only: ['dev', 'feature/timeline-milestone']
+              only: ['dev']
       - deployProd:
           requires:
             - test

.gitignore

@@ -1,6 +1,8 @@
 
 # Created by https://www.gitignore.io/api/node
 config/local.js
+# can be used locally to config some env variables and after apply them using `source .env`
+.env
 ### Node ###
 # Logs
 logs

README.md

@@ -18,7 +18,7 @@ Microservice to manage CRUD operations for all things Projects.
 Copy config/sample.local.js as config/local.js, update the properties and according to your env setup
 
 #### Database
-Once you start your PostgreSQL database through docker, it will create a projectsDB.
+Once you start your PostgreSQL database through docker, it will create a projectsdb.
 *To create tables - note this will drop tables if they already exist*
 ```
 NODE_ENV=development npm run sync:db
@@ -40,6 +40,33 @@ Run `npm run sync:es` from the root of project to execute the script.
 
 **NOTE**: In production these dependencies / services are hosted & managed outside tc-projects-service.
 
+#### Kafka
+Kafka must be installed and configured prior starting the application.
+Following topics must be created:
+```
+notifications.connect.project.updated
+notifications.connect.project.files.updated
+notifications.connect.project.team.updated
+notifications.connect.project.plan.updated
+notifications.connect.project.topic.created
+notifications.connect.project.topic.updated
+notifications.connect.project.post.created
+notifications.connect.project.post.edited
+```
+
+New Kafka related configuration options has been introduced:
+```
+"kafkaConfig": {
+    "hosts": List of Kafka brokers. Default: localhost: 9092
+    "clientCert": SSL certificate
+    "clientCertKey": Certificate key
+}
+```
+Environment variables:
+KAFKA_HOSTS - same as "hosts"
+KAFKA_CLIENT_CERT - same as "clientCert"
+KAFKA_CLIENT_CERT_KEY - same as "clientCertKey"
+
 ### Test
 
 Each of the individual modules/services are unit tested.

config/custom-environment-variables.json

@@ -4,6 +4,7 @@
   "logLevel": "LOG_LEVEL",
   "version": "APP_VERSION",
   "captureLogs": "CAPTURE_LOGS",
+  "enableFileUpload": "ENABLE_FILE_UPLOAD",
   "logentriesToken": "LOGENTRIES_TOKEN",
   "elasticsearchConfig": {
     "host": "PROJECTS_ES_URL",
@@ -29,6 +30,12 @@
     "maxPoolSize": "DB_MAX_POOL_SIZE",
     "minPoolSize": "DB_MIN_POOL_SIZE"
   },
+  "kafkaConfig": {
+    "groupId": "KAFKA_GROUP_ID",
+    "url": "KAFKA_URL",
+    "clientCert": "KAFKA_CLIENT_CERT",
+    "clientCertKey": "KAFKA_CLIENT_CERT_KEY"
+  },
   "analyticsKey": "SEGMENT_ANALYTICS_KEY",
   "VALID_ISSUERS": "VALID_ISSUERS",
   "jwksUri": "JWKS_URI",
@@ -38,5 +45,6 @@
   "AUTH0_CLIENT_ID": "AUTH0_CLIENT_ID",
   "AUTH0_CLIENT_SECRET": "AUTH0_CLIENT_SECRET",
   "AUTH0_AUDIENCE": "AUTH0_AUDIENCE",
-  "TOKEN_CACHE_TIME" : "TOKEN_CACHE_TIME"
+  "TOKEN_CACHE_TIME" : "TOKEN_CACHE_TIME",
+  "whitelistedOriginsForUserIdAuth": "WHITELISTED_ORIGINS_FOR_USERID_AUTH"
 }

config/default.json

@@ -4,6 +4,7 @@
   "logLevel": "info",
   "version": "v4",
   "captureLogs": "false",
+  "enableFileUpload": "true",
   "logentriesToken": "",
   "rabbitmqURL": "",
   "pubsubQueueName": "project.service",
@@ -33,6 +34,8 @@
     "minPoolSize": 4,
     "idleTimeout": 1000
   },
+  "kafkaConfig": {
+  },
   "analyticsKey": "",
   "VALID_ISSUERS": "[\"https:\/\/topcoder-newauth.auth0.com\/\",\"https:\/\/api.topcoder-dev.com\"]",
   "validIssuers": "[\"https:\/\/topcoder-newauth.auth0.com\/\",\"https:\/\/api.topcoder-dev.com\"]",
@@ -46,5 +49,6 @@
   "AUTH0_CLIENT_SECRET": "",
   "AUTH0_AUDIENCE": "",
   "AUTH0_URL": "",
-  "TOKEN_CACHE_TIME": ""
+  "TOKEN_CACHE_TIME": "",
+  "whitelistedOriginsForUserIdAuth": "[\"https:\/\/topcoder-newauth.auth0.com\/\",\"https:\/\/api.topcoder-dev.com\"]"
 }

config/sample.local.js

@@ -5,6 +5,7 @@ if (process.env.NODE_ENV === 'test') {
   config = require('./test.json');
 } else {
   config = {
+    identityServiceEndpoint: "http://dockerhost:3001/",
     authSecret: 'secret',
     authDomain: 'topcoder-dev.com',
     logLevel: 'debug',
@@ -14,9 +15,9 @@ if (process.env.NODE_ENV === 'test') {
     fileServiceEndpoint: 'https://api.topcoder-dev.com/v3/files/',
     directProjectServiceEndpoint: 'https://api.topcoder-dev.com/v3/direct',
     connectProjectsUrl: 'https://connect.topcoder-dev.com/projects/',
-    memberServiceEndpoint: 'http://dockerhost:3001/members',
+    memberServiceEndpoint: 'http://dockerhost:3001/v3/members',
     dbConfig: {
-      masterUrl: 'postgres://coder:mysecretpassword@dockerhost:54321/projectsdb',
+      masterUrl: 'postgres://coder:mysecretpassword@dockerhost:5432/projectsdb',
       maxPoolSize: 50,
       minPoolSize: 4,
       idleTimeout: 1000,
@@ -28,6 +29,7 @@ if (process.env.NODE_ENV === 'test') {
       indexName: 'projects',
       docType: 'projectV4'
     },
+    whitelistedOriginsForUserIdAuth: "[\"\"]",
   };
 }
 module.exports = config;

config/test.json

@@ -18,5 +18,6 @@
         "maxPoolSize": 50,
         "minPoolSize": 4,
         "idleTimeout": 1000
-    }
+    },
+    "whitelistedOriginsForUserIdAuth": "[\"\"]"
 }

deploy.sh

@@ -54,13 +54,17 @@ make_task_def(){
       "name": "%s",
       "image": "%s.dkr.ecr.%s.amazonaws.com/%s:%s",
       "essential": true,
-      "memory": 200,
-      "cpu": 10,
+      "memory": 1536,
+      "cpu": 768,
       "environment": [
         {
           "name": "NODE_ENV",
           "value": "%s"
         },
+        {
+          "name": "ENABLE_FILE_UPLOAD",
+          "value": "%s"
+        },
         {
           "name": "LOG_LEVEL",
           "value": "%s"
@@ -176,6 +180,22 @@ make_task_def(){
         {
           "name": "TOKEN_CACHE_TIME",
           "value": "%s"
+        },
+        {
+          "name": "KAFKA_CLIENT_CERT",
+          "value": "%s"
+        },
+        {
+          "name": "KAFKA_CLIENT_CERT_KEY",
+          "value": "%s"
+        },
+        {
+          "name": "KAFKA_GROUP_ID",
+          "value": "%s"
+        },
+        {
+          "name": "KAFKA_URL",
+          "value": "%s"
         }
       ],
       "portMappings": [
@@ -220,15 +240,20 @@ make_task_def(){
   fi
   echo "NODE_ENV"
   echo $NODE_ENV
+  ENABLE_FILE_UPLOAD=$(eval "echo \$${ENV}_ENABLE_FILE_UPLOAD")
 
   AUTH0_URL=$(eval "echo \$${ENV}_AUTH0_URL")
   AUTH0_AUDIENCE=$(eval "echo \$${ENV}_AUTH0_AUDIENCE")
   AUTH0_CLIENT_ID=$(eval "echo \$${ENV}_AUTH0_CLIENT_ID")
   AUTH0_CLIENT_SECRET=$(eval "echo \$${ENV}_AUTH0_CLIENT_SECRET")
   TOKEN_CACHE_TIME=$(eval "echo \$${ENV}_TOKEN_CACHE_TIME")
+  KAFKA_CLIENT_CERT=$(eval "echo \$${ENV}_KAFKA_CLIENT_CERT")
+  KAFKA_CLIENT_CERT_KEY=$(eval "echo \$${ENV}_KAFKA_CLIENT_CERT_KEY")
+  KAFKA_GROUP_ID=$(eval "echo \$${ENV}_KAFKA_GROUP_ID")
+  KAFKA_URL=$(eval "echo \$${ENV}_KAFKA_URL")
 
 
-  task_def=$(printf "$task_template" $family $ACCOUNT_ID $AWS_ECS_CONTAINER_NAME $ACCOUNT_ID $AWS_REGION $AWS_REPOSITORY $CIRCLE_SHA1 $NODE_ENV $LOG_LEVEL $CAPTURE_LOGS $LOGENTRIES_TOKEN $API_VERSION $AWS_REGION $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY $AUTH_DOMAIN $AUTH_SECRET $VALID_ISSUERS $DB_MASTER_URL $MEMBER_SERVICE_ENDPOINT $IDENTITY_SERVICE_ENDPOINT $BUS_API_URL $MESSAGE_SERVICE_URL $SYSTEM_USER_CLIENT_ID $SYSTEM_USER_CLIENT_SECRET $PROJECTS_ES_URL $PROJECTS_ES_INDEX_NAME $RABBITMQ_URL $DIRECT_PROJECT_SERVICE_ENDPOINT $FILE_SERVICE_ENDPOINT $CONNECT_PROJECTS_URL $SEGMENT_ANALYTICS_KEY "$AUTH0_URL" "$AUTH0_AUDIENCE" $AUTH0_CLIENT_ID "$AUTH0_CLIENT_SECRET" $TOKEN_CACHE_TIME $PORT $PORT $AWS_ECS_CLUSTER $AWS_REGION $NODE_ENV)
+  task_def=$(printf "$task_template" $family $ACCOUNT_ID $AWS_ECS_CONTAINER_NAME $ACCOUNT_ID $AWS_REGION $AWS_REPOSITORY $CIRCLE_SHA1 $NODE_ENV $ENABLE_FILE_UPLOAD $LOG_LEVEL $CAPTURE_LOGS $LOGENTRIES_TOKEN $API_VERSION $AWS_REGION $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY $AUTH_DOMAIN $AUTH_SECRET $VALID_ISSUERS $DB_MASTER_URL $MEMBER_SERVICE_ENDPOINT $IDENTITY_SERVICE_ENDPOINT $BUS_API_URL $MESSAGE_SERVICE_URL $SYSTEM_USER_CLIENT_ID $SYSTEM_USER_CLIENT_SECRET $PROJECTS_ES_URL $PROJECTS_ES_INDEX_NAME $RABBITMQ_URL $DIRECT_PROJECT_SERVICE_ENDPOINT $FILE_SERVICE_ENDPOINT $CONNECT_PROJECTS_URL $SEGMENT_ANALYTICS_KEY "$AUTH0_URL" "$AUTH0_AUDIENCE" $AUTH0_CLIENT_ID "$AUTH0_CLIENT_SECRET" $TOKEN_CACHE_TIME "$KAFKA_CLIENT_CERT" "$KAFKA_CLIENT_CERT_KEY" $KAFKA_GROUP_ID $KAFKA_URL $PORT $PORT $AWS_ECS_CLUSTER $AWS_REGION $NODE_ENV)
 }
 
 push_ecr_image(){

local/mock-services/services.json

@@ -249,6 +249,56 @@
                 }
             },
             "version": "v3"
+        },
+        {
+            "id": "test_admin1",
+            "result": {
+                "success": true,
+                "status": 200,
+                "metadata": null,
+                "content": {
+                    "maxRating": {
+                        "rating": 1114,
+                        "track": "DATA_SCIENCE",
+                        "subTrack": "SRM"
+                    },
+                    "createdBy": "40011578",
+                    "updatedBy": "40011578",
+                    "userId": 40135978,
+                    "firstName": "Adminname",
+                    "lastName": "Adminlastname",
+                    "quote": "It is a mistake to think you can solve any major problems just with potatoes.",
+                    "description": null,
+                    "otherLangName": null,
+                    "handle": "test_admin1",
+                    "handleLower": "test_admin1",
+                    "status": "ACTIVE",
+                    "email": "pshah1@test.com",
+                    "addresses": [
+                        {
+                            "streetAddr1": "100 Main Street",
+                            "streetAddr2": "",
+                            "city": "Chicago",
+                            "zip": "60601",
+                            "stateCode": "IL",
+                            "type": "HOME",
+                            "updatedAt": null,
+                            "createdAt": null,
+                            "createdBy": null,
+                            "updatedBy": null
+                        }
+                    ],
+                    "homeCountryCode": "USA",
+                    "competitionCountryCode": "USA",
+                    "photoURL": null,
+                    "tracks": [
+                        "DEVELOP"
+                    ],
+                    "updatedAt": "2015-12-02T14:00Z",
+                    "createdAt": "2014-04-10T10:55Z"
+                }
+            },
+            "version": "v3"
         }
     ]
 }

migrations/20180608_project_add_templateId_and_new_tables.sql

@@ -45,6 +45,7 @@ CREATE TABLE milestones (
     "activeText" character varying(512) NOT NULL,
     "completedText" character varying(512) NOT NULL,
     "blockedText" character varying(512) NOT NULL,
+    "hidden" boolean DEFAULT false,
     "deletedAt" timestamp with time zone,
     "createdAt" timestamp with time zone,
     "updatedAt" timestamp with time zone,

migrations/20180824_milestone_templates_metadata.sql

@@ -0,0 +1,27 @@
+--
+-- UPDATE EXISTING TABLES:
+--   product_milestone_templates:
+--     removed constraint `product_milestone_templates_productTemplateId_fkey`
+--     changed column `productTemplateId` to `referenceId`
+--     added column `reference`
+--     added column `metadata`
+--     changed table name to `milestone_templates`
+
+--
+-- product_milestone_templates
+--
+ALTER TABLE product_milestone_templates DROP CONSTRAINT "product_milestone_templates_productTemplateId_fkey";
+
+ALTER TABLE product_milestone_templates RENAME COLUMN "productTemplateId" TO "referenceId";
+
+ALTER TABLE product_milestone_templates ADD COLUMN "reference" character varying(45);
+UPDATE product_milestone_templates set reference='productTemplate' where reference is null;
+ALTER TABLE product_milestone_templates ALTER COLUMN "reference" SET NOT NULL;
+
+ALTER TABLE product_milestone_templates ADD COLUMN "metadata" json;
+UPDATE product_milestone_templates set metadata='{}' where metadata is null;
+ALTER TABLE product_milestone_templates ALTER COLUMN "metadata" SET NOT NULL;
+
+ALTER TABLE product_milestone_templates RENAME TO milestone_templates;
+
+

migrations/20180827_project_phases_order.sql

@@ -0,0 +1,9 @@
+--
+-- UPDATE EXISTING TABLES:
+--   project_phases:
+--     added column `order`
+
+--
+-- project_phases
+--
+ALTER TABLE project_phases ADD COLUMN "order" integer NULL;

migrations/20180910_project_activity.sql

@@ -0,0 +1,18 @@
+--
+-- UPDATE EXISTING TABLES:
+--   projects:
+--     added column `lastActivityAt`
+--     added column `lastActivityUserId`
+
+--
+-- projects
+
+-- Add new columns
+ALTER TABLE projects ADD COLUMN "lastActivityAt" timestamp with time zone;
+ALTER TABLE projects ADD COLUMN "lastActivityUserId" character varying(45);
+-- Update new colums
+UPDATE projects SET "lastActivityAt"="updatedAt" WHERE "lastActivityAt" is NULL;
+UPDATE projects SET "lastActivityUserId"=cast("updatedBy" as varchar) WHERE "lastActivityUserId" is NULL;
+-- Set not null
+ALTER TABLE projects ALTER COLUMN "lastActivityAt" SET NOT NULL;
+ALTER TABLE projects ALTER COLUMN "lastActivityUserId" SET NOT NULL;

migrations/elasticsearch_sync.js

@@ -281,6 +281,13 @@ function getRequestBody(indexName) {
       updatedBy: {
         type: 'integer',
       },
+      lastActivityAt: {
+        type: 'date',
+        format: 'strict_date_optional_time||epoch_millis',
+      },
+      lastActivityUserId: {
+        type: 'string',
+      },
       utm: {
         properties: {
           campaign: {