Syncing V5 changes into new develop branch

.circleci/config.yml

@@ -2,19 +2,19 @@ version: 2
 python_env: &python_env
   docker:
     - image: circleci/python:2.7-stretch-browsers
-    
+
 install_awscli: &install_awscli
   name: "Install awscli"
   command: |
     sudo pip install awscli --upgrade
 install_deploysuite: &install_deploysuite
-          name: Installation of install_deploysuite.
-          command: |
-            git clone --branch master https://github.com/topcoder-platform/tc-deploy-scripts ../buildscript
-            cp ./../buildscript/master_deploy.sh .
-            cp ./../buildscript/buildenv.sh .
-            cp ./../buildscript/awsconfiguration.sh .
-            
+  name: Installation of install_deploysuite.
+  command: |
+    git clone --branch v1.4.1 https://github.com/topcoder-platform/tc-deploy-scripts ../buildscript
+    cp ./../buildscript/master_deploy.sh .
+    cp ./../buildscript/buildenv.sh .
+    cp ./../buildscript/awsconfiguration.sh .
+
 # Instructions of deployment
 deploy_steps: &deploy_steps
   - checkout
@@ -23,22 +23,16 @@ deploy_steps: &deploy_steps
   - run: *install_awscli
   - run: *install_deploysuite
   - setup_remote_docker
-  - run: docker build -t tc-project-service:latest .
+  - run: docker build -t ${APPNAME}:latest .
   - deploy:
-      name: "Running Masterscript -  deploy tc-project-service " 
+      name: "Running Masterscript -  deploy tc-project-service "
       command: |
-        ./awsconfiguration.sh $DEPLOY_ENV  
+        ./awsconfiguration.sh $DEPLOY_ENV
         source awsenvconf
-        ./buildenv.sh -e $DEPLOY_ENV -b ${VAR_ENV}-tc-project-service-deployvar
+        ./buildenv.sh -e $DEPLOY_ENV -b ${LOGICAL_ENV}-${APPNAME}-deployvar
         source buildenvvar
-        ./master_deploy.sh -d ECS -e $DEPLOY_ENV -t latest -s ${VAR_ENV}-global-appvar,${VAR_ENV}-tc-project-service-appvar -i tc-project-service -p FARGATE 
+        ./master_deploy.sh -d ECS -e $DEPLOY_ENV -t latest -s ${LOGICAL_ENV}-global-appvar,${LOGICAL_ENV}-${APPNAME}-appvar -i ${APPNAME}
 
-        echo "======= Running Masterscript - deploy tc-project-service-consumers ==========="
-        if [ -e ${VAR_ENV}-tc-project-service-appvar.json ]; then  sudo rm -vf ${VAR_ENV}-tc-project-service-appvar.json; fi
-        ./buildenv.sh -e $DEPLOY_ENV -b ${VAR_ENV}-tc-project-service-consumers-deployvar
-        source buildenvvar
-        ./master_deploy.sh -d ECS -e $DEPLOY_ENV -t latest -s ${VAR_ENV}-global-appvar,${VAR_ENV}-tc-project-service-appvar -i tc-project-service -p FARGATE
-
 jobs:
   test:
     docker:
@@ -50,13 +44,20 @@ jobs:
       - image: elasticsearch:2.3
       - image: rabbitmq:3-management
     environment:
-      DB_MASTER_URL: postgres://circle_test:@127.0.0.1:5432/circle_test
-      AUTH_SECRET: secret
-      AUTH_DOMAIN: topcoder-dev.com
-      LOG_LEVEL: debug
-      APP_VERSION: v4
+      DEPLOY_ENV: "DEV"
+      LOGICAL_ENV: "dev"
+      APPNAME: "projects-api"
     steps:
       - checkout
+      - run:
+          name: "Install dependeency"
+          command: |
+              sudo apt update
+              sudo apt install curl
+              sudo apt install python-pip
+      - run: *install_awscli
+      - run: *install_deploysuite
+      - setup_remote_docker
       - restore_cache:
           key: test-node-modules-{{ checksum "package.json" }}
       - run: npm install
@@ -65,39 +66,50 @@ jobs:
           paths:
             - node_modules
       - run: npm run lint
-      - run: npm run test
+      - run:
+          name: "Running Masterscript -  deploy tc-project-service "
+          command: |
+            ./awsconfiguration.sh $DEPLOY_ENV
+            source awsenvconf
+            ./buildenv.sh -e $DEPLOY_ENV -b ${LOGICAL_ENV}-${APPNAME}-testvar
+            source buildenvvar
+            npm run test
+            rm -f buildenvvar
       - run: npm run build
       - persist_to_workspace:
           root: .
           paths:
             - dist
-            
+
   deployProd:
     <<: *python_env
     environment:
       DEPLOY_ENV: "PROD"
-      VAR_ENV: "prod"
+      LOGICAL_ENV: "prod"
+      APPNAME: "projects-api"
     steps: *deploy_steps
 
   deployDev:
     <<: *python_env
     environment:
       DEPLOY_ENV: "DEV"
-      VAR_ENV: "dev"
-    steps: *deploy_steps 
-
+      LOGICAL_ENV: "dev"
+      APPNAME: "projects-api"
+    steps: *deploy_steps
+
 workflows:
   version: 2
   build:
     jobs:
-      - test
+      - test:
+          context : org-global
       - deployDev:
           context : org-global
           requires:
             - test
           filters:
             branches:
-              only: ['dev', 'dev-sts', 'feature/looker-api-integration']
+              only: ['develop']
       - deployProd:
           context : org-global
           requires:

.nvmrc

@@ -1 +1 @@
-v8.2.1
+v8.9.4

README.md

@@ -9,7 +9,7 @@ Microservice to manage CRUD operations for all things Projects.
 ### Requirements
 
 * [docker-compose](https://docs.docker.com/compose/install/) - We use docker-compose for running dependencies locally.
-* Nodejs 8.2.1 - consider using [nvm](https://github.com/creationix/nvm) or equivalent to manage your node version
+* Nodejs 8.9.4 - consider using [nvm](https://github.com/creationix/nvm) or equivalent to manage your node version
 * Install [libpg](https://www.npmjs.com/package/pg-native)
 
 ### Steps to run locally
@@ -54,6 +54,7 @@ Microservice to manage CRUD operations for all things Projects.
   ```
   Alternatively, you may update `config/local.js` and replace `dockerhost` with your docker IP address.<br>
   You may try using command `docker-machine ip` to get your docker IP, but it works not for all systems.
+  Also, be sure to update `busApiUrl` if you are running `tc-bus-api` locally. (See below)
 
   Explanation of configs:
   - `config/mock.local.js` - Use local `mock-services` from docker to mock Identity and Member services instead of using deployed at Topcoder dev environment.
@@ -85,12 +86,66 @@ Microservice to manage CRUD operations for all things Projects.
   The project service will be served on `http://localhost:8001`.
 
 ### Import sample metadata & projects
+
 ```bash
 CONNECT_USER_TOKEN=<connect user token> npm run demo-data
 ```
-This command will create sample metadata entries in the DB (duplicate what is currently in development environment).
+To retrieve data from DEV env we have to provide a valid user token (`CONNECT_USER_TOKEN`). You may login to http://connect.topcoder-dev.com and find the Bearer token in the request headers using browser dev tools.
+
+This command for importing data uses API to create demo data. Which has a few pecularities:
+- data in DB would be for sure created
+- data in ElasticSearch Index (ES) would be only created if services [project-processor-es](https://github.com/topcoder-platform/project-processor-es) and [tc-bus-api](https://github.com/topcoder-platform/tc-bus-api) are also started locally. If you don't start them, then imported data wouldn't be indexed in ES, and would be only added to DB. You may start them locally separately, or better use `local/full/docker-compose.yml` as described [next section](#local-deployment-with-other-topcoder-services) which would start them automatically.
+   - **NOTE** During data importing a lot of records has to be indexed in ES, so you have to wait about 5-10 minutes after `npm run demo-data` is finished until imported data is indexed in ES. You may watch logs of `project-processor-es` to see if its done or no.
+
+### Local Deployment with other Topcoder Services.
+
+* There exists an alternate `docker-compose.yml` file that can be used to spawn containers for the following services:
+
+  |  Service | Name | Port  |
+  |----------|:-----:|:----:|
+  | PostGreSQL DB | db | 5432  |
+  | ElasticSearch | esearch | 9200,9300 |
+  | RabbitMQ | queue | 5672, 15672  |
+  | Zookeeper | zookeeper | 2181  |
+  | Kafka | kafka | 9092  |
+  | [tc-bus-api](https://github.com/topcoder-platform/tc-bus-api) | tc-bus-api | 8002  |
+  | [project-processor-es](https://github.com/topcoder-platform/project-processor-es) | project-processor-es | 5000  |
+  | [tc-notifications-api](https://github.com/topcoder-platform/tc-notifications) | tc-notifications-api | 4000  |
+  | [tc-notifications-processor](https://github.com/topcoder-platform/tc-notifications) | tc-notifications-processor | 4001  |
+
+* To have kafka create a list of desired topics on startup, there exists a file with the path `local/full/kafka-client/topics.txt`. Each line from the file will be added as a topic.
+* To run these services simply run the following commands:
+
+  ```bash
+  export AUTH0_CLIENT_ID=<insert required value here>
+  export AUTH0_CLIENT_SECRET=<insert required value here>
+  export AUTH0_URL=<insert required value here>
+  export AUTH0_AUDIENCE=<insert required value here>
+  export AUTH0_PROXY_SERVER_URL=<insert required value here>
+
+  cd local/full
+  docker-compose up -d
+  ```
 
-To retrieve data from DEV env we need to provide a valid user token. You may login to http://connect.topcoder-dev.com and find the Bearer token in the request headers using browser dev tools.
+* The environment variables specified in the commands above will be passed onto the containers that have been configured to read them.
+* The above command will start all containers in the background.
+* To view the logs of any of the services use the following command, replacing "SERVICE_NAME" with the corresponding value under the "Name" column in the above table:
+
+  ```bash
+  cd local/full
+  docker-compose logs -f SERVICE_NAME
+  ```
+
+* The containers have been configured such that all Topcoder services will wait until all the topics listed in `local/full/kafka-client/topics.txt` have been created. To monitor the progress of topic creation, you can view the logs of the `kafka-client` service, which will exit when all topics have been created.
+
+* **WARNING**<br>
+  After all the containers are started, make sure that `project-processor-es` service started successfully, as sometimes it doesn't start successfully as Kafka wasn't yet properly started at that moment. So run `docker-compose logs -f project-processor-es` to see its logs, you should see 3 lines with text `Subscribed to project.action.` like:
+  ```
+  project-processor-es_1        | 2019-12-18T11:10:12.849Z DEBUG no-kafka-client Subscribed to project.action.update:0 offset 0 leader 96e65c46c746:9092
+  project-processor-es_1        | 2019-12-18T11:10:12.851Z DEBUG no-kafka-client Subscribed to project.action.delete:0 offset 0 leader 96e65c46c746:9092
+  project-processor-es_1        | 2019-12-18T11:10:12.852Z DEBUG no-kafka-client Subscribed to project.action.create:0 offset 0 leader 96e65c46c746:9092
+  ```
+  If you don't see such lines, restart `project-processor-es` service ONLY by running `docker-compose restart project-processor-es`.
 
 ### Run Connect App with Project Service locally
 
@@ -99,7 +154,9 @@ To be able to run [Connect App](https://github.com/appirio-tech/connect-app) wit
 
    ```js
    PROJECTS_API_URL: 'http://localhost:8001'
+   TC_NOTIFICATION_URL: 'http://localhost:4000/v5/notifications' # if tc-notfication-api has been locally deployed
    ```
+
 2. Bypass token validation in Project Service.
 
    In `tc-project-service/node_modules/tc-core-library-js/lib/auth/verifier.js` add this to line 23:
@@ -142,3 +199,33 @@ You can paste **swagger.yaml** to  [swagger editor](http://editor.swagger.io/) o
 
 #### Deploying without docker
 If you don't want to use docker to deploy to localhost. You can simply run `npm run start:dev` from root of project. This should start the server on default port `8001`.
+
+### Kafka Commands
+
+If you've used `docker-compose` with the file `local/full/docker-compose.yml` to spawn kafka & zookeeper, you can use the following commands to manipulate kafka topics and messages:
+(Replace TOPIC_NAME with the name of the desired topic)
+
+**Create Topic**
+
+```bash
+docker exec tc-projects-kafka /usr/bin/kafka-topics --create --zookeeper zookeeper:2181 --partitions 1 --replication-factor 1 --topic TOPIC_NAME
+```
+
+**List Topics**
+
+```bash
+docker exec -it tc-projects-kafka /usr/bin/kafka-topics --list --zookeeper zookeeper:2181
+```
+
+**Watch Topic**
+
+```bash
+docker exec -it tc-projects-kafka /usr/bin/kafka-console-consumer --bootstrap-server localhost:9092 --zookeeper zookeeper:2181 --topic TOPIC_NAME
+```
+
+**Post Message to Topic**
+
+```bash
+docker exec -it tc-projects-kafka /usr/bin/kafka-console-producer --topic TOPIC_NAME --broker-list localhost:9092
+```
+The message can be passed using `stdin`

config/custom-environment-variables.json

@@ -10,9 +10,11 @@
     "host": "PROJECTS_ES_URL",
     "apiVersion": "2.3",
     "indexName": "PROJECTS_ES_INDEX_NAME",
-    "docType": "projectV4",
+    "docType": "PROJECTS_ES_DOC_TYPE",
     "timelineIndexName": "TIMELINES_ES_INDEX_NAME",
-    "timelineDocType": "TIMELINES_ES_DOC_TYPE"
+    "timelineDocType": "TIMELINES_ES_DOC_TYPE",
+    "metadataIndexName": "METADATA_ES_INDEX_NAME",
+    "metadataDocType": "METADATA_ES_DOC_TYPE"
   },
   "rabbitmqURL": "RABBITMQ_URL",
   "pubsubQueueName": "PUBSUB_QUEUE_NAME",
@@ -50,6 +52,7 @@
   "accountsAppUrl": "ACCOUNTS_APP_URL",
   "inviteEmailSubject": "INVITE_EMAIL_SUBJECT",
   "inviteEmailSectionTitle": "INVITE_EMAIL_SECTION_TITLE",
+  "pageSize": "PAGE_SIZE",
   "SSO_REFCODES": "SSO_REFCODES",
   "lookerConfig": {
     "BASE_URL": "LOOKER_API_BASE_URL",

config/default.json

@@ -1,8 +1,8 @@
 {
-  "apiVersion": "v4",
+  "apiVersion": "v5",
   "AUTH_SECRET": "secret",
   "logLevel": "info",
-  "version": "v4",
+  "version": "v5",
   "captureLogs": "false",
   "enableFileUpload": "true",
   "logentriesToken": "",
@@ -19,11 +19,13 @@
   "projectAttachmentPathSuffix": "attachments",
   "elasticsearchConfig": {
     "host": "",
-    "apiVersion": "2.3",
+    "apiVersion": "6.8",
     "indexName": "projects",
-    "docType": "projectV4",
+    "docType": "doc",
     "timelineIndexName": "timelines",
-    "timelineDocType": "timelineV4"
+    "timelineDocType": "doc",
+    "metadataIndexName": "metadata",
+    "metadataDocType": "doc"
   },
   "connectProjectUrl":"",
   "dbConfig": {
@@ -44,13 +46,8 @@
   "busApiToken": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoicHJvamVjdC1zZXJ2aWNlIiwiaWF0IjoxNTEyNzQ3MDgyLCJleHAiOjE1MjEzODcwODJ9.PHuNcFDaotGAL8RhQXQMdpL8yOKXxjB5DbBIodmt7RE",
   "HEALTH_CHECK_URL": "_health",
   "maxPhaseProductCount": 1,
-  "AUTH0_CLIENT_ID": "",
-  "AUTH0_CLIENT_SECRET": "",
-  "AUTH0_AUDIENCE": "",
-  "AUTH0_URL": "",
-  "TOKEN_CACHE_TIME": "",
+  "TOKEN_CACHE_TIME": "86000",
   "whitelistedOriginsForUserIdAuth": "[\"https:\/\/topcoder-newauth.auth0.com\/\",\"https:\/\/api.topcoder-dev.com\"]",
-  "AUTH0_PROXY_SERVER_URL" : "",
   "EMAIL_INVITE_FROM_NAME":"Topcoder",
   "EMAIL_INVITE_FROM_EMAIL":"noreply@connect.topcoder.com",
   "inviteEmailSubject": "You are invited to Topcoder",
@@ -59,8 +56,9 @@
   "accountsAppUrl": "https://accounts.topcoder-dev.com",
   "MAX_REVISION_NUMBER": 100,
   "UNIQUE_GMAIL_VALIDATION": false,
-  "SSO_REFCODES": "[]",
+  "pageSize": 20,
   "VALID_STATUSES_BEFORE_PAUSED": "[\"active\"]",
+  "SSO_REFCODES": "[]",
   "lookerConfig": {
     "BASE_URL": "",
     "CLIENT_ID": "",

config/m2m.local.js

@@ -5,6 +5,7 @@ if (process.env.NODE_ENV === 'test') {
   config = require('./test.json');
 } else {
   config = {
+    busApiUrl: "http://localhost:8002/v5",
     identityServiceEndpoint: "https://api.topcoder-dev.com/v3/",
     authSecret: 'secret',
     authDomain: 'topcoder-dev.com',
@@ -23,11 +24,7 @@ if (process.env.NODE_ENV === 'test') {
       idleTimeout: 1000,
     },
     elasticsearchConfig: {
-      host: 'dockerhost:9200',
-      // target elasticsearch 2.3 version
-      apiVersion: '2.3',
-      indexName: 'projects',
-      docType: 'projectV4'
+      host: 'dockerhost:9200'
     },
     whitelistedOriginsForUserIdAuth: "[\"\"]",
   };

config/mock.local.js

@@ -5,6 +5,7 @@ if (process.env.NODE_ENV === 'test') {
   config = require('./test.json');
 } else {
   config = {
+    busApiUrl: "http://localhost:8002/v5",
     identityServiceEndpoint: "http://dockerhost:3001/",
     authSecret: 'secret',
     authDomain: 'topcoder-dev.com',
@@ -23,11 +24,7 @@ if (process.env.NODE_ENV === 'test') {
       idleTimeout: 1000,
     },
     elasticsearchConfig: {
-      host: 'dockerhost:9200',
-      // target elasticsearch 2.3 version
-      apiVersion: '2.3',
-      indexName: 'projects',
-      docType: 'projectV4'
+      host: 'dockerhost:9200'
     },
     whitelistedOriginsForUserIdAuth: "[\"\"]",
   };