Merge pull request #124 from topcoder-platform/feature/improve-local-setup

feat: improve local setup

Author: maxceem

.nvmrc

@@ -0,0 +1 @@
+12.18.0

README.md

@@ -1,63 +1,101 @@
 require('dotenv').config()
 module.exports = {
+  // the log level, default is 'debug'
   LOG_LEVEL: process.env.LOG_LEVEL || 'debug',
+  // the server port, default is 3000
   PORT: process.env.PORT || 3000,
+  // the server api base path
   BASE_PATH: process.env.BASE_PATH || '/api/v5',
 
+  // The authorization secret used during token verification.
   AUTH_SECRET: process.env.AUTH_SECRET || 'mysecret',
+  // The valid issuer of tokens, a json array contains valid issuer.
   VALID_ISSUERS: process.env.VALID_ISSUERS || '["https://api.topcoder-dev.com", "https://api.topcoder.com", "https://topcoder-dev.auth0.com/", "https://auth.topcoder-dev.com/"]',
+  // Auth0 URL, used to get TC M2M token
   AUTH0_URL: process.env.AUTH0_URL,
+  // Auth0 audience, used to get TC M2M token
   AUTH0_AUDIENCE: process.env.AUTH0_AUDIENCE,
+  // Auth0 audience for U-Bahn
   AUTH0_AUDIENCE_UBAHN: process.env.AUTH0_AUDIENCE_UBAHN,
+  // Auth0 token cache time, used to get TC M2M token
   TOKEN_CACHE_TIME: process.env.TOKEN_CACHE_TIME,
+  // Auth0 client id, used to get TC M2M token
   AUTH0_CLIENT_ID: process.env.AUTH0_CLIENT_ID,
+  // Auth0 client secret, used to get TC M2M token
   AUTH0_CLIENT_SECRET: process.env.AUTH0_CLIENT_SECRET,
+  // Proxy Auth0 URL, used to get TC M2M token
   AUTH0_PROXY_SERVER_URL: process.env.AUTH0_PROXY_SERVER_URL,
 
   m2m: {
     M2M_AUDIT_USER_ID: process.env.M2M_AUDIT_USER_ID || '00000000-0000-0000-0000-000000000000',
     M2M_AUDIT_HANDLE: process.env.M2M_AUDIT_HANDLE || 'TopcoderService'
   },
 
+  // the Topcoder v5 url
   TC_API: process.env.TC_API || 'https://api.topcoder-dev.com/v5',
+  // the organization id
   ORG_ID: process.env.ORG_ID || '36ed815b-3da1-49f1-a043-aaed0a4e81ad',
+  // the referenced skill provider id
   TOPCODER_SKILL_PROVIDER_ID: process.env.TOPCODER_SKILL_PROVIDER_ID || '9cc0795a-6e12-4c84-9744-15858dba1861',
 
   TOPCODER_USERS_API: process.env.TOPCODER_USERS_API || 'https://api.topcoder-dev.com/v3/users',
 
+  // PostgreSQL database url.
   DATABASE_URL: process.env.DATABASE_URL || 'postgres://postgres:postgres@localhost:5432/postgres',
+  // string - PostgreSQL database target schema
   DB_SCHEMA_NAME: process.env.DB_SCHEMA_NAME || 'bookings',
+  // the project service url
   PROJECT_API_URL: process.env.PROJECT_API_URL || 'https://api.topcoder-dev.com',
 
   esConfig: {
+    // the elasticsearch host
     HOST: process.env.ES_HOST || 'http://localhost:9200',
 
     ELASTICCLOUD: {
+      // The elastic cloud id, if your elasticsearch instance is hosted on elastic cloud. DO NOT provide a value for ES_HOST if you are using this
       id: process.env.ELASTICCLOUD_ID,
+      // The elastic cloud username for basic authentication. Provide this only if your elasticsearch instance is hosted on elastic cloud
       username: process.env.ELASTICCLOUD_USERNAME,
+      // The elastic cloud password for basic authentication. Provide this only if your elasticsearch instance is hosted on elastic cloud
       password: process.env.ELASTICCLOUD_PASSWORD
     },
 
+    // The Amazon region to use when using AWS Elasticsearch service
     AWS_REGION: process.env.AWS_REGION || 'us-east-1', // AWS Region to be used if we use AWS ES
 
+    // the job index
     ES_INDEX_JOB: process.env.ES_INDEX_JOB || 'job',
+    // the job candidate index
     ES_INDEX_JOB_CANDIDATE: process.env.ES_INDEX_JOB_CANDIDATE || 'job_candidate',
+    // the resource booking index
     ES_INDEX_RESOURCE_BOOKING: process.env.ES_INDEX_RESOURCE_BOOKING || 'resource_booking'
   },
 
+  // Topcoder Bus API URL
   BUSAPI_URL: process.env.BUSAPI_URL || 'https://api.topcoder-dev.com/v5',
+  // The error topic at which bus api will publish any errors
   KAFKA_ERROR_TOPIC: process.env.KAFKA_ERROR_TOPIC || 'common.error.reporting',
+  // The originator value for the kafka messages
   KAFKA_MESSAGE_ORIGINATOR: process.env.KAFKA_MESSAGE_ORIGINATOR || 'taas-api',
   // topics for job service
+  // the create job entity Kafka message topic
   TAAS_JOB_CREATE_TOPIC: process.env.TAAS_JOB_CREATE_TOPIC || 'taas.job.create',
+  // the update job entity Kafka message topic
   TAAS_JOB_UPDATE_TOPIC: process.env.TAAS_JOB_UPDATE_TOPIC || 'taas.job.update',
+  // the delete job entity Kafka message topic
   TAAS_JOB_DELETE_TOPIC: process.env.TAAS_JOB_DELETE_TOPIC || 'taas.job.delete',
   // topics for jobcandidate service
+  // the create job candidate entity Kafka message topic
   TAAS_JOB_CANDIDATE_CREATE_TOPIC: process.env.TAAS_JOB_CANDIDATE_CREATE_TOPIC || 'taas.jobcandidate.create',
+  // the update job candidate entity Kafka message topic
   TAAS_JOB_CANDIDATE_UPDATE_TOPIC: process.env.TAAS_JOB_CANDIDATE_UPDATE_TOPIC || 'taas.jobcandidate.update',
+  // the delete job candidate entity Kafka message topic
   TAAS_JOB_CANDIDATE_DELETE_TOPIC: process.env.TAAS_JOB_CANDIDATE_DELETE_TOPIC || 'taas.jobcandidate.delete',
   // topics for job service
+  // the create resource booking entity Kafka message topic
   TAAS_RESOURCE_BOOKING_CREATE_TOPIC: process.env.TAAS_RESOURCE_BOOKING_CREATE_TOPIC || 'taas.resourcebooking.create',
+  // the update resource booking entity Kafka message topic
   TAAS_RESOURCE_BOOKING_UPDATE_TOPIC: process.env.TAAS_RESOURCE_BOOKING_UPDATE_TOPIC || 'taas.resourcebooking.update',
+  // the delete resource booking entity Kafka message topic
   TAAS_RESOURCE_BOOKING_DELETE_TOPIC: process.env.TAAS_RESOURCE_BOOKING_DELETE_TOPIC || 'taas.resourcebooking.delete'
 }

Verification.md

@@ -0,0 +1,88 @@
+version: "3"
+services:
+  postgres:
+    container_name: tc-taas-postgres
+    image: postgres
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+    ports:
+      - 5432:5432
+
+  zookeeper:
+    image: wurstmeister/zookeeper
+    container_name: tc-taas-zookeeper
+    ports:
+      - 2181:2181
+
+  kafka:
+    image: wurstmeister/kafka
+    container_name: tc-taas-kafka
+    depends_on:
+      - zookeeper
+    ports:
+      - 9092:9092
+    environment:
+      KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka:9093,OUTSIDE://localhost:9092
+      KAFKA_LISTENERS: INSIDE://kafka:9093,OUTSIDE://0.0.0.0:9092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+
+  kafka-client:
+    container_name: tc-kafka-client
+    build: ./kafka-client
+    depends_on:
+      - kafka
+      - zookeeper
+
+  elasticsearch:
+    image: elasticsearch:7.7.1
+    container_name: tc-taas-elasticsearch
+    environment:
+      - discovery.type=single-node
+    ports:
+      - 9200:9200
+
+  taas-es-processor:
+    container_name: tc-taas-es-procesor
+    build:
+      context: ./generic-tc-service
+      args:
+        NODE_VERSION: 12.16.3
+        GIT_URL: https://github.com/topcoder-platform/taas-es-processor
+        GIT_BRANCH: dev
+    command: start kafka-client
+    ports:
+      - 5000:5000
+    depends_on:
+      - kafka-client
+      - elasticsearch
+    environment:
+      - KAFKA_URL=kafka:9093
+      - ES_HOST=http://elasticsearch:9200
+
+  tc-bus-api:
+    container_name: tc-bus-api
+    build:
+      context: ./generic-tc-service
+      args:
+        NODE_VERSION: 8.11.3
+        GIT_URL: https://github.com/topcoder-platform/tc-bus-api
+        GIT_BRANCH: dev
+        BYPASS_TOKEN_VALIDATION: 1
+    command: start kafka-client
+    ports:
+      - 8002:8002
+    depends_on:
+      - kafka-client
+    environment:
+      - PORT=8002
+      - KAFKA_URL=kafka:9093
+      - JWT_TOKEN_SECRET=secret
+      - VALID_ISSUERS="[\"https:\/\/topcoder-newauth.auth0.com\/\",\"https:\/\/api.topcoder-dev.com\",\"https:\/\/topcoder-dev.auth0.com\/\"]"
+      - AUTH0_URL
+      - AUTH0_AUDIENCE
+      - AUTH0_CLIENT_ID
+      - AUTH0_CLIENT_SECRET
+      - AUTH0_PROXY_SERVER_URL

config/default.js

@@ -0,0 +1,15 @@
+ARG NODE_VERSION=12.16.3
+
+FROM node:$NODE_VERSION
+ARG GIT_URL
+ARG GIT_BRANCH
+ARG BYPASS_TOKEN_VALIDATION
+
+RUN git clone $GIT_URL /opt/app
+WORKDIR /opt/app
+RUN git checkout -b node-branch origin/$GIT_BRANCH
+
+RUN npm install
+RUN if [ $BYPASS_TOKEN_VALIDATION -eq 1 ]; then sed -i '/decodedToken = jwt.decode/a \      callback(undefined, decodedToken.payload); return;' node_modules/tc-core-library-js/lib/auth/verifier.js; fi
+COPY docker-entrypoint.sh /opt/
+ENTRYPOINT ["/opt/docker-entrypoint.sh"]

local/docker-compose.yml

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+HOST_DOMAIN="host.docker.internal"
+ping -q -c1 $HOST_DOMAIN > /dev/null 2>&1
+if [ $? -ne 0 ]; then
+  HOST_IP=$(ip route | awk 'NR==1 {print $3}')
+  echo -e "$HOST_IP\t$HOST_DOMAIN" >> /etc/hosts
+fi
+
+if [ $# -eq 2 ]; then
+  echo "Waiting for $2 to exit...."
+  while ping -c1 $2 &>/dev/null
+    do
+      sleep 1
+    done
+  echo "$2 exited!"
+fi
+
+cd /opt/app/ && npm run $1

local/generic-tc-service/Dockerfile

@@ -0,0 +1,5 @@
+From wurstmeister/kafka
+WORKDIR /app/
+COPY topics.txt .
+COPY create-topics.sh .
+ENTRYPOINT ["/app/create-topics.sh"]

local/generic-tc-service/docker-entrypoint.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+/opt/kafka/bin/kafka-topics.sh --list --zookeeper zookeeper:2181 > exists-topics.txt
+while read topic; do
+  /opt/kafka/bin/kafka-topics.sh --create --if-not-exists --zookeeper zookeeper:2181 --partitions 1 --replication-factor 1 --topic $topic
+done < <(sort topics.txt exists-topics.txt exists-topics.txt | uniq -u)

local/kafka-client/Dockerfile

@@ -0,0 +1,9 @@
+taas.job.create
+taas.jobcandidate.create
+taas.resourcebooking.create
+taas.job.update
+taas.jobcandidate.update
+taas.resourcebooking.update
+taas.job.delete
+taas.jobcandidate.delete
+taas.resourcebooking.delete

local/kafka-client/create-topics.sh

@@ -15,6 +15,10 @@
     "migrate:undo": "npx sequelize db:migrate:undo",
     "test-data": "node scripts/insert-es-data.js",
     "test": "mocha test/unit/*.test.js --timeout 30000 --exit",
+    "services:up": "docker-compose -f ./local/docker-compose.yml up -d",
+    "services:down": "docker-compose -f ./local/docker-compose.yml down",
+    "services:logs": "docker-compose -f ./local/docker-compose.yml logs",
+    "local:init": "npm run create-index && npm run init-db",
     "cov": "nyc --reporter=html --reporter=text mocha test/unit/*.test.js --timeout 30000 --exit"
   },
   "keywords": [],