Merge pull request #480 from topcoder-platform/feature/improve-local-setup

Improve local setup

Author: maxceem

README.md

@@ -17,7 +17,7 @@
     "rabbitmqUrl": "amqp://localhost:5672",
     "connectProjectsUrl": "https://local.topcoder-dev.com/projects/",
     "dbConfig": {
-        "masterUrl": "postgres://coder:mysecretpassword@localhost:5433/projectsdb_test",
+        "masterUrl": "postgres://coder:mysecretpassword@localhost:5432/projectsdb_test",
         "maxPoolSize": 50,
         "minPoolSize": 4,
         "idleTimeout": 1000

config/test.json

@@ -0,0 +1,76 @@
+# Topcoder Project Service Architecture
+
+- [Overview](#overview)
+- [Elasticsearch indexing](#elasticsearch-indexing)
+  - [Read data](#read-data)
+  - [Write data](#write-data)
+- [Kafka messages structure](#kafka-messages-structure)
+
+## Overview
+
+Topcoder Project Service is a microservice to manage CRUD operations for all things related to Projects. To communicate with other microservices like `tc-notifications`, `project-processor-es`, `legacy-project-processor` and event with itself **Project Service** produces Kafka messages and these service listen to the Kafka messages do some stuff. **Project Service** don't send Kafka messages directly, but uses a special service to send Kafka messages called `tc-bus-api`. So no matter what service we want to update, first we have to setup Kafka with Zookeeper and `tc-bus-api`.
+
+![diagram](./images/diagram.svg)
+
+*This diagram shows just some part of relations and services that are most important, it doesn't show all of them.*
+
+## Elasticsearch indexing
+
+It's important to keep in mind how the indexing and reading data from Elasticsearch works.
+
+### Read data
+
+As per global Topcoder API V5 standards, all endpoints in **Project Service** should get data from the Elasticsearch index first. If no data is found, endpoints should try to get data from Database.
+
+### Write data
+
+When some data is updated by **Project Service** it's directly changed in the Database. But **Project Service** doesn't change data in Elasticsearch directly. Instead of that, when some data is changed **Project Service** sends event to the Kafka (using `tc-bus-api`), and `project-processor-es` listens to the Kafka event and index updated data in Elasticsearch for **Project Service**.
+As a consequences, data in Elasticsearch is not updated immediately.
+
+## Kafka messages structure
+
+Project Service should send messages to 3 Kafka topics:
+- `project.action.create` - when some objects is created
+- `project.action.update` - when some objects is updated
+- `project.action.delete` - when some objects is deleted
+
+The `payload` of any of this messages should contain the next required properties:
+```js
+payload: {
+  // the name of the resource which has been create, updated or deleted,
+  // see constant `RESOURCES` in `src/constants.js` for possible values
+  "resource": "...",
+
+  // object which has been created, updated or deleted
+  "data": {...},
+
+  // should be present only in `project.action.update` topic, and contain the objects before update
+  "previousData": {...},
+}
+```
+
+Example:
+```js
+topic: "project.action.update",
+payload: {
+  "resource": "project.template",
+  "data": {
+    "id": 1234,
+    "name": "Name of project template UPDATED",
+    "key": "app",
+    "createdAt": 1,
+    "createdBy": 1,
+    "updatedAt": 2,
+    "updatedBy": 2,
+  },
+  "previousData": {
+    "id": 1234,
+    "name": "Name of project template",
+    "key": "app",
+    "createdAt": 1,
+    "createdBy": 1,
+    "updatedAt": 1,
+    "updatedBy": 1,
+  },
+}
+```

docs/guides/architercture/architecture.md

@@ -5,21 +5,13 @@ services:
     ports:
       - "3001:3001"
   db:
-    image: "postgres:9.5"
+    build: "postgres-db"
     ports:
       - "5432:5432"
     environment:
       - POSTGRES_PASSWORD=mysecretpassword
       - POSTGRES_USER=coder
-      - POSTGRES_DB=projectsdb
-  db_test:
-    image: "postgres:9.5"
-    ports:
-      - "5433:5432"
-    environment:
-      - POSTGRES_PASSWORD=mysecretpassword
-      - POSTGRES_USER=coder
-      - POSTGRES_DB=projectsdb_test
+      - POSTGRES_MULTIPLE_DATABASES=projectsdb,projectsdb_test
   esearch:
     image: "elasticsearch:2.3"
     ports:

docs/guides/architercture/images/diagram.svg

@@ -13,9 +13,9 @@ services:
       - TC_API_V5_BASE_URL=http://host.docker.internal:8001/v5
       - TC_API_V4_BASE_URL=https://api.topcoder-dev.com/v4
       - TC_API_V3_BASE_URL=https://api.topcoder-dev.com/v3
-      - KAFKA_URL=kafka:9092
+      - KAFKA_URL=kafka:9093
       - AUTH_SECRET=secret
-      - DATABASE_URL=postgresql://coder:mysecretpassword@notifications_db:5432/tc_notifications
+      - DATABASE_URL=postgresql://coder:mysecretpassword@db:5432/tc_notifications
       - JWKS_URI=test
       - LOG_LEVEL=debug
       - ENV=development

local/docker-compose.yml

@@ -8,18 +8,8 @@ services:
     extends:
       file: ../docker-compose.yml
       service: db
-  notifications_db:
-    image: "postgres:9.5"
-    expose:
-      - "5432"
     environment:
-      - POSTGRES_PASSWORD=mysecretpassword
-      - POSTGRES_USER=coder
-      - POSTGRES_DB=tc_notifications
-  db_test:
-    extends:
-      file: ../docker-compose.yml
-      service: db_test
+      - POSTGRES_MULTIPLE_DATABASES=projectsdb,projectsdb_test,tc_notifications
   esearch:
     extends:
       file: ../docker-compose.yml
@@ -29,18 +19,24 @@ services:
       file: ../docker-compose.yml
       service: queue
   zookeeper:
-    image: confluent/zookeeper
+    image: wurstmeister/zookeeper
     ports:
       - "2181:2181"
     environment:
       zk_id: "1"
   kafka:
-    image: confluent/kafka
+    image: wurstmeister/kafka
     container_name: tc-projects-kafka
     depends_on:
       - zookeeper
     ports:
       - "9092:9092"
+    environment:
+      KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka:9093,OUTSIDE://localhost:9092
+      KAFKA_LISTENERS: INSIDE://kafka:9093,OUTSIDE://0.0.0.0:9092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
   kafka-client:
     build: ./kafka-client
     depends_on:
@@ -63,7 +59,7 @@ services:
       - kafka-client
     environment:
       - PORT=3000
-      - KAFKA_URL=kafka:9092
+      - KAFKA_URL=kafka:9093
       - JWT_TOKEN_SECRET=secret
       - VALID_ISSUERS="[\"https://topcoder-newauth.auth0.com/\",\"https://api.topcoder-dev.com\"]"
   project-processor-es:
@@ -82,7 +78,7 @@ services:
       - kafka-client
     environment:
       - PORT=5000
-      - KAFKA_URL=kafka:9092
+      - KAFKA_URL=kafka:9093
       - ES_HOST=esearch:9200
   tc-notifications-reset-db:
     extends:
@@ -95,8 +91,7 @@ services:
     ports:
       - "4002:4002"
     depends_on:
-      - notifications_db
-      - esearch
+      - db
     environment:
       - PORT=4002
   tc-notifications-processor:

local/full/docker-compose.base.yml

@@ -1,4 +1,4 @@
-From confluent/kafka
+From wurstmeister/kafka
 WORKDIR /app/
 COPY topics.txt .
 COPY create-topics.sh .

local/full/docker-compose.yml

@@ -1,5 +1,5 @@
 #!/bin/bash
 
 while read topic; do
-  /usr/bin/kafka-topics --create --zookeeper zookeeper:2181 --partitions 1 --replication-factor 1 --topic $topic
+  /opt/kafka/bin/kafka-topics.sh --create --zookeeper zookeeper:2181 --partitions 1 --replication-factor 1 --topic $topic
 done < topics.txt

local/full/kafka-client/Dockerfile

@@ -0,0 +1,2 @@
+FROM postgres:9.5
+COPY create-multiple-postgresql-databases.sh /docker-entrypoint-initdb.d/

local/full/kafka-client/create-topics.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e
+set -u
+
+function create_user_and_database() {
+	local database=$1
+	echo "  Creating database '$database'"
+	psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL
+	    CREATE DATABASE $database;
+	    GRANT ALL PRIVILEGES ON DATABASE $database TO $POSTGRES_USER;
+EOSQL
+}
+
+if [ -n "$POSTGRES_MULTIPLE_DATABASES" ]; then
+	echo "Multiple database creation requested: $POSTGRES_MULTIPLE_DATABASES"
+	for db in $(echo $POSTGRES_MULTIPLE_DATABASES | tr ',' ' '); do
+		create_user_and_database $db
+	done
+	echo "Multiple databases created"
+fi