From 04a7a50352a9308517e21b52397e0ba72d640081 Mon Sep 17 00:00:00 2001 From: Maksym Mykhailenko Date: Wed, 11 Mar 2020 18:27:44 +0800 Subject: [PATCH 1/3] feat: use .env file for "npm run start:dev" --- package-lock.json | 76 ++++++++++++++++++++++++++++++++++------------- package.json | 3 +- 2 files changed, 58 insertions(+), 21 deletions(-) diff --git a/package-lock.json b/package-lock.json index 44dcac10..16c66154 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2238,7 +2238,6 @@ "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dev": true, "requires": { "color-name": "1.1.3" } @@ -2246,8 +2245,7 @@ "color-name": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", - "dev": true + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" }, "colors": { "version": "1.4.0", @@ -2683,6 +2681,11 @@ "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", "dev": true }, + "diff-match-patch": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/diff-match-patch/-/diff-match-patch-1.0.4.tgz", + "integrity": "sha512-Uv3SW8bmH9nAtHKaKSanOQmj2DnlH65fUpcrMdfdaOxUG02QQ4YGZ8AE7kKOMisF7UqvOlGKVYWRvezdncW9lg==" + }, "doctrine": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz", @@ -2701,6 +2704,11 @@ "is-obj": "^1.0.0" } }, + "dotenv": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz", + "integrity": "sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw==" + }, "dottie": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.2.tgz", @@ -4319,7 +4327,6 @@ "version": "4.7.2", "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.2.tgz", "integrity": "sha512-4PwqDL2laXtTWZghzzCtunQUTLbo31pcCJrd/B/9JP8XbhVzpS5ZXuKqlOzsd1rtcaLo4KqAn8nl8mkknS4MHw==", - "dev": true, "requires": { "neo-async": "^2.6.0", "optimist": "^0.6.1", @@ -5285,6 +5292,48 @@ "minimist": "^1.2.0" } }, + "jsondiffpatch": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/jsondiffpatch/-/jsondiffpatch-0.4.1.tgz", + "integrity": "sha512-t0etAxTUk1w5MYdNOkZBZ8rvYYN5iL+2dHCCx/DpkFm/bW28M6y5nUS83D4XdZiHy35Fpaw6LBb+F88fHZnVCw==", + "requires": { + "chalk": "^2.3.0", + "diff-match-patch": "^1.0.0" + }, + "dependencies": { + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "requires": { + "color-convert": "^1.9.0" + } + }, + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "requires": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + }, + "has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=" + }, + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "requires": { + "has-flag": "^3.0.0" + } + } + } + }, "jsonfile": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", @@ -6052,8 +6101,7 @@ "neo-async": { "version": "2.6.1", "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.1.tgz", - "integrity": "sha512-iyam8fBuCUpWeKPGpaNMetEocMt364qkCsfL9JuhjXX6dRnguRVOfk2GZaDpPjcOKiiXCPINZC1GczQ7iTq3Zw==", - "dev": true + "integrity": "sha512-iyam8fBuCUpWeKPGpaNMetEocMt364qkCsfL9JuhjXX6dRnguRVOfk2GZaDpPjcOKiiXCPINZC1GczQ7iTq3Zw==" }, "next-tick": { "version": "1.0.0", @@ -6733,7 +6781,6 @@ "version": "0.6.1", "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz", "integrity": "sha1-2j6nRob6IaGaERwybpDrFaAZZoY=", - "dev": true, "requires": { "minimist": "~0.0.1", "wordwrap": "~0.0.2" @@ -6742,14 +6789,12 @@ "minimist": { "version": "0.0.10", "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz", - "integrity": "sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8=", - "dev": true + "integrity": "sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8=" }, "wordwrap": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz", - "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=", - "dev": true + "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=" } } }, @@ -8442,14 +8487,6 @@ "escodegen": "^1.8.1" } }, - "static-eval": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/static-eval/-/static-eval-2.0.2.tgz", - "integrity": "sha512-N/D219Hcr2bPjLxPiV+TQE++Tsmrady7TqAJugLy7Xk1EumfDWS/f5dtBbkRCGE7wKKXuYockQoj8Rm2/pVKyg==", - "requires": { - "escodegen": "^1.8.1" - } - }, "static-extend": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/static-extend/-/static-extend-0.1.2.tgz", @@ -8903,7 +8940,6 @@ "version": "3.7.7", "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.7.7.tgz", "integrity": "sha512-FeSU+hi7ULYy6mn8PKio/tXsdSXN35lm4KgV2asx00kzrLU9Pi3oAslcJT70Jdj7PHX29gGUPOT6+lXGBbemhA==", - "dev": true, "optional": true, "requires": { "commander": "~2.20.3", diff --git a/package.json b/package.json index e1ccee32..838f212d 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "migrate:es:metadata": "./node_modules/.bin/babel-node migrations/helpers/indexMetadataDirectly.js", "prestart": "npm run -s build", "start": "node dist", - "start:dev": "NODE_ENV=development PORT=8001 nodemon -w src --exec \"babel-node src --presets es2015\" | ./node_modules/.bin/bunyan", + "start:dev": "NODE_ENV=development PORT=8001 nodemon -w src --exec \"node --require dotenv/config --require babel-core/register src\" | ./node_modules/.bin/bunyan", "startKafkaConsumers": "npm run -s build && node dist/index-kafka.js", "startKafkaConsumers:dev": "NODE_ENV=development nodemon -w src --exec \"babel-node src/index-kafka.js --presets es2015\" | ./node_modules/.bin/bunyan", "test": "NODE_ENV=test npm run lint && NODE_ENV=test npm run sync:es && NODE_ENV=test npm run sync:db && NODE_ENV=test ./node_modules/.bin/istanbul cover ./node_modules/mocha/bin/_mocha -- --timeout 10000 --require babel-core/register $(find src -path '*spec.js*') --exit", @@ -49,6 +49,7 @@ "config": "^1.20.1", "continuation-local-storage": "^3.1.7", "cors": "^2.8.4", + "dotenv": "^8.2.0", "elasticsearch": "^16.1.1", "express": "^4.13.4", "express-list-routes": "^0.1.4", From c168736b155b1ce1f0d002dbe8b0ebc77c9e6d52 Mon Sep 17 00:00:00 2001 From: Maksym Mykhailenko Date: Wed, 18 Mar 2020 13:06:32 +0800 Subject: [PATCH 2/3] docs: use .env file for docker and "start:dev" --- README.md | 71 ++++++++++++++++++++++++++---------------------------- sample.env | 5 ++++ 2 files changed, 39 insertions(+), 37 deletions(-) create mode 100644 sample.env diff --git a/README.md b/README.md index 63e9f838..2af61979 100644 --- a/README.md +++ b/README.md @@ -36,15 +36,32 @@ Local setup should work good on **Linux** and **macOS**. But **Windows** is not npm install ``` +2. Local config + + 1. Copy `sample.env` to `.env` file. And inside `.env` file set values for `AUTH0_CLIENT_ID` and `AUTH0_CLIENT_SECRET`.
+ Values from this file would be automatically used by `docker-compose` and command `npm run start:dev` below. + + 2. Copy config file `config/m2m.local.js` into `config/local.js`: + ```bash + cp config/m2m.local.js config/local.js + ``` + + 3. Set `dockerhost` to point the IP address of Docker. Docker IP address depends on your system. For example if docker is run on IP `127.0.0.1` add a the next line to your `/etc/hosts` file: + ``` + 127.0.0.1 dockerhost + ``` + + Alternatively, you may update `config/local.js` and replace `dockerhost` with your docker IP address. + 3. Start **ONE** of the docker-compose files with dependant services which are required for Project Service to work 1. **Minimal** `./local/docker-compose.yml`: *Use this docker-compose if you only want to test and modify code of Project Service and you don't need Elasticsearch (ES) to work.* - Run, inside folder `./local`: + Run, in the project root folder: ```bash - docker-compose up + docker-compose -f local/docker-compose.yml up ```
Click to see details @@ -64,17 +81,16 @@ Local setup should work good on **Linux** and **macOS**. But **Windows** is not *Use this docker-compose if you want to test and modify code of Project Service together with one of the next relative services: [tc-bus-api](https://github.com/topcoder-platform/tc-bus-api), [project-processor-es](https://github.com/topcoder-platform/project-processor-es), [tc-notifications](https://github.com/topcoder-platform/tc-notifications) or you need Elasticsearch (ES) to work.* - 1. Set environment variables `AUTH0_CLIENT_ID`, `AUTH0_CLIENT_SECRET`, `AUTH0_URL`, `AUTH0_AUDIENCE`, `AUTH0_PROXY_SERVER_URL` - 2. Run, inside folder `./local/full` + 1. Run, in the project root folder: ```bash - docker-compose up -d + docker-compose -f local/full/docker-compose.yml up -d ``` - 3. Wait until all containers are fully started. As a good indicator, wait until `project-processor-es` successfully started by viewing its logs: + 2. Wait until all containers are fully started. As a good indicator, wait until `project-processor-es` successfully started by viewing its logs: ```bash - docker-compose logs -f project-processor-es + docker-compose -f local/full/docker-compose.yml logs -f project-processor-es ```
Click to see example logs @@ -95,7 +111,7 @@ Local setup should work good on **Linux** and **macOS**. But **Windows** is not ```
- 4. If you want to modify the code of any of the services which are run inside this docker-compose file, you can stop such service inside docker-compose by command `docker-compose stop -f ` and run the service separately, following its README file. + 3. If you want to modify the code of any of the services which are run inside this docker-compose file, you can stop such service inside docker-compose by command `docker-compose -f local/full/docker-compose.yml stop -f ` and run the service separately, following its README file.
Click to see details
@@ -121,56 +137,37 @@ Local setup should work good on **Linux** and **macOS**. But **Windows** is not - To view the logs from any container inside docker-compose use the following command, replacing `SERVICE_NAME` with the corresponding value under the **Name** column in the above table: ```bash - cd local/full - docker-compose logs -f SERVICE_NAME + docker-compose -f local/full/docker-compose.yml logs -f SERVICE_NAME ```
*NOTE: In production these dependencies / services are hosted & managed outside Project Service.* -4. Local config - - 1. Copy config file `config/m2m.local.js` into `config/local.js`: - ```bash - cp config/m2m.local.js config/local.js - ``` - - 2. Set `dockerhost` to point the IP address of Docker. Docker IP address depends on your system. For example if docker is run on IP `127.0.0.1` add a the next line to your `/etc/hosts` file: - ``` - 127.0.0.1 dockerhost - ``` - - Alternatively, you may update `config/local.js` and replace `dockerhost` with your docker IP address. - -5. Create tables in DB +4. Create tables in DB ```bash NODE_ENV=development npm run sync:db ``` *NOTE: this will drop tables if they already exist.* -6. Create ES (Elasticsearch) indexes +5. Create ES (Elasticsearch) indexes ```bash NODE_ENV=development npm run sync:es ``` *NOTE: This will first clear all the indices and than recreate them. So use with caution.* -7. Start Project Service - - 1. Set environment variables `AUTH0_CLIENT_ID`, `AUTH0_CLIENT_SECRET`, `AUTH0_URL`, `AUTH0_AUDIENCE`, `AUTH0_PROXY_SERVER_URL` +6. Start Project Service - 2. Run - - ```bash - npm run start:dev - ``` + ```bash + npm run start:dev + ``` - Runs the Project Service using nodemon, so it would be restarted after any of the files is updated. - The project service will be served on `http://localhost:8001`. + Runs the Project Service using nodemon, so it would be restarted after any of the files is updated. + The project service will be served on `http://localhost:8001`. -8. *(Optional)* Start Project Service Kafka Consumer +7. *(Optional)* Start Project Service Kafka Consumer *Run this only if you want to test or modify logic of `lastActivityAt` or `lastActivityBy`.* diff --git a/sample.env b/sample.env new file mode 100644 index 00000000..706a0950 --- /dev/null +++ b/sample.env @@ -0,0 +1,5 @@ +AUTH0_CLIENT_ID= +AUTH0_CLIENT_SECRET= +AUTH0_URL=https://topcoder-dev.auth0.com/oauth/token +AUTH0_AUDIENCE=https://m2m.topcoder-dev.com/ +AUTH0_PROXY_SERVER_URL=https://auth0proxy.topcoder-dev.com/token \ No newline at end of file From 02808a5a1155afba870cc13a1507cf33ba735567 Mon Sep 17 00:00:00 2001 From: Maksym Mykhailenko Date: Wed, 1 Apr 2020 17:12:13 +0300 Subject: [PATCH 3/3] chore: reduce chance for committing Auth0 values --- README.md | 12 +++++++++--- sample.env | 5 ----- 2 files changed, 9 insertions(+), 8 deletions(-) delete mode 100644 sample.env diff --git a/README.md b/README.md index 7811be41..00db1b96 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,14 @@ Local setup should work good on **Linux** and **macOS**. But **Windows** is not 2. Local config - 1. Copy `sample.env` to `.env` file. And inside `.env` file set values for `AUTH0_CLIENT_ID` and `AUTH0_CLIENT_SECRET`.
+ 1. In the `tc-project-service` root directory create `.env` file with the environment variables _(values should be shared with you on the forum)_:
+ ``` + AUTH0_CLIENT_ID=... + AUTH0_CLIENT_SECRET=... + AUTH0_URL=... + AUTH0_AUDIENCE=... + AUTH0_PROXY_SERVER_URL=... + ``` Values from this file would be automatically used by `docker-compose` and command `npm run start:dev` below. 2. Copy config file `config/m2m.local.js` into `config/local.js`: @@ -272,5 +279,4 @@ docker exec -it tc-projects-kafka /opt/kafka/bin/kafka-console-producer.sh --bro ## References -- [Projects Service Architecture](./docs/guides/architercture/architecture.md) -- [Projects Service Architecture](./docs/guides/architercture/architecture.md) +- [Projects Service Architecture](./docs/guides/architercture/architecture.md) \ No newline at end of file diff --git a/sample.env b/sample.env deleted file mode 100644 index 706a0950..00000000 --- a/sample.env +++ /dev/null @@ -1,5 +0,0 @@ -AUTH0_CLIENT_ID= -AUTH0_CLIENT_SECRET= -AUTH0_URL=https://topcoder-dev.auth0.com/oauth/token -AUTH0_AUDIENCE=https://m2m.topcoder-dev.com/ -AUTH0_PROXY_SERVER_URL=https://auth0proxy.topcoder-dev.com/token \ No newline at end of file