Initial commit

Author: ThomasKranitsas

.gitignore

@@ -0,0 +1,8 @@
+.idea
+node_modules
+mock-api/node_modules
+*.log
+.DS_Store
+.env
+coverage
+.nyc_output

README.md

@@ -0,0 +1,139 @@
+# Topcoder Challenge Resources Processor
+
+## Dependencies
+
+- nodejs https://nodejs.org/en/ (v10+)
+- Kafka
+
+## Configuration
+
+Configuration for the processor is at `config/default.js` and `config/production.js`.
+The following parameters can be set in config files or in env variables:
+
+- DISABLE_LOGGING: whether to disable logging, default is false
+- LOG_LEVEL: the log level, default value: 'debug'
+- AUTH0_URL: AUTH0 URL, used to get M2M token
+- AUTH0_AUDIENCE: AUTH0 audience, used to get M2M token, default value is 'https://www.topcoder-dev.com'
+- TOKEN_CACHE_TIME: AUTH0 token cache time, used to get M2M token
+- AUTH0_PROXY_SERVER_URL: Auth0 proxy server url, used to get TC M2M token
+- AUTH0_CLIENT_ID: AUTH0 client id, used to get M2M token
+- AUTH0_CLIENT_SECRET: AUTH0 client secret, used to get M2M token
+- KAFKA_URL: comma separated Kafka hosts, default value: 'localhost:9092'
+- KAFKA_GROUP_ID: the Kafka group id, default value: 'challenge-resources-processor'
+- KAFKA_CLIENT_CERT: Kafka connection certificate, optional, default value is undefined;
+    if not provided, then SSL connection is not used, direct insecure connection is used;
+    if provided, it can be either path to certificate file or certificate content
+- KAFKA_CLIENT_CERT_KEY: Kafka connection private key, optional, default value is undefined;
+    if not provided, then SSL connection is not used, direct insecure connection is used;
+    if provided, it can be either path to private key file or private key content
+- KAFKA_TOPIC: Kafka topic to listen, default value is 'challenge.notification.create'
+- REQUEST_TIMEOUT: superagent request timeout in milliseconds, default value is 20000
+- RESOURCE_ROLE_ID: the challenge member resource role id
+- GET_PROJECT_API_BASE: get project API base URL, default value is mock API 'http://localhost:4000/v4/projects'
+- SEARCH_MEMBERS_API_BASE: search members API base URL, default value is 'https://api.topcoder.com/v3/members/_search'
+- CREATE_RESOURCE_API: create resource API URL, default value is mock API 'http://localhost:4000/v5/resources'
+
+
+Set the following environment variables so that the app can get TC M2M token (use 'set' insted of 'export' for Windows OS):
+
+- export AUTH0_CLIENT_ID=EkE9qU3Ey6hdJwOsF1X0duwskqcDuElW
+- export AUTH0_CLIENT_SECRET=Iq7REiEacFmepPh0UpKoOmc6u74WjuoJriLayeVnt311qeKNBvhRNBe9BZ8WABYk
+- export AUTH0_URL=https://topcoder-dev.auth0.com/oauth/token
+- export AUTH0_AUDIENCE=https://m2m.topcoder-dev.com/
+- export TOKEN_CACHE_TIME=90
+
+Also note that there is a `/health` endpoint that checks for the health of the app. This sets up an expressjs server and listens on the environment variable `PORT`. It's not part of the configuration file and needs to be passed as an environment variable
+
+
+Configuration for the tests is at `config/test.js`, only add such new configurations if different than `config/default.js`:
+- WAIT_TIME: wait time used in test, default is 2000 or 2 seconds
+
+## Local Kafka setup
+
+- `http://kafka.apache.org/quickstart` contains details to setup and manage Kafka server,
+  below provides details to setup Kafka server in Mac, Windows will use bat commands in bin/windows instead
+- download kafka at `https://www.apache.org/dyn/closer.cgi?path=/kafka/1.1.0/kafka_2.11-1.1.0.tgz`
+- extract out the doanlowded tgz file
+- go to extracted directory kafka_2.11-0.11.0.1
+- start ZooKeeper server:
+  `bin/zookeeper-server-start.sh config/zookeeper.properties`
+- use another terminal, go to same directory, start the Kafka server:
+  `bin/kafka-server-start.sh config/server.properties`
+- note that the zookeeper server is at localhost:2181, and Kafka server is at localhost:9092
+- use another terminal, go to same directory, create topic:
+  `bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic challenge.notification.create`
+- verify that the topics are created:
+  `bin/kafka-topics.sh --list --zookeeper localhost:2181`,
+  it should list out the created topic
+- run the producer and then write some message into the console to send to the `challenge.notification.create` topic:
+  `bin/kafka-console-producer.sh --broker-list localhost:9092 --topic challenge.notification.create`
+  in the console, write message, one message per line:
+  `{ "topic": "challenge.notification.create", "originator": "challenge-api", "timestamp": "2019-02-16T00:00:00", "mime-type": "application/json", "payload": { "id": "5505f779-b28b-428f-888b-e523b443f3ea", "typeId": "7705f779-b28b-428f-888b-e523b443f3ea", "track": "code", "name": "test", "description": "desc", "timelineTemplateId": "8805f779-b28b-428f-888b-e523b443f3ea", "phases": [{ "id": "8805f779-b28b-428f-888b-e523b443f3eb", "name": "phase", "isActive": true, "duration": 1000 }], "prizeSets": [{ "type": "Challenge prizes", "prizes": [{ "type": "1st", "value": 600 }] }], "reviewType": "community", "tags": ["tag1"], "projectId": 30055214, "startDate": "2019-02-19T00:00:00", "status": "Draft", "created": "2019-02-16T00:00:00", "createdBy": "tester" } }`
+- optionally, use another terminal, go to same directory, start a consumer to view the messages:
+  `bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic challenge.notification.create --from-beginning`
+
+
+## Local deployment
+
+- start mock-api, go to `mock-api` folder, run `npm i` and `npm start`, mock api is running at `http://localhost:4000`
+- go to project root folder, install dependencies `npm i`
+- run code lint check `npm run lint`, running `npm run lint:fix` can fix some lint errors
+- start processor app `npm start`
+
+
+## Unit Tests and E2E Tests
+
+Before running tests, setup and start kafka server, start the mock API, but do not start the processor app.
+
+- run `npm run test` to execute unit tests.
+- run `npm run test:cov` to execute unit tests and generate coverage report.
+- run `npm run e2e` to execute e2e tests.
+- run `npm run e2e:cov` to execute e2e tests and generate coverage report.
+
+
+## Verification
+
+- setup and start kafka server, start mock API, start processor app
+- start kafka-console-producer to write messages to `challenge.notification.create` topic:
+  `bin/kafka-console-producer.sh --broker-list localhost:9092 --topic challenge.notification.create`
+- write message:
+  `{ "topic": "challenge.notification.create", "originator": "challenge-api", "timestamp": "2019-02-16T00:00:00", "mime-type": "application/json", "payload": { "id": "5505f779-b28b-428f-888b-e523b443f3ea", "typeId": "7705f779-b28b-428f-888b-e523b443f3ea", "track": "code", "name": "test", "description": "desc", "timelineTemplateId": "8805f779-b28b-428f-888b-e523b443f3ea", "phases": [{ "id": "8805f779-b28b-428f-888b-e523b443f3eb", "name": "phase", "isActive": true, "duration": 1000 }], "prizeSets": [{ "type": "Challenge prizes", "prizes": [{ "type": "1st", "value": 600 }] }], "reviewType": "community", "tags": ["tag1"], "projectId": 30055214, "startDate": "2019-02-19T00:00:00", "status": "Draft", "created": "2019-02-16T00:00:00", "createdBy": "tester" } }`
+
+- you will see app logging:
+```bash
+info: Process message of challenge id 5505f779-b28b-428f-888b-e523b443f3ea and project id 30055214
+info: Found member ids [40152933, 40141336] of project id 30055214
+info: Created resource: {
+    "challengeId": "5505f779-b28b-428f-888b-e523b443f3ea",
+    "memberHandle": "lordofparadox",
+    "roleId": "6605f779-b28b-428f-888b-e523b443f3ea",
+    "id": "d4ac5715-b1ed-430b-86ca-56d022c97ce9",
+    "memberId": "9e2ad182-d4f7-4c4d-9421-29bc8bd56dfb",
+    "created": "2019-08-20T23:02:45.818Z",
+    "createdBy": "mock-api"
+}
+info: Created resource: {
+    "challengeId": "5505f779-b28b-428f-888b-e523b443f3ea",
+    "memberHandle": "SethHafferkamp",
+    "roleId": "6605f779-b28b-428f-888b-e523b443f3ea",
+    "id": "8a835c7e-5a90-47f1-a020-c95c415e32db",
+    "memberId": "980bd2a6-b4b6-4253-87e7-67cb931a76a2",
+    "created": "2019-08-20T23:02:45.822Z",
+    "createdBy": "mock-api"
+}
+info: Successfully processed message of challenge id 5505f779-b28b-428f-888b-e523b443f3ea and project id 30055214
+```
+
+
+- you may write invalid messages like:
+  `{ "topic": "challenge.notification.create", "originator": "challenge-api", "timestamp": "2019-02-16T00:00:00", "mime-type": "application/json", "payload": { "id": "abc", "typeId": "7705f779-b28b-428f-888b-e523b443f3ea", "track": "code", "name": "test", "description": "desc", "timelineTemplateId": "8805f779-b28b-428f-888b-e523b443f3ea", "phases": [{ "id": "8805f779-b28b-428f-888b-e523b443f3eb", "name": "phase", "isActive": true, "duration": 1000 }], "prizeSets": [{ "type": "Challenge prizes", "prizes": [{ "type": "1st", "value": 600 }] }], "reviewType": "community", "tags": ["tag1"], "projectId": 30055214, "startDate": "2019-02-19T00:00:00", "status": "Draft", "created": "2019-02-16T00:00:00", "createdBy": "tester" } }`
+
+  `{ "topic": "challenge.notification.create", "originator": "challenge-api", "timestamp": "2019-02-16T00:00:00", "mime-type": "application/json", "payload": { "id": "5505f779-b28b-428f-888b-e523b443f3ea", "typeId": "7705f779-b28b-428f-888b-e523b443f3ea", "track": "code", "name": "test", "description": "desc", "timelineTemplateId": "8805f779-b28b-428f-888b-e523b443f3ea", "phases": [{ "id": "8805f779-b28b-428f-888b-e523b443f3eb", "name": "phase", "isActive": true, "duration": 1000 }], "prizeSets": [{ "type": "Challenge prizes", "prizes": [{ "type": "1st", "value": 600 }] }], "reviewType": "community", "tags": ["tag1"], "projectId": 30055214, "startDate": "2019-02-19T00:00:00", "status": "Draft", "created": "abc", "createdBy": "tester" } }`
+
+  `{ [ { abc`
+- then in the app console, you will see error messages
+
+
+- to test the health check API, start the processor, then browse `http://localhost:3000/health` in a browser,
+  and you will see result `{"checksRun":1}`, you may change the health check API port by setting `PORT` environment variable
+

config/default.js

@@ -0,0 +1,34 @@
+/**
+ * The default configuration file.
+ */
+
+module.exports = {
+  DISABLE_LOGGING: process.env.DISABLE_LOGGING
+    ? process.env.DISABLE_LOGGING.toLowerCase() === 'true' : false, // If true, logging will be disabled
+  LOG_LEVEL: process.env.LOG_LEVEL || 'debug',
+
+  // used to get M2M token
+  AUTH0_URL: process.env.AUTH0_URL,
+  AUTH0_AUDIENCE: process.env.AUTH0_AUDIENCE || 'https://www.topcoder-dev.com',
+  TOKEN_CACHE_TIME: process.env.TOKEN_CACHE_TIME,
+  AUTH0_PROXY_SERVER_URL: process.env.AUTH0_PROXY_SERVER_URL,
+  AUTH0_CLIENT_ID: process.env.AUTH0_CLIENT_ID,
+  AUTH0_CLIENT_SECRET: process.env.AUTH0_CLIENT_SECRET,
+
+  KAFKA_URL: process.env.KAFKA_URL || 'localhost:9092',
+  KAFKA_GROUP_ID: process.env.KAFKA_GROUP_ID || 'challenge-resources-processor',
+  // below are used for secure Kafka connection, they are optional
+  // for the local Kafka, they are not needed
+  KAFKA_CLIENT_CERT: process.env.KAFKA_CLIENT_CERT,
+  KAFKA_CLIENT_CERT_KEY: process.env.KAFKA_CLIENT_CERT_KEY,
+  KAFKA_TOPIC: process.env.KAFKA_TOPIC || 'challenge.notification.create',
+
+  // superagent request timeout in milliseconds
+  REQUEST_TIMEOUT: process.env.REQUEST_TIMEOUT ? Number(process.env.REQUEST_TIMEOUT) : 20000,
+
+  RESOURCE_ROLE_ID: process.env.RESOURCE_ROLE_ID || '6605f779-b28b-428f-888b-e523b443f3ea',
+
+  GET_PROJECT_API_BASE: process.env.GET_PROJECT_API_BASE || 'http://localhost:4000/v4/projects',
+  SEARCH_MEMBERS_API_BASE: process.env.SEARCH_MEMBERS_API_BASE || 'https://api.topcoder.com/v3/members/_search',
+  CREATE_RESOURCE_API: process.env.CREATE_RESOURCE_API || 'http://localhost:4000/v5/resources'
+}

config/production.js

@@ -0,0 +1,7 @@
+/**
+ * Production configuration file
+ */
+
+module.exports = {
+  LOG_LEVEL: process.env.LOG_LEVEL || 'info'
+}

config/test.js

@@ -0,0 +1,7 @@
+/**
+ * The test configuration.
+ */
+
+module.exports = {
+  WAIT_TIME: 2000
+}

mock-api/README.md

@@ -0,0 +1,19 @@
+# Mock TC APIs
+
+It provides a mock API to create TC resource, and a mock API to get project by id.
+
+## Dependencies
+- NodeJS https://nodejs.org/en/ (v10+)
+
+
+## Configuration
+Configuration is at `config/default.js`, you may also set env variables.
+There are following config params:
+- `PORT`: REST app port, default value is 4000
+
+
+## Local deployment
+- Install dependencies `npm i`
+- Start app `npm start`
+- App is running at `http://localhost:4000`
+

mock-api/app.js

@@ -0,0 +1,153 @@
+/**
+ * The application entry point for mock TC APIs
+ */
+
+const express = require('express')
+const bodyParser = require('body-parser')
+const cors = require('cors')
+const config = require('config')
+const winston = require('winston')
+const uuid = require('uuid/v4')
+
+const app = express()
+app.set('port', config.PORT)
+
+app.use(cors())
+app.use(bodyParser.json())
+app.use(bodyParser.urlencoded({ extended: true }))
+
+// create resource
+app.post('/v5/resources', (req, res) => {
+  winston.info(`Create resource: ${JSON.stringify(req.body, null, 4)}`)
+
+  if (req.body.challengeId === '55ba42a4-f7f9-4f1e-bf81-a8553ba53778') {
+    winston.info('Failed to create resource.')
+    res.status(500).json({ message: 'there is error' })
+    return
+  }
+
+  const result = req.body
+  result.id = uuid()
+  result.memberId = uuid()
+  result.created = new Date()
+  result.createdBy = 'mock-api'
+
+  winston.info(`Created resource: ${JSON.stringify(result, null, 4)}`)
+  res.json(result)
+})
+
+// get project by id
+app.get('/v4/projects/:projectId', (req, res) => {
+  const projectId = req.params.projectId
+  winston.info(`Get project of id: ${projectId}`)
+
+  let result
+  if (projectId === '123') {
+    result = {
+      id: '6eba42a4-f7f9-4f1e-bf81-a8553ba5372a',
+      version: 'v4',
+      result: {
+        success: false,
+        status: 404,
+        content: 'it is not found'
+      }
+    }
+  } else {
+    result = {
+      id: '7eba42a4-f7f9-4f1e-bf81-a8553ba5371a',
+      version: 'v4',
+      result: {
+        success: true,
+        status: 200,
+        content: {
+          id: projectId,
+          directProjectId: null,
+          billingAccountId: null,
+          name: 'Alpha',
+          description: 'And away we go.',
+          external: null,
+          bookmarks: null,
+          estimatedPrice: '0.00',
+          actualPrice: null,
+          terms: [],
+          type: 'app_dev',
+          status: 'paused',
+          details: {
+            fontIds: [],
+            offlineAccess: null,
+            deviceIds: ['PHONE'],
+            offlineAccessComment: null,
+            orientationIds: ['PORTRAIT'],
+            iconsetIds: [],
+            usesPersonalInformation: null,
+            colorSwatchIds: [],
+            modelType: 'app-project',
+            version: null,
+            securityLevel: null,
+            features: '[ ]',
+            apiIntegration: null,
+            designNotes: null
+          },
+          challengeEligibility: null,
+          cancelReason: null,
+          templateId: null,
+          createdAt: '2016-04-27T22:15:04.285Z',
+          updatedAt: '2016-10-01T17:33:44.000Z',
+          deletedBy: null,
+          createdBy: 40141336,
+          updatedBy: 40152933,
+          version: 'v2',
+          lastActivityAt: '2016-10-01T17:33:44.000Z',
+          lastActivityUserId: '40152933',
+          members: [{
+            id: 995,
+            userId: 40152933,
+            role: 'manager',
+            isPrimary: true,
+            createdAt: '2016-10-01T17:33:32.000Z',
+            updatedAt: '2016-10-01T17:33:32.000Z',
+            deletedBy: null,
+            createdBy: 40152933,
+            updatedBy: 40152933,
+            projectId
+          }, {
+            id: 998,
+            userId: 40141336,
+            role: 'manager',
+            isPrimary: true,
+            createdAt: '2016-10-02T17:33:32.000Z',
+            updatedAt: '2016-10-02T17:33:32.000Z',
+            deletedBy: null,
+            createdBy: 40152933,
+            updatedBy: 40152933,
+            projectId
+          }],
+          attachments: [],
+          invites: [],
+          scopeChangeRequests: []
+        },
+        metadata: {
+          totalCount: 1
+        }
+      }
+    }
+  }
+
+  winston.info(`Result: ${JSON.stringify(result, null, 4)}`)
+  res.json(result)
+})
+
+app.use((req, res) => {
+  res.status(404).json({ error: 'route not found' })
+})
+
+app.use((err, req, res, next) => {
+  winston.error(err)
+  res.status(500).json({
+    error: err.message
+  })
+})
+
+app.listen(app.get('port'), '0.0.0.0', () => {
+  winston.info(`Express server listening on port ${app.get('port')}`)
+})

mock-api/config/default.js

@@ -0,0 +1,7 @@
+/**
+ * The default configuration file.
+ */
+
+module.exports = {
+  PORT: process.env.PORT || 4000
+}