cleanup KAFKA_TOPIC_IGNORE_PREFIX from code as we are now using CloudKarfka with same topic name for dev and prod.

Author: sachin-maheshwari

README.md

@@ -24,7 +24,6 @@ The following parameters can be set in config files or in env variables:
   - `JWKS_URI`: TC auth JWKS URI (need only for local deployment)
 - **KAFKA**
   - `KAFKA_URL`: comma separated Kafka hosts
-  - `KAFKA_TOPIC_IGNORE_PREFIX`: ignore this prefix for topics in the Kafka
   - `KAFKA_GROUP_ID`: Kafka consumer group id
   - `KAFKA_CLIENT_CERT`: Kafka connection certificate, optional;
       if not provided, then SSL connection is not used, direct insecure connection is used;
@@ -97,7 +96,6 @@ You may reuse it during review.
 - for local development environment you can set variables as following:
   - `AUTH_SECRET`,`VALID_ISSUERS` can get from [tc-project-service config](https://github.com/topcoder-platform/tc-project-service/blob/dev/config/default.json)
   - `PORT=4000` because **connect-app** call this port by default
-  - `KAFKA_TOPIC_IGNORE_PREFIX=joan-26673.` (with point at the end)
   - `TC_API_V4_BASE_URL=https://api.topcoder-dev.com/v4`
   - `TC_API_V3_BASE_URL=https://api.topcoder-dev.com/v3`
   - `KAFKA_URL`, `KAFKA_CLIENT_CERT` and `KAFKA_CLIENT_CERT_KEY` get from [tc-bus-api readme](https://github.com/topcoder-platform/tc-bus-api/tree/dev)

config/default.js

@@ -23,7 +23,6 @@ module.exports = {
   // which skips token validation when locally deployed
 
   KAFKA_URL: process.env.KAFKA_URL,
-  KAFKA_TOPIC_IGNORE_PREFIX: process.env.KAFKA_TOPIC_IGNORE_PREFIX,
   KAFKA_GROUP_ID: process.env.KAFKA_GROUP_ID,
   KAFKA_CLIENT_CERT: process.env.KAFKA_CLIENT_CERT ? process.env.KAFKA_CLIENT_CERT.replace('\\n', '\n') : null,
   KAFKA_CLIENT_CERT_KEY: process.env.KAFKA_CLIENT_CERT_KEY ?

deploy.sh

@@ -34,7 +34,6 @@ AWS_ECS_SERVICE=$(eval "echo \$${ENV}_AWS_ECS_SERVICE")
 KAFKA_CLIENT_CERT=$(eval "echo \$${ENV}_KAFKA_CLIENT_CERT")
 KAFKA_CLIENT_CERT_KEY=$(eval "echo \$${ENV}_KAFKA_CLIENT_CERT_KEY")
 KAFKA_GROUP_ID=$(eval "echo \$${ENV}_KAFKA_GROUP_ID")
-KAFKA_TOPIC_IGNORE_PREFIX=$(eval "echo \$${ENV}_KAFKA_TOPIC_IGNORE_PREFIX")
 KAFKA_URL=$(eval "echo \$${ENV}_KAFKA_URL")
 AUTHSECRET=$(eval "echo \$${ENV}_AUTHSECRET")
 VALID_ISSUERS=$(eval "echo \$${ENV}_VALID_ISSUERS")
@@ -138,10 +137,6 @@ make_task_def(){
         "name": "KAFKA_GROUP_ID",
         "value": "%s"
       },
-      {
-        "name": "KAFKA_TOPIC_IGNORE_PREFIX",
-        "value": "%s"
-      },
       {
         "name": "KAFKA_URL",
         "value": "%s"
@@ -261,7 +256,7 @@ make_task_def(){
   }
 ]'
 
-	task_def=$(printf "$task_template" $AWS_ECS_CONTAINER_NAME $AWS_ACCOUNT_ID $AWS_REGION $AWS_REPOSITORY $TAG $ENV "$KAFKA_CLIENT_CERT" "$KAFKA_CLIENT_CERT_KEY" $KAFKA_GROUP_ID "$KAFKA_TOPIC_IGNORE_PREFIX" $KAFKA_URL $DATABASE_URL $AUTHSECRET $TC_API_BASE_URL $TC_API_V3_BASE_URL $TC_API_V4_BASE_URL $TC_API_V5_BASE_URL $MESSAGE_API_BASE_URL $ENABLE_EMAILS $MENTION_EMAIL $REPLY_EMAIL_PREFIX $REPLY_EMAIL_DOMAIN $REPLY_EMAIL_FROM $DEFAULT_REPLY_EMAIL $ENABLE_DEV_MODE $DEV_MODE_EMAIL $LOG_LEVEL $VALID_ISSUERS $PORT "$API_CONTEXT_PATH" "$AUTH0_URL" "$AUTH0_AUDIENCE" $AUTH0_CLIENT_ID "$AUTH0_CLIENT_SECRET" $TOKEN_CACHE_TIME $AWS_ECS_CLUSTER $AWS_REGION $AWS_ECS_CLUSTER $ENV)
+	task_def=$(printf "$task_template" $AWS_ECS_CONTAINER_NAME $AWS_ACCOUNT_ID $AWS_REGION $AWS_REPOSITORY $TAG $ENV "$KAFKA_CLIENT_CERT" "$KAFKA_CLIENT_CERT_KEY" $KAFKA_GROUP_ID $KAFKA_URL $DATABASE_URL $AUTHSECRET $TC_API_BASE_URL $TC_API_V3_BASE_URL $TC_API_V4_BASE_URL $TC_API_V5_BASE_URL $MESSAGE_API_BASE_URL $ENABLE_EMAILS $MENTION_EMAIL $REPLY_EMAIL_PREFIX $REPLY_EMAIL_DOMAIN $REPLY_EMAIL_FROM $DEFAULT_REPLY_EMAIL $ENABLE_DEV_MODE $DEV_MODE_EMAIL $LOG_LEVEL $VALID_ISSUERS $PORT "$API_CONTEXT_PATH" "$AUTH0_URL" "$AUTH0_AUDIENCE" $AUTH0_CLIENT_ID "$AUTH0_CLIENT_SECRET" $TOKEN_CACHE_TIME $AWS_ECS_CLUSTER $AWS_REGION $AWS_ECS_CLUSTER $ENV)
 }
 
 register_definition() {

src/app.js

@@ -34,11 +34,9 @@ function startKafkaConsumer(handlers, notificationServiceHandlers) {
     const message = m.message.value.toString('utf8');
     logger.info(`Handle Kafka event message; Topic: ${topic}; Partition: ${partition}; Offset: ${
       m.offset}; Message: ${message}.`);
-    // ignore configured Kafka topic prefix
+    
     let topicName = topic;
-    if (config.KAFKA_TOPIC_IGNORE_PREFIX && topicName.startsWith(config.KAFKA_TOPIC_IGNORE_PREFIX)) {
-      topicName = topicName.substring(config.KAFKA_TOPIC_IGNORE_PREFIX.length);
-    }
+    
     // find handler
     const handler = handlers[topicName];
     if (!handler) {
@@ -75,8 +73,7 @@ function startKafkaConsumer(handlers, notificationServiceHandlers) {
   consumer
     .init()
     .then(() => _.each(_.keys(handlers),
-      // add back the ignored topic prefix to use full topic name
-      (topicName) => consumer.subscribe(`${config.KAFKA_TOPIC_IGNORE_PREFIX || ''}${topicName}`, dataHandler)))
+      (topicName) => consumer.subscribe(topicName, dataHandler)))
     .catch((err) => logger.error(err));
 }