diff --git a/circle.yml b/circle.yml index f5993dff..ddc060f3 100644 --- a/circle.yml +++ b/circle.yml @@ -24,7 +24,7 @@ dependencies: deployment: development: - branch: dev + branch: [dev, 'feature/admin-endpoints'] commands: - ./ebs_deploy.sh tc-project-service DEV $CIRCLE_BUILD_NUM diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index 36774d4d..59ff0ddd 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -11,6 +11,8 @@ "docType": "projectV4" }, "rabbitmqURL": "RABBITMQ_URL", + "pubsubQueueName": "PUBSUB_QUEUE_NAME", + "pubsubExchangeName": "PUBSUB_EXCHANGE_NAME", "directProjectServiceEndpoint": "DIRECT_PROJECT_SERVICE_ENDPOINT", "directProjectServiceTimeout": "DIRECT_PROJECT_SERVICE_TIMEOUt", "fileServiceEndpoint": "FILE_SERVICE_ENDPOINT", diff --git a/src/models/project.js b/src/models/project.js index be2e49ce..0c832e5f 100644 --- a/src/models/project.js +++ b/src/models/project.js @@ -155,6 +155,13 @@ module.exports = function defineProject(sequelize, DataTypes) { .then(projects => ({ rows: projects, count })); }); }, + findProjectRange(startId, endId, fields) { + return this.findAll({ + where: { id: { $between: [startId, endId] } }, + attributes: _.get(fields, 'projects', null), + raw: true, + }); + }, }, }); diff --git a/src/permissions/admin.ops.js b/src/permissions/admin.ops.js new file mode 100644 index 00000000..a1924e18 --- /dev/null +++ b/src/permissions/admin.ops.js @@ -0,0 +1,19 @@ +import util from '../util'; + +/** + * Only super admin are allowed to perform admin operations + * @param {Object} freq the express request instance + * @return {Promise} Returns a promise + */ +module.exports = freq => new Promise((resolve, reject) => { + const req = freq; + req.context = req.context || {}; + // check if auth user has acecss to this project + const hasAccess = util.hasAdminRole(req); + + if (!hasAccess) { + // user is not an admin nor is a registered project member + return reject(new Error('You do not have permissions to perform this action')); + } + return resolve(true); +}); diff --git a/src/permissions/index.js b/src/permissions/index.js index bf19b40b..e0797b3c 100644 --- a/src/permissions/index.js +++ b/src/permissions/index.js @@ -5,6 +5,7 @@ const projectView = require('./project.view'); const projectEdit = require('./project.edit'); const projectDelete = require('./project.delete'); const projectMemberDelete = require('./projectMember.delete'); +const projectAdmin = require('./admin.ops'); module.exports = () => { Authorizer.setDeniedStatusCode(403); @@ -21,4 +22,5 @@ module.exports = () => { Authorizer.setPolicy('project.removeAttachment', projectEdit); Authorizer.setPolicy('project.downloadAttachment', projectView); Authorizer.setPolicy('project.updateMember', projectEdit); + Authorizer.setPolicy('project.admin', projectAdmin); }; diff --git a/src/routes/admin/project-create-index.js b/src/routes/admin/project-create-index.js new file mode 100644 index 00000000..e03a6e2b --- /dev/null +++ b/src/routes/admin/project-create-index.js @@ -0,0 +1,327 @@ + +/* globals Promise */ + +import _ from 'lodash'; +import config from 'config'; +import { middleware as tcMiddleware } from 'tc-core-library-js'; +import util from '../../util'; + +/** +/** + * API to handle retrieving a single project by id + * + * Permissions: + * Only users that have access to the project can retrieve it. + * + */ + +// var permissions = require('tc-core-library-js').middleware.permissions +const permissions = tcMiddleware.permissions; +const ES_PROJECT_INDEX = config.get('elasticsearchConfig.indexName'); +const ES_PROJECT_TYPE = config.get('elasticsearchConfig.docType'); + +/** + * Get the request body for the specified index name + * @private + * + * @param {String} indexName the index name + * @param {String} docType document type + * @return {Object} the request body for the specified index name + */ +function getRequestBody(indexName, docType) { + const projectMapping = { + _all: { enabled: false }, + properties: { + actualPrice: { + type: 'double', + }, + attachments: { + type: 'nested', + properties: { + category: { + type: 'string', + index: 'not_analyzed', + }, + contentType: { + type: 'string', + index: 'not_analyzed', + }, + createdAt: { + type: 'date', + format: 'strict_date_optional_time||epoch_millis', + }, + createdBy: { + type: 'integer', + }, + description: { + type: 'string', + }, + filePath: { + type: 'string', + }, + id: { + type: 'long', + }, + projectId: { + type: 'long', + }, + size: { + type: 'double', + }, + title: { + type: 'string', + }, + updatedAt: { + type: 'date', + format: 'strict_date_optional_time||epoch_millis', + }, + updatedBy: { + type: 'integer', + }, + }, + }, + billingAccountId: { + type: 'long', + }, + bookmarks: { + type: 'nested', + properties: { + address: { + type: 'string', + }, + title: { + type: 'string', + }, + }, + }, + cancelReason: { + type: 'string', + }, + challengeEligibility: { + type: 'nested', + properties: { + groups: { + type: 'long', + }, + role: { + type: 'string', + index: 'not_analyzed', + }, + users: { + type: 'long', + }, + }, + }, + createdAt: { + type: 'date', + format: 'strict_date_optional_time||epoch_millis', + }, + createdBy: { + type: 'integer', + }, + description: { + type: 'string', + }, + details: { + type: 'nested', + properties: { + TBD_features: { + type: 'nested', + properties: { + description: { + type: 'string', + }, + id: { + type: 'integer', + }, + isCustom: { + type: 'boolean', + }, + title: { + type: 'string', + }, + }, + }, + TBD_usageDescription: { + type: 'string', + }, + appDefinition: { + properties: { + goal: { + properties: { + value: { + type: 'string', + }, + }, + }, + primaryTarget: { + type: 'string', + }, + users: { + properties: { + value: { + type: 'string', + }, + }, + }, + }, + }, + hideDiscussions: { + type: 'boolean', + }, + products: { + type: 'string', + }, + summary: { + type: 'string', + }, + utm: { + type: 'nested', + properties: { + code: { + type: 'string', + }, + }, + }, + }, + }, + directProjectId: { + type: 'long', + }, + estimatedPrice: { + type: 'double', + }, + external: { + properties: { + data: { + type: 'string', + }, + id: { + type: 'string', + index: 'not_analyzed', + }, + type: { + type: 'string', + index: 'not_analyzed', + }, + }, + }, + id: { + type: 'long', + }, + members: { + type: 'nested', + properties: { + createdAt: { + type: 'date', + format: 'strict_date_optional_time||epoch_millis', + }, + createdBy: { + type: 'integer', + }, + email: { + type: 'string', + index: 'not_analyzed', + }, + firstName: { + type: 'string', + }, + handle: { + type: 'string', + index: 'not_analyzed', + }, + id: { + type: 'long', + }, + isPrimary: { + type: 'boolean', + }, + lastName: { + type: 'string', + }, + projectId: { + type: 'long', + }, + role: { + type: 'string', + index: 'not_analyzed', + }, + updatedAt: { + type: 'date', + format: 'strict_date_optional_time||epoch_millis', + }, + updatedBy: { + type: 'integer', + }, + userId: { + type: 'long', + }, + }, + }, + name: { + type: 'string', + }, + status: { + type: 'string', + index: 'not_analyzed', + }, + terms: { + type: 'integer', + }, + type: { + type: 'string', + index: 'not_analyzed', + }, + updatedAt: { + type: 'date', + format: 'strict_date_optional_time||epoch_millis', + }, + updatedBy: { + type: 'integer', + }, + utm: { + properties: { + campaign: { + type: 'string', + }, + medium: { + type: 'string', + }, + source: { + type: 'string', + }, + }, + }, + }, + }; + const result = { + index: indexName, + updateAllTypes: true, + body: { + mappings: { }, + }, + }; + result.body.mappings[docType] = projectMapping; + return result; +} + + +module.exports = [ + permissions('project.admin'), + /** + * GET projects/{projectId} + * Get a project by id + */ + (req, res, next) => { // eslint-disable-line no-unused-vars + const logger = req.log; + logger.debug('Entered Admin#createIndex'); + const indexName = _.get(req, 'body.param.indexName', ES_PROJECT_INDEX); + const docType = _.get(req, 'body.param.docType', ES_PROJECT_TYPE); + logger.debug('indexName', indexName); + logger.debug('docType', docType); + + const esClient = util.getElasticSearchClient(); + esClient.indices.create(getRequestBody(indexName, docType)); + res.status(200).json(util.wrapResponse(req.id, { message: 'Create index request successfully submitted' })); + }, +]; diff --git a/src/routes/admin/project-delete-index.js b/src/routes/admin/project-delete-index.js new file mode 100644 index 00000000..472a1322 --- /dev/null +++ b/src/routes/admin/project-delete-index.js @@ -0,0 +1,45 @@ + +/* globals Promise */ + +import _ from 'lodash'; +import config from 'config'; +import { middleware as tcMiddleware } from 'tc-core-library-js'; +import util from '../../util'; + +/** +/** + * API to handle retrieving a single project by id + * + * Permissions: + * Only users that have access to the project can retrieve it. + * + */ + +// var permissions = require('tc-core-library-js').middleware.permissions +const permissions = tcMiddleware.permissions; +const ES_PROJECT_INDEX = config.get('elasticsearchConfig.indexName'); +// const ES_PROJECT_TYPE = config.get('elasticsearchConfig.docType'); + +module.exports = [ + permissions('project.admin'), + /** + * GET projects/{projectId} + * Get a project by id + */ + (req, res, next) => { // eslint-disable-line no-unused-vars + const logger = req.log; + logger.debug('Entered Admin#deleteIndex'); + const indexName = _.get(req, 'body.param.indexName', ES_PROJECT_INDEX); + // const docType = _.get(req, 'body.param.docType', ES_PROJECT_TYPE); + logger.debug('indexName', indexName); + // logger.debug('docType', docType); + + const esClient = util.getElasticSearchClient(); + esClient.indices.delete({ + index: indexName, + // we would want to ignore no such index error + ignore: [404], + }); + res.status(200).json(util.wrapResponse(req.id, { message: 'Delete index request successfully submitted' })); + }, +]; diff --git a/src/routes/admin/project-index-create.js b/src/routes/admin/project-index-create.js new file mode 100644 index 00000000..96e4757e --- /dev/null +++ b/src/routes/admin/project-index-create.js @@ -0,0 +1,122 @@ + +/* globals Promise */ + +import _ from 'lodash'; +import config from 'config'; +import Promise from 'bluebird'; +import { middleware as tcMiddleware } from 'tc-core-library-js'; +import models from '../../models'; +import util from '../../util'; + +/** +/** + * API to handle retrieving a single project by id + * + * Permissions: + * Only users that have access to the project can retrieve it. + * + */ + +// var permissions = require('tc-core-library-js').middleware.permissions +const permissions = tcMiddleware.permissions; +const PROJECT_ATTRIBUTES = _.without(_.keys(models.Project.rawAttributes), 'utm', 'deletedAt'); +const PROJECT_MEMBER_ATTRIBUTES = _.without(_.keys(models.ProjectMember.rawAttributes), 'deletedAt'); +const ES_PROJECT_INDEX = config.get('elasticsearchConfig.indexName'); +const ES_PROJECT_TYPE = config.get('elasticsearchConfig.docType'); + +module.exports = [ + permissions('project.admin'), + /** + * GET projects/{projectId} + * Get a project by id + */ + (req, res, next) => { + const logger = req.log; + logger.debug('Entered Admin#index'); + const projectIdStart = Number(req.body.param.projectIdStart); + const projectIdEnd = Number(req.body.param.projectIdEnd); + const indexName = _.get(req, 'body.param.indexName', ES_PROJECT_INDEX); + const docType = _.get(req, 'body.param.docType', ES_PROJECT_TYPE); + logger.debug('projectIdStart', projectIdStart); + logger.debug('projectIdEnd', projectIdEnd); + logger.debug('indexName', indexName); + logger.debug('docType', docType); + let fields = req.query.fields; + fields = fields ? fields.split(',') : []; + // parse the fields string to determine what fields are to be returned + fields = util.parseFields(fields, { + projects: PROJECT_ATTRIBUTES, + project_members: PROJECT_MEMBER_ATTRIBUTES, + }); + + const eClient = util.getElasticSearchClient(); + return models.Project.findProjectRange(projectIdStart, projectIdEnd, fields) + .then((_projects) => { + const projects = _projects.map((_project) => { + const project = _project; + if (!project) { + return Promise.resolve(null); + } + return models.ProjectMember.getActiveProjectMembers(project.id) + .then((currentProjectMembers) => { + // check context for project members + project.members = _.map(currentProjectMembers, m => _.pick(m, fields.project_members)); + + const userIds = project.members ? project.members.map(single => `userId:${single.userId}`) : []; + return util.getMemberDetailsByUserIds(userIds, logger, req.id) + .then((memberDetails) => { + // update project member record with details + project.members = project.members.map((single) => { + const detail = _.find(memberDetails, md => md.userId === single.userId); + return _.merge(single, _.pick(detail, 'handle', 'firstName', 'lastName', 'email')); + }); + return Promise.delay(1000).return(project); + }) + .catch((error) => { + logger.error(`Error in getting project member details for (projectId: ${project.id})`, error); + return null; + }); + }) + .catch((error) => { + logger.error(`Error in getting project active members (projectId: ${project.id})`, error); + return null; + }); + }); + Promise.all(projects).then((projectResponses) => { + const body = []; + projectResponses.map((p) => { + if (p) { + body.push({ index: { _index: indexName, _type: docType, _id: p.id } }); + body.push(p); + } + // dummy return + return p; + }); + logger.debug('body.length', body.length); + if (body.length > 0) { + logger.trace('body[0]', body[0]); + logger.trace('body[length-1]', body[body.length - 1]); + } + + res.status(200).json(util.wrapResponse(req.id, { + message: `Reindex request successfully submitted for ${body.length / 2} projects`, + })); + // bulk index + eClient.bulk({ + body, + }) + .then((result) => { + logger.debug(`project indexed successfully (projectId: ${projectIdStart}-${projectIdEnd})`, result); + }) + .catch((error) => { + logger.error(`Error in indexing project (projectId: ${projectIdStart}-${projectIdEnd})`, error); + }); + }).catch((error) => { + logger.error( + `Error in getting project details for indexing (projectId: ${projectIdStart}-${projectIdEnd})`, + error); + }); + }) + .catch(err => next(err)); + }, +]; diff --git a/src/routes/admin/project-index-delete.js b/src/routes/admin/project-index-delete.js new file mode 100644 index 00000000..2564aca8 --- /dev/null +++ b/src/routes/admin/project-index-delete.js @@ -0,0 +1,86 @@ + +/* globals Promise */ + +import _ from 'lodash'; +import config from 'config'; +import { middleware as tcMiddleware } from 'tc-core-library-js'; +import models from '../../models'; +import util from '../../util'; + +/** +/** + * API to handle retrieving a single project by id + * + * Permissions: + * Only users that have access to the project can retrieve it. + * + */ + +// var permissions = require('tc-core-library-js').middleware.permissions +const permissions = tcMiddleware.permissions; +const PROJECT_ATTRIBUTES = _.without(_.keys(models.Project.rawAttributes), 'utm', 'deletedAt'); +const PROJECT_MEMBER_ATTRIBUTES = _.without(_.keys(models.ProjectMember.rawAttributes), 'deletedAt'); +const ES_PROJECT_INDEX = config.get('elasticsearchConfig.indexName'); +const ES_PROJECT_TYPE = config.get('elasticsearchConfig.docType'); + +module.exports = [ + permissions('project.admin'), + /** + * GET projects/{projectId} + * Get a project by id + */ + (req, res, next) => { + const logger = req.log; + logger.debug('Entered Admin#deleteIndex'); + const projectIdStart = Number(req.body.param.projectIdStart); + const projectIdEnd = Number(req.body.param.projectIdEnd); + const indexName = _.get(req, 'body.param.indexName', ES_PROJECT_INDEX); + const docType = _.get(req, 'body.param.docType', ES_PROJECT_TYPE); + logger.debug('projectIdStart', projectIdStart); + logger.debug('projectIdEnd', projectIdEnd); + logger.debug('indexName', indexName); + logger.debug('docType', docType); + let fields = req.query.fields; + fields = fields ? fields.split(',') : []; + // parse the fields string to determine what fields are to be returned + fields = util.parseFields(fields, { + projects: PROJECT_ATTRIBUTES, + project_members: PROJECT_MEMBER_ATTRIBUTES, + }); + + const eClient = util.getElasticSearchClient(); + return models.Project.findProjectRange(projectIdStart, projectIdEnd, fields) + .then((_projects) => { + const projects = _projects.map((_project) => { + const project = _project; + if (!project) { + return Promise.resolve(null); + } + return Promise.resolve(project); + }); + const body = []; + Promise.all(projects).then((projectResponses) => { + projectResponses.map((p) => { + if (p) { + body.push({ delete: { _index: indexName, _type: docType, _id: p.id } }); + } + // dummy return + return p; + }); + + // bulk delete + eClient.bulk({ + body, + }) + .then((result) => { + logger.debug(`project index deleted successfully (projectId: ${projectIdStart}-${projectIdEnd})`, result); + }) + .catch((error) => { + logger.error(`Error in deleting indexes for project (projectId: ${projectIdStart}-${projectIdEnd})`, error); + }); + res.status(200).json(util.wrapResponse(req.id, { message: 'Delete index request successfully submitted' })); + }); + }) + .catch(err => next(err)); + }, +]; diff --git a/src/routes/index.js b/src/routes/index.js index f29c2872..16018aef 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -33,6 +33,15 @@ router.route('/v4/projects') router.route('/v4/projects/db') .get(require('./projects/list-db')); +router.route('/v4/projects/admin/es/project/createIndex') + .post(require('./admin/project-create-index')); +router.route('/v4/projects/admin/es/project/deleteIndex') + .delete(require('./admin/project-delete-index')); +router.route('/v4/projects/admin/es/project/index') + .post(require('./admin/project-index-create')); +router.route('/v4/projects/admin/es/project/remove') + .delete(require('./admin/project-index-delete')); + router.route('/v4/projects/:projectId(\\d+)') .get(require('./projects/get')) .patch(require('./projects/update'))