@escueladigital/micro/src/load.js

Microservice manager for nodejs with kafka

/* eslint-disable no-unused-vars */
/* eslint-disable no-shadow */
const Boom = require('@hapi/boom')
const path = require('path')
const includeAll = require('include-all')
const _ = require('lodash')
const chalk = require('chalk')
const mongoose = require('mongoose')
const Router = require('@koa/router')
const log = require('./logger').logger
const Reporter = require('./logger').Reporter

const { name } = require(`${process.cwd()}/package.json`)
const logger = log()

const upMachineDate = new Date()

const NotImplemented = Boom.notImplemented
const MethodNotAllowed = Boom.methodNotAllowed

const init = {
  controllers: {},
  routes: {},
  modules: {},
  services: {},
  models: {},
}

const schemaOptions = {
  toObject: {
    virtuals: true,
  },
  toJSON: {
    virtuals: true,
  },
  timestamps: true,
}

/**
 * controllers - Generate the controllers
 */
const controllers = () =>
  new Promise((resolve, reject) => {
    includeAll.optional(
      {
        dirname: path.resolve(`${process.cwd()}/api/controllers`),
        filter: /(.+)\.js$/,
        identity: false,
        useGlobalIdForKeyName: false,
      },
      (e, res) => {
        if (e) {
          Reporter.report(e)
          return reject(e)
        }
        return resolve(res)
      }
    )
  })

/**
 * modelsSchemas - Generates the model schemas for the api.
 */
const modelsSchemas = () =>
  new Promise((resolve, reject) => {
    includeAll.optional(
      {
        dirname: path.resolve(`${process.cwd()}/api/schemas`),
        filter: /(.+)\.js/,
        identity: false,
        useGlobalIdForKeyName: false,
      },
      (e, res) => {
        if (e) {
          Reporter.report(e)
          return reject(e)
        }
        return resolve(res)
      }
    )
  })

/**
 * modelsClass - Generate the models.
 */
const modelsClass = () =>
  new Promise((resolve, reject) => {
    includeAll.optional(
      {
        dirname: path.resolve(`${process.cwd()}/api/models`),
        filter: /(.+)\.js$/,
        identity: false,
        useGlobalIdForKeyName: false,
      },
      (e, res) => {
        if (e) {
          Reporter.report(e)
          return reject(e)
        }
        return resolve(res)
      }
    )
  })

/**
 *
 * @param {Object} ctx
 * @param {Object} plcs
 */
const validatePolices = async (ctx, plcs) => {
  for (let index = 0; index < plcs.length; index += 1) {
    let fnName = plcs[index]
    fnName = fnName.replace('fw.', '')
    // if (config.polices[fnName]) {

    //   const response = await config.polices[fnName](ctx);
    //   if (!response) return response;
    // }
  }
  return true
}

/**
 *
 * @param {Models} mdls
 * @param {Classes} classes
 * @param {DataSources} datasources
 */
const generateModels = (mdls, classes, datasources) => {
  if (_.isEmpty(mdls)) {
    logger.warn('No hay esquemas')
    return
  }
  const response = {}
  Object.keys(mdls).forEach(nam => {
    const model = mdls[nam]
    const modelName = _.camelCase(nam)

    const modelSchema = new mongoose.Schema(model.properties, schemaOptions)
    response[modelName] = mongoose.model(modelName, modelSchema)

    if (model.softDelete) {
      const modelDepName = `${modelName}DLT`
      const modelDepSchema = model.properties
      Object.keys(modelDepSchema).forEach(obj => {
        if (modelDepSchema[obj].unique) delete modelDepSchema[obj].unique
      })
      response[modelDepName] = datasources(model.datasource).model(modelDepName, modelDepSchema)
    }
  })
  return response
}

/**
 * getRootRoutes - Get the base routes of the API
 */
const getRootRoutes = () => {
  const router = new Router()
  router.get('/', async ctx => {
    ctx.status = 404
    return ctx.status
  })
  router.get('/api', async ctx => {
    ctx.body = { res: 'API running :D' }
    ctx.status = 200
    return ctx.status
  })
  return router
}

/**
 *
 * @param {*} routes
 * @param {*} ctrls
 */
const generateRoutes = (routes, ctrls) => {
  if (_.isEmpty(routes)) {
    logger.warn('No hay rutas')
    return
  }
  const router = new Router()
  Object.keys(routes).forEach(r => {
    const [verb, uri] = r.split(' ')
    const { controller, action: act, polices } = routes[r]

    if (r !== 'identity' && r !== 'globalId') {
      if (!ctrls[controller]) {
        logger.error('Controlador de ruta no encontrado')
        return
      }
      const action = ctrls[controller][act]
      router[_.toLower(verb)](uri, async ctx => {
        try {
          if (polices) await validatePolices(ctx, polices)
          return await action(ctx)
        } catch (ex) {
          ctx.body = new Boom(ex)
          ctx.status = ctx.body.output.statusCode
          if (ctx.body.output.statusCode === 500) logger.error(`Error ${ex}`)
          return ctx
        }
      })
    }
  })
  return router
}

const config = () =>
  new Promise((resolve, reject) => {
    includeAll.optional(
      {
        dirname: path.resolve(`${process.cwd()}/api/config`),
        filter: /(.+)\.js$/,
        identity: false,
        useGlobalIdForKeyName: false,
      },
      (e, res) => {
        if (e) {
          return reject(e)
        }
        return resolve(res)
      }
    )
  })

/**
 * Cargar todos los parasitos para la API
 */
const parasites = () =>
  new Promise((resolve, reject) => {
    includeAll.optional(
      {
        dirname: path.resolve(`${process.cwd()}/api/events`),
        filter: /(.+)\.js$/,
        identity: false,
        useGlobalIdForKeyName: false,
      },
      (e, res) => {
        if (e) {
          Reporter.report(e)
          return reject(e)
        }
        return resolve(_.omit(res, ['helper']))
      }
    )
  })

/**
 * Carga todos los consumidores de la API
 */
const consumers = () =>
  new Promise((resolve, reject) => {
    includeAll.optional(
      {
        dirname: path.resolve(`${process.cwd()}/api/consumer`),
        filter: /(.+)\.js$/,
        identity: false,
        useGlobalIdForKeyName: false,
      },
      (e, res) => {
        if (e) {
          Reporter.report(e)
          return reject(e)
        }
        return resolve(_.omit(res, ['helper']))
      }
    )
  })

const makeConsumers = async consumers => {
  const cons = []
  _.forOwn(consumers, (value, key) => {
    _.forOwn(value, (iv, action) => {
      const req = `${process.env.NODE_ENV}.req.${name}.${key}.${action}`
      const res = `${process.env.NODE_ENV}.res.${name}.${key}.${action}`
      const micro = `${process.env.NODE_ENV}.micro.${name}.${key}.${action}`
      cons.push({
        req,
        res,
        micro,
        action,
        key,
      })
    })
  })
  return cons
}

/**
 *
 * @param {*} topicList - Lista de topicos para hacer de parasito.
 */
const loadParasites = async (parasite, pubsub) => {
  if (_.isEmpty(parasite)) {
    logger.warn('No hay parasitos')
    return
  }
  const elements = Object.keys(parasite)
  // const topics = []
  logger.info(chalk.bgRed(chalk.black('*** CREATING PARASITES ***')))
  for (let i = 0; i < elements.length; i += 1) {
    const micro = elements[i]
    const consumers = Object.keys(parasite[micro])
    for (let j = 0; j < consumers.length; j += 1) {
      const consumer = consumers[j]
      const methods = Object.keys(parasite[micro][consumer])
      for (let z = 0; z < methods.length; z += 1) {
        const method = methods[z]
        const topic = `${process.env.NODE_ENV || 'local'}.res.${micro}.${consumer}.${method}`
        try {
          await pubsub.parasite.subscribe({
            topic,
          })
          logger.info(`${topic} - ${chalk.red('PARASITE')} - subscribed`)
        } catch (e) {
          logger.warn(`${topic} - ${chalk.red('PARASITE')} - already subscribed`)
        }
      }
    }
  }
  const accs = {}
  const messages = []
  await pubsub.parasite.run({
    eachBatchAutoResolve: false,
    eachBatch: async ({ batch, resolveOffset, heartbeat, isStale, isRunning }) => {
      try {
        const topicOffset = await pubsub.admin.fetchTopicOffsets(batch.topic)
        for (let i = 0; i < batch.messages.length; i++) {
          const message = batch.messages[i]
          if (!isRunning() || isStale()) break
          const val = JSON.parse(Buffer.from(message.value).toString())
          const splited = batch.topic.split('.')
          const micro = splited[2]
          const consume = splited[3]
          const method = splited[4]

          await parasite[micro][consume][method](
            val,
            ready => {
              if (!ready) return
              if (Number(message.offset) >= Number(topicOffset[0].offset) - 1) {
                ready(accs[batch.topic])
                delete accs[batch.topic]
              }
            },
            reduce => {
              if (!reduce) return
              reduce(val, accs[batch.topic] || 0, val => (accs[batch.topic] = val))
            }
          )
          await resolveOffset(message.offset)
          await heartbeat()
        }
      } catch (e) {
        Reporter.report(e)
      }
    },
  })
  // await pubsub.parasite.run({
  //   eachMessage: async ({ topic, message }) => {
  // const val = JSON.parse(Buffer.from(message.value).toString())
  // const splited = topic.split('.')
  // const micro = splited[2]
  // const consume = splited[3]
  // const method = splited[4]
  // try {
  //   await parasite[micro][consume][method](val, ready)
  // } catch (e) {
  //   Reporter.report(e)
  // }
  //   },
  // })
  logger.info(chalk.bgRed(chalk.black('*** CREATING PARASITES - END ***')))
}

const loadConsumers = async (consumer, pubsub, gPubsub, app, communication) => {
  if (_.isEmpty(consumer)) {
    logger.warn('No hay consumidores')
    return
  }
  if (!pubsub) {
    logger.warn('No pubsub')
    return
  }

  try {
    console.log(communication)
    const microTopicsT = await gPubsub.getMicroTopics(communication)
    console.log(microTopicsT)

    const reqTopics = pubsub.map(topic => ({ topic: topic.req }))
    const microTopics = microTopicsT.map(topic => ({ topic: topic.name }))
    console.log(JSON.stringify(microTopics))
    const microConsumer = gPubsub.kafka.consumer({
      groupId: `${name}-consumer-m-Micro-${process.env.NODE_ENV}`,
    })
    await microConsumer.connect()
    const cleanMicro = microTopics.filter(topic => topic.topic.includes(`${process.env.NODE_ENV}.`))
    const cleanReq = reqTopics.filter(topic => topic.topic.includes(`${process.env.NODE_ENV}.`))

    console.log(JSON.stringify(cleanMicro))

    logger.info(`${chalk.bgCyan(chalk.black('*** NORMAL SUBSCRIPTIONS ***'))}`)
    for (let i = 0; i < cleanMicro.length; i += 1) {
      const topic = cleanMicro[i]
      try {
        await microConsumer.subscribe(topic)
        logger.info(`${topic.topic} - subscribed`)
      } catch (e) {
        logger.warn(`${topic.topic} - already subscribed`)
      }
    }
    for (let i = 0; i < cleanReq.length; i += 1) {
      const topic = cleanReq[i]
      try {
        await gPubsub.consumer.subscribe(topic)
        logger.info(`${topic.topic} - subscribed`)
      } catch (e) {
        logger.warn(`${topic.topic} - already subscribed`)
      }
    }
    logger.info(`${chalk.bgCyan(chalk.black('*** NORMAL SUBSCRIPTIONS - END ***'))}`)
    await microConsumer.run({
      eachMessage: ({ message }) => {
        if (message.timestamp < upMachineDate) return
        const val = JSON.parse(Buffer.from(message.value).toString())
        const promise = gPubsub.reqResources[val.uuid]

        if (promise) {
          if (!val.data) promise.resolve(val.data)
          else if (val.data.isBoom) {
            promise.reject(val.data.message)
          } else {
            try {
              promise.resolve(val.data)
            } catch (e) {
              promise.reject(e.toString())
            }
          }
          delete gPubsub.reqResources[val.uuid]
        } else {
          logger.error('No subscription promise')
          delete gPubsub.reqResources[val.uuid]
        }
      },
    })

    await gPubsub.consumer.run({
      eachMessage: async ({ topic, partition, message }) => {
        if (message.timestamp < upMachineDate) return
        const val = JSON.parse(Buffer.from(message.value).toString())
        const responseKind = Buffer.from(message.headers.responseKind).toString()

        const splited = topic.split('.')
        const consume = splited[3]
        const method = splited[4]
        const res = `${responseKind}.${name}.${consume}.${method}`

        try {
          const data = await consumer[consume][method](val)
          gPubsub.writeMessageToTopic(
            JSON.stringify({ data, uuid: val.uuid, val: val }),
            res,
            responseKind === 'micro' ? 'resmicro' : 'res'
          )
        } catch (e) {
          Reporter.report(e)
          gPubsub.writeMessageToTopic(
            JSON.stringify({
              data: {
                isBoom: true,
                message: _.get(e, 'message', 'No hay descripción para este error'),
              },
              uuid: val.uuid,
            }),
            res
          )
        }
      },
    })
  } catch (e) {
    Reporter.report(e)
    throw Boom.conflict('Error', e)
  }
}

module.exports = async (app, port) => {
  try {
    app = _.merge(app, init)
    app.controllers = await controllers()
    app.modelsClass = await modelsClass()
    app.modelsSchemas = await modelsSchemas()
    app.consumers = await consumers()
    app.parasites = await parasites()

    if (_.isEmpty(app.consumers)) {
      app.PubSub.parasite.on('consumer.group_join', () => {
        app.listen(port, '0.0.0.0')
        app.logger.info(`Server running on port ${port}`)
        app.logger.info(`ENVIROMENT: ${process.env.NODE_ENV || 'development'}`)
      })
    } else {
      app.PubSub.consumer.on('consumer.group_join', () => {
        app.listen(port, '0.0.0.0')
        app.logger.info(`Server running on port ${port}`)
        app.logger.info(`ENVIROMENT: ${process.env.NODE_ENV || 'development'}`)
      })
    }

    const buildedConsumers = await makeConsumers(app.consumers)
    const { communication, routes } = await config()
    app.routes = routes
    await app.PubSub.setUpProject(buildedConsumers)
    await loadParasites(app.parasites, app.PubSub)
    await loadConsumers(app.consumers, buildedConsumers, app.PubSub, app, communication)
    app.models = generateModels(app.modelsSchemas, app.modelsClass, app.datasources)

    const rootRoutes = getRootRoutes()
    const router = generateRoutes(app.routes, app.controllers)

    app.use(rootRoutes.routes())

    if (router) {
      app.use(router.routes())
      app.use(
        router.allowedMethods({
          throw: true,
          notImplemented: () => new NotImplemented(),
          methodNotAllowed: () => new MethodNotAllowed(),
        })
      )
    }

    app.types = mongoose.Types
  } catch (e) {
    Reporter.report(e)
    logger.warn(e)
  }
}