@taotiejs/server/lib/log-receiver.js

Log collecting and querying based on pinojs and clickhouse

const dgram = require('dgram');
const snakeCase = require('lodash.snakecase');
const createTable = require('./clickhouse-create');

const receiver = (port, msgHandler) => new Promise((resolve) => {
  const socket = dgram.createSocket('udp4');
  socket.on('message', msg => msgHandler(msg.toString()));
  socket.on('error', err => console.error('log receiver error', err.message));
  socket.on('listening', () => console.info('log receiver listening on', port));
  socket.bind(port, () => resolve(socket));
});

module.exports = (port, db, interval, dataSkippingIndices) => {
  const projects = {};
  receiver(port, (data) => {
    try {
      const log = JSON.parse(data);
      const {
        time = Date.now(),
        level = 0,
        hostname = '',
        project,
        module: mod = '',
        msg = '',
      } = log;
      if (!project) {
        console.error('project not specific', data);
        return;
      }

      delete log.project;
      delete log.module;
      delete log.time;
      delete log.level;
      delete log.hostname;
      delete log.msg;
      delete log.pid;
      delete log.v;

      const tableName = snakeCase(project);
      const logs = projects[tableName];
      const record = {
        timestamp: time,
        level,
        time: Math.round(time / 1000),
        hostname,
        module: mod,
        message: `${msg}`,
        detail: Object.keys(log).length ? JSON.stringify(log) : '',
      };
      if (!logs) {
        projects[tableName] = [record];
        createTable(tableName, dataSkippingIndices)
          .then(() => {}, err => console.error('create table failed', err));
      } else {
        logs.push(record);
      }
    } catch (err) {
      console.error('receive log failed', err);
    }
  });
  setInterval(async () => {
    for (const p of Object.keys(projects)) {
      const logs = projects[p];
      const { length } = logs;
      if (length) {
        try {
          // eslint-disable-next-line no-await-in-loop
          await db.insert(p, logs.slice(0, length));
        } catch (err) {
          console.error('save log failed', err, logs.slice(0, length));
        }
        logs.splice(0, length);
      }
    }
  }, interval * 1000);
};