log.io-ng/lib/harvester.js

Realtime log monitoring in the browser

// Generated by CoffeeScript 1.10.0

/*
	 * Log.io Log Harvester #

	Watches local files and sends new log message to server via TCP.

	 * Sample configuration:
	config =
		nodeName: 'my_server01'
		logStreams:
			web_server: [
				'/var/log/nginx/access.log',
				'/var/log/nginx/error.log'
			],
		server:
			host: '0.0.0.0',
			port: 28777

	 * Sends the following TCP messages to the server:
	"+node|my_server01|web_server\r\n"
	"+bind|node|my_server01\r\n"
	"+log|web_server|my_server01|info|this is log messages\r\n"

	 * Usage:
	harvester = new LogHarvester config
	harvester.run()
 */

(function() {
  var LogHarvester, LogStream, events, fs, net, winston,
    extend = function(child, parent) { for (var key in parent) { if (hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; },
    hasProp = {}.hasOwnProperty,
    bind = function(fn, me){ return function(){ return fn.apply(me, arguments); }; },
    slice = [].slice;

  fs = require('fs');

  net = require('net');

  events = require('events');

  winston = require('winston');


  /*
  	LogStream is a group of local files paths.  It watches each file for
  	changes, extracts new log messages, and emits 'new_log' events.
   */

  LogStream = (function(superClass) {
    extend(LogStream, superClass);

    function LogStream(name, paths1, debug) {
      this.name = name;
      this.paths = paths1;
      this.debug = debug;
    }

    LogStream.prototype.watch = function() {
      var i, len, path, ref;
      this.debug.info("Starting log stream: '" + this.name + "'");
      ref = this.paths;
      for (i = 0, len = ref.length; i < len; i++) {
        path = ref[i];
        this._watchFile(path);
      }
      return this;
    };

    LogStream.prototype._watchFile = function(path) {
      var currSize, watcher;
      if (!fs.existsSync(path)) {
        this.debug.error("File doesn't exist: '" + path + "'");
        setTimeout(((function(_this) {
          return function() {
            return _this._watchFile(path);
          };
        })(this)), 1000);
        return;
      }
      this.debug.info("Watching file: '" + path + "'");
      currSize = fs.statSync(path).size;
      return watcher = fs.watch(path, (function(_this) {
        return function(event, filename) {
          if (event === 'rename') {
            watcher.close();
            _this._watchFile(path);
          }
          if (event === 'change') {
            return fs.stat(path, function(err, stat) {
              _this._readNewLogs(path, stat.size, currSize);
              return currSize = stat.size;
            });
          }
        };
      })(this));
    };

    LogStream.prototype._readNewLogs = function(path, curr, prev) {
      var rstream;
      if (curr < prev) {
        return;
      }
      rstream = fs.createReadStream(path, {
        encoding: 'utf8',
        start: prev,
        end: curr
      });
      return rstream.on('data', (function(_this) {
        return function(data) {
          var i, len, line, lines, results;
          lines = data.split("\n");
          results = [];
          for (i = 0, len = lines.length; i < len; i++) {
            line = lines[i];
            if (line) {
              results.push(_this.emit('new_log', line));
            }
          }
          return results;
        };
      })(this));
    };

    return LogStream;

  })(events.EventEmitter);


  /*
  	LogHarvester creates LogStreams and opens a persistent TCP connection to the server.
  
  	On startup it announces itself as Node with Stream associations.
  	Log messages are sent to the server via string-delimited TCP messages
  
  	- NOTE -
  	Support for 'nodeName' and 'logStreams' is deprecated in
  	the process of simplifying the API alltogether. They will
  	be replaced by respectively 'name' and 'streams'.
  
  	LogHarvester
  		name: "appserver"
  		streams: ['errors', 'registrations']
   */

  LogHarvester = (function() {
    function LogHarvester(config) {
      this._reconnect = bind(this._reconnect, this);
      this._connect = bind(this._connect, this);
      this._commit = bind(this._commit, this);
      this.send = bind(this.send, this);
      var paths, ref, ref1, stream;
      this.name = config.name, this.server = config.server;
      if ((this.name == null) && (config.nodeName != null)) {
        this.name = config.nodeName;
      }
      if (this.server == null) {
        this.server = {
          host: '127.0.0.1',
          port: 28777
        };
      }
      this.queue = [];
      this.delimiter = (ref = config.delimiter) != null ? ref : '\r\n';
      this.debug = (ref1 = config.logging) != null ? ref1 : winston;
      this.streams = (function() {
        var ref2, ref3, ref4, results;
        ref4 = (ref2 = config.streams) != null ? ref2 : (ref3 = config.logStreams) != null ? ref3 : [];
        results = [];
        for (stream in ref4) {
          paths = ref4[stream];
          results.push(new LogStream(stream, paths, this.debug));
        }
        return results;
      }).call(this);
    }

    LogHarvester.prototype.run = function() {
      this._connect((function(_this) {
        return function(err) {
          return _this._announce();
        };
      })(this));
      return this.streams.forEach((function(_this) {
        return function(stream) {
          return stream.watch().on('new_log', function(msg) {
            if (!_this.instance) {
              return;
            }
            _this.debug.debug("Sending log: (" + stream.name + ") " + msg);
            return _this.send('+log', stream.name, _this.name, 'info', msg);
          });
        };
      })(this));
    };

    LogHarvester.prototype._announce = function() {
      var l, streamList;
      streamList = ((function() {
        var i, len, ref, results;
        ref = this.streams;
        results = [];
        for (i = 0, len = ref.length; i < len; i++) {
          l = ref[i];
          results.push(l.name);
        }
        return results;
      }).call(this)).join(",");
      this.debug.info("Announcing: " + this.name + " (" + streamList + ")");
      this.send('+node', this.name, streamList);
      return this.send('+bind', 'node', this.name);
    };

    LogHarvester.prototype.send = function() {
      var args, msg, type;
      type = arguments[0], args = 2 <= arguments.length ? slice.call(arguments, 1) : [];
      msg = type + "|" + (args.join('|')) + this.delimiter;
      msg = msg.trim();
      return this._connect((function(_this) {
        return function(err) {
          _this.queue.push(msg);
          if (err == null) {
            return _this._commit();
          }
        };
      })(this));
    };


    /*
    		Below is based on https://gist.github.com/KenanSulayman/f7e55a28df614c520576.
     */

    LogHarvester.prototype._commit = function() {
      var _queue, item, line;
      if (!this.instance) {
        return;
      }
      _queue = (function() {
        var i, len, ref, results;
        ref = this.queue;
        results = [];
        for (i = 0, len = ref.length; i < len; i++) {
          item = ref[i];
          results.push(this.queue.shift());
        }
        return results;
      }).call(this);
      line = (_queue.join('\r\n')) + "\r\n";
      if (_queue.length) {
        return this.instance.write(line);
      }
    };

    LogHarvester.prototype._connect = function(cb) {
      if (this.instance != null) {
        if (this.instance.readyState === 'open') {
          return cb(null);
        } else {
          return cb(true);
        }
      }
      this.instance = net.connect({
        host: this.server.host,
        port: this.server.port
      });
      this.instance.setKeepAlive(true);
      this.instance.setNoDelay();
      return this.instance.on('connect', (function(_this) {
        return function() {
          _this._commit();
          _this.queue = [];
          _this.retries = 0;
          _this.connected = true;
          return cb(null);
        };
      })(this)).on('error', function(e) {
        return console.log(e);
      }).on('end', (function() {})).on('close', (function(_this) {
        return function() {
          return _this._reconnect(cb);
        };
      })(this)).on('timeout', (function(_this) {
        return function(e) {
          if (_this.instance.readyState !== 'open') {
            _this.instance.destroy();
            return _this._reconnect(cb);
          }
        };
      })(this));
    };

    LogHarvester.prototype._reconnect = function(cb) {
      var interval;
      interval = Math.pow(2, this.retries);
      this.connected = false;
      return setTimeout((function(_this) {
        return function() {
          _this.retries += 1;
          return _this._connect(cb);
        };
      })(this), interval * 1000);
    };

    return LogHarvester;

  })();

  exports.LogHarvester = LogHarvester;

}).call(this);