filip/lib/snapshot-send.js

Manage differential snapshots for zfs pools

var pg = require('pg');
var path = require('path');
var queries = require('./queries');
var spawn = require('child_process').spawn;
var winston = require('winston');
var aws = require('aws-sdk');
var async = require('async');

module.exports = function(opts, callback) {
  return new Send(opts, callback);
};

function Send(opts, callback) {
  this.opts = opts;
  this.callback = callback;
  this.snapshotsSynced = 0;

  this.initUploader();
  this.initLogger();

  this.log.info('snapshot-send: pool=%s, status=starting.', this.opts['pool-uuid'], {
    type: 'snapshot',
    action: 'send',
    status: 'starting',
    pool_uuid: this.opts['pool-uuid']
  });

  this.initDatabase();
}

Send.prototype.initUploader = function() {
  var conf = {
    accessKeyId: this.opts['aws-key'],
    secretAccessKey: this.opts['aws-secret']
  };

  if(this.opts['aws-region'])
    conf.region = this.opts['aws-region'];

  this.s3 = new aws.S3(conf);
};

Send.prototype.initLogger = function() {
  this.log = new winston.Logger();

  if(this.opts['log-file-path']) {
    var file_opts = {filename: this.opts['log-file-path']};

    if(this.opts['log-file-size'])
      file_opts.maxsize = (this.opts['log-file-size']*1024*1024);

    if(this.opts['log-file-limit'])
      file_opts.maxFiles = this.opts['log-file-limit'];

    this.log.add(winston.transports.File, file_opts);
  }
};

Send.prototype.initDatabase = function() {
  this.db = new pg.Client(this.opts['pg-conn']);
  this.db.connect(this.handleInitDatabase.bind(this));
};

Send.prototype.handleInitDatabase = function(err) {
  if(err) return this.handleError(err);
  return this.findSnapshots();
};

Send.prototype.findSnapshots = function() {
  var query = this.opts['snapshot-uuid']
    ? queries.sendSnapshot
    : queries.sendSnapshots;

  var params = this.opts['snapshot-uuid']
   ? [this.opts['pool-uuid'], this.opts['snapshot-uuid']]
   : [this.opts['pool-uuid']];

  var handle = this.handleFindSnapshots.bind(this);
  this.db.query(query, params, handle);
};

Send.prototype.handleFindSnapshots = function(err, result) {
  if(err || result.rowCount < 1) {
    if(!err) err = new Error("Could not find snapshots to sync.");
    return this.handleError(err);
  }

  var log = 'snapshot-send: pool=%s, status=found, count=%d';
  this.log.info(log, this.opts['pool-uuid'], result.rows.length, {
    type: 'snapshot',
    action: 'send',
    status: 'found',
    pool_uuid: this.opts['pool-uuid'],
    count: result.rows.length
  });

  var fn = this.sync.bind(this);
  var handle = this.handleSendComplete.bind(this);
  async.eachSeries(result.rows, fn, handle);
};

Send.prototype.handleSendComplete = function(err) {
  if(err) return this.handleError(err);
  return this.handleComplete();
};

Send.prototype.sync = function(snapshot, done) {
  var gpg = this.spawnGpg();
  var lzop = this.spawnLzop();
  var key = [this.opts['pool-uuid'],snapshot['uuid']].join('/');
  var handle = this.handleSyncComplete.bind(this, done, snapshot);
  var zfs = this.spawnZfs(snapshot.pool, snapshot.uuid, snapshot.parent_uuid);

  var log = 'snapshot-send: pool=%s, status=sync, snapshot=%s';
  this.log.info(log, this.opts['pool-uuid'], snapshot.uuid, {
    type: 'snapshot',
    action: 'sync',
    status: 'start',
    pool_uuid: this.opts['pool-uuid'],
    snapshot_uuid: snapshot.uuid
  });

  snapshot.synced_bytes = 0;

  zfs.stdout.pipe(lzop.stdin);
  if(gpg) lzop.stdout.pipe(gpg.stdin);

  var size = parseInt(this.opts['s3-part-size'], 10) || (5*1024*1024);
  var concurrency = parseInt(this.opts['s3-part-concurrency'], 10) || 1;

  this.s3.upload({
    Bucket: this.opts['s3-bucket'],
    Key: key,
    Body: gpg ? gpg.stdout : lzop.stdout
  },{
    partSize: size,
    queueSize: concurrency
  }).on('httpUploadProgress', function(evt) {
    snapshot.synced_bytes = evt.loaded;
  }).send(function(err, data) {
    if(err) return done(err);
    return handle(data);
  });

};

Send.prototype.handleSyncComplete = function(done, snapshot, result) {

  var log = 'snapshot-send: pool=%s, status=synced, snapshot=%s.';
  this.log.info(log, this.opts['pool-uuid'], snapshot.uuid, {
    type: 'snapshot',
    action: 'sync',
    status: 'synced',
    pool_uuid: this.opts['pool-uuid'],
    snapshot_uuid: snapshot.uuid
  });

  this.snapshotsSynced++;
  var params = [snapshot.synced_bytes, this.opts['pool-uuid'], snapshot.uuid];
  this.db.query(queries.syncedSnapshot, params, done);
};

Send.prototype.spawnZfs = function(pool, snapshot_uuid, parent_uuid) {
  var bin = this.opts['zfs-bin'];
  var snapshot = [pool, '@', snapshot_uuid].join('');

  var args = parent_uuid
    ? ['send', '-i', parent_uuid, snapshot]
    : ['send', snapshot];

  if(this.opts['zfs-sudo']) {
    bin = 'sudo';
    args.unshift(this.opts['zfs-bin']);
  }

  return spawn(bin, args);
};

Send.prototype.spawnLzop = function() {
  return spawn(this.opts['lzop-bin'], ['-c']);
};

Send.prototype.spawnGpg = function() {
  if(!this.opts['gpg-id']) return false;
  var args = ['-e', '-z', '0', '-r', this.opts['gpg-id'], '-'];
  return spawn(this.opts['gpg-bin'], args);
};

Send.prototype.handleComplete = function() {
  this.handleDbEnd();

  var log = 'snapshot-send: pool=%s, status=complete, count=%d';
  var handle = this.callback.bind(undefined, null);

  this.log.info(log, this.opts['pool-uuid'], this.snapshotsSynced, {
    type: 'snapshot',
    action: 'send',
    status: 'complete',
    pool_uuid: this.opts['pool-uuid']
  }, handle);
};

Send.prototype.handleDbEnd = function() {
  if(this.db && this.db.end) this.db.end();
};

Send.prototype.handleError = function(err) {
  this.handleDbEnd();

  var log = 'snapshot-send: pool=%s, status=error';
  var handle = this.callback.bind(undefined, err);

  this.log.error(log, this.opts['pool-uuid'], {
    type: 'snapshot',
    action: 'send',
    status: 'error',
    error: err.message,
    pool_uuid: this.opts['pool-uuid']
  }, handle);
};