walter/lib/wal-push.js

Backup utilities for postgresql

var env = process.env;
var fs = require('fs');
var knox = require('knox');
var path = require('path');
var es = require('event-stream');
var buffers = require('buffers');
var optimist = require('optimist');
var spawn = require('child_process').spawn;
var winston = require('winston');

var argv = optimist
  .usage('Push a wal file to s3.\nUsage: walter wal-push [options]')
  .describe('prefix', 'Prefix used to store files on s3 (env: WALTER_PREFIX)')
  .describe('gpg-id', 'Pass a GPG ID to encrypt your backups (env: WALTER_GPG_ID)')
  .describe('aws-key', 'Your AWS Key (env: WALTER_AWS_KEY)')
  .describe('aws-secret', 'Your AWS Secret (env: WALTER_AWS_SECRET)')
  .describe('aws-region', 'AWS region for your bucket (env: WALTER_AWS_REGION)')
  .describe('s3-bucket', 'S3 Bucket Name (env: WALTER_S3_BUCKET)')
  .describe('log-file-path', 'Winston file transport path (env: WALTER_LOG_FILE_PATH)')
  .describe('log-file-size', 'Winston file transport size in MB (env: WALTER_LOG_FILE_SIZE)')
  .describe('log-file-limit', 'Winston file transport rotate limit (env: WALTER_LOG_FILE_LIMIT)')
  .describe('labs-api-key', 'Walter Labs API Key (env: WALTER_LABS_API_KEY)')
  .describe('log-http-host', 'Winston HTTP transport host (env: WALTER_LOG_HTTP_HOST)')
  .describe('log-http-port', 'Winston HTTP transport port (env: WALTER_LOG_HTTP_PORT)')
  .describe('log-http-path', 'Winston HTTP transport path (env: WALTER_LOG_HTTP_PATH)')
  .describe('log-http-ssl', 'Winston HTTP transport ssl (env: WALTER_LOG_HTTP_SSL)')
  .describe('tar-bin', 'Path to tar binary')
  .describe('gpg-bin', 'Path to gpg binary')
  .describe('lzop-bin', 'Path to lzop binary')
  .describe('wal-file', 'Path to wal file')
  .default('prefix', env.WALTER_PREFIX)
  .default('gpg-id', env.WALTER_GPG_ID)
  .default('aws-key', env.WALTER_AWS_KEY)
  .default('aws-secret', env.WALTER_AWS_SECRET)
  .default('s3-bucket', env.WALTER_S3_BUCKET)
  .default('log-file-path', env.WALTER_LOG_FILE_PATH)
  .default('log-file-size', env.WALTER_LOG_FILE_SIZE)
  .default('log-file-limit', env.WALTER_LOG_FILE_LIMIT)
  .default('labs-api-key', env.WALTER_LABS_API_KEY)
  .default('log-http-host', env.WALTER_LOG_HTTP_HOST)
  .default('log-http-port', env.WALTER_LOG_HTTP_PORT)
  .default('log-http-path', env.WALTER_LOG_HTTP_PATH)
  .default('log-http-ssl', env.WALTER_LOG_HTTP_SSL)
  .default('tar-bin', 'tar')
  .default('gpg-bin', 'gpg')
  .default('lzop-bin', 'lzop')
  .demand('prefix')
  .demand('aws-key')
  .demand('aws-secret')
  .demand('s3-bucket')
  .demand('wal-file')
  .wrap(120)
  .argv;

winston.remove(winston.transports.Console);

if(argv['labs-api-key']) {
  argv['log-http-host'] = 'api.walter-labs.com';
  argv['log-http-port'] = 443;
  argv['log-http-path'] = '/servers/' + argv['labs-api-key'];
  argv['log-http-ssl'] = true;
}

if(argv['log-http-host']) {
  var logHttpOpts = {
    host: argv['log-http-host']
  };

  if(argv['log-http-port'])
    logHttpOpts.port = argv['log-http-port'];

  if(argv['log-http-path'])
    logHttpOpts.path = argv['log-http-path'];

  if(argv['log-http-ssl'] === 'true' || argv['log-http-ssl'] === true)
    logHttpOpts.ssl = true;

  winston.add(winston.transports.Http, logHttpOpts);
}

if(argv['log-file-path']) {
  var logFileOpts = {
    filename: argv['log-file-path']
  };

  if(argv['log-file-size'])
    logFileOpts.maxsize = (argv['log-file-size']*1024*1024);

  if(argv['log-file-limit'])
    logFileOpts.maxFiles = argv['log-file-limit'];

  winston.add(winston.transports.File, logFileOpts);
}

var raw = 0;
var compressed = 0;
var started = new Date();
var filename = path.basename(argv['wal-file']);

winston.info('wal-push: %s: %s, started.', argv['prefix'], filename, {
  type: 'wal',
  action: 'started',
  prefix: argv['prefix'],
  filename: filename,
  started: started
});

module.exports = function() {
  var bufs = buffers();
  var gpg = spawnGpg();
  var lzop = spawnLzop();
  var reader = fs.createReadStream(argv['wal-file']);

  var raw_counter = es.through(function(data) {
    raw += data.length;
    this.emit('data', data);
  }, function() {
    this.emit('end');
  });

  var compressed_counter = es.through(function(data) {
    compressed += data.length;
    this.emit('data', data);
  }, function() {
    this.emit('end');
  });

  var bufferer = es.through(function(data) {
    bufs.push(data);
  }, function() {
    this.emit('end');
  });

  var pipeline = gpg
    ? [reader, raw_counter, lzop, compressed_counter, gpg, bufferer]
    : [reader, raw_counter, lzop, compressed_counter, bufferer];

  var pipe = es.pipeline.apply(null, pipeline);
  pipe.on('end', onEnd);

  function onEnd() {
    putWal(bufs.toBuffer());
  }
};

function putWal(buffer) {
  var s3 = createS3Client();
  var wal_path = argv['prefix'] + '/wal/' + filename;

  s3.putBuffer(buffer, wal_path, {
    'Content-Length': buffer.length
  }, onComplete);
}

function onComplete(err, response) {
  if(err || response.statusCode < 200 || response.statusCode > 299) {
    if(!err) err = new Error('Failed upload to S3');
    return handleError(err);
  }

  winston.info('wal-push: %s: %s, completed.',
    argv['prefix'], filename, {
    type: 'wal',
    action: 'completed',
    prefix: argv['prefix'],
    filename: filename,
    size: raw,
    compressed: compressed,
    started: started,
    completed: new Date()
  }, function() {
    process.exit();
  });
}

function createS3Client() {
  var options = {
    key: argv['aws-key'],
    secret: argv['aws-secret'],
    bucket: argv['s3-bucket']
  };

  if(argv['aws-region'])
    options.region = argv['aws-region'];

  return new knox.createClient(options);
}

function spawnLzop() {
  return es.child(spawn(argv['lzop-bin'], ['-c', '-']));
}

function spawnGpg() {
  if(!argv['gpg-id']) return false;
  var args = ['-e', '-z', '0', '-r', argv['gpg-id'], '-'];
  return es.child(spawn(argv['gpg-bin'], args));
}

function handleError(err) {
  winston.error('wal-push: %s: %s, error occured.', argv['prefix'], filename, {
    prefix: argv['prefix'],
    filename: filename,
    error: err.message,
    started: started,
    errored: new Date()
  }, errorExit);
}

function errorExit() {
  process.exit(1);
}