walter/lib/wal-fetch.js

Backup utilities for postgresql

var fs = require('fs');
var pg = require('pg.js');
var env = process.env;
var path = require('path');
var knox = require('knox');
var mkdirp = require('mkdirp').sync;
var moment = require('moment');
var es = require('event-stream');
var optimist = require('optimist');
var spawn = require('child_process').spawn;

var argv = optimist
  .usage('Fetch a wal file from s3.\nUsage: walter wal-fetch [options]')
  .describe('prefix', 'Prefix used to store files on s3 (env: WALTER_PREFIX)')
  .describe('gpg-id', 'GPG ID to encrypt your backups (env: WALTER_GPG_ID)')
  .describe('aws-key', 'AWS Key (env: WALTER_AWS_KEY)')
  .describe('aws-secret', 'AWS Secret (env: WALTER_AWS_SECRET)')
  .describe('aws-region', 'AWS region for your bucket (env: WALTER_AWS_REGION)')
  .describe('s3-bucket', 'S3 Bucket Name (env: WALTER_S3_BUCKET)')
  .describe('gpg-bin', 'Path to gpg binary')
  .describe('lzop-bin', 'Path to lzop binary')
  .describe('wal-name', 'Name of wal file to fetch')
  .describe('wal-file', 'Path to extract wal file')
  .string('wal-name')
  .string('wal-file')
  .default('prefix', env.WALTER_PREFIX)
  .default('gpg-id', env.WALTER_GPG_ID)
  .default('aws-key', env.WALTER_AWS_KEY)
  .default('aws-secret', env.WALTER_AWS_SECRET)
  .default('s3-bucket', env.WALTER_S3_BUCKET)
  .default('gpg-bin', 'gpg')
  .default('lzop-bin', 'lzop')
  .demand('wal-name')
  .demand('wal-file')
  .demand('prefix')
  .demand('aws-key')
  .demand('aws-secret')
  .demand('s3-bucket')
  .wrap(120)
  .argv;

var source = argv['prefix'] + '/wal/' + argv['wal-name'];
module.exports = function() {
  var s3 = createS3Client();
  s3.get(source).on('response', onFetch).end();
};

function onFetch(res) {
  if(res.statusCode < 200 || res.statusCode > 299)
    return handleError(new Error('Error fetching data from s3.'));

  var lzop = spawnLzop();
  var gpg = spawnGpg();
  var ws = fs.createWriteStream(argv['wal-file']);

  var pipeline = gpg
    ? [res, gpg, lzop, ws]
    : [res, lzop, ws];

  var pipe = es.pipeline.apply(null, pipeline);
  pipe.on('error', handleError);
  ws.on('end', onEnd);

  function onEnd() {
    console.log('Wal file', source, 'successfully extracted to', argv['wal-file']);
    process.exit();
  }
}

function createS3Client() {
  var options = {
    key: argv['aws-key'],
    secret: argv['aws-secret'],
    bucket: argv['s3-bucket']
  };

  if(argv['aws-region'])
    options.region = argv['aws-region'];

  return new knox.createClient(options);
}

function spawnLzop() {
  return es.child(spawn(argv['lzop-bin'], ['-cd']));
}

function spawnGpg() {
  if(!argv['gpg-id']) return false;
  var args = ['-d', '-r', argv['gpg-id']];
  return es.child(spawn(argv['gpg-bin'], args));
}

function handleError(err) {
  console.error(err);
  process.exit(1);
}