walter/lib/backup-fetch.js

Backup utilities for postgresql

var fs = require('fs');
var pg = require('pg.js');
var env = process.env;
var path = require('path');
var knox = require('knox');
var mkdirp = require('mkdirp').sync;
var moment = require('moment');
var es = require('event-stream');
var optimist = require('optimist');
var spawn = require('child_process').spawn;

var argv = optimist
  .usage('Fetch a base backup from s3.\nUsage: walter backup-fetch [options]')
  .describe('source', 'Name of base backup to fetch')
  .describe('destination', 'Path to extract base backup')
  .describe('prefix', 'Prefix used to store files on s3 (env: WALTER_PREFIX)')
  .describe('gpg-id', 'GPG ID to encrypt your backups (env: WALTER_GPG_ID)')
  .describe('aws-key', 'AWS Key (env: WALTER_AWS_KEY)')
  .describe('aws-secret', 'AWS Secret (env: WALTER_AWS_SECRET)')
  .describe('aws-region', 'AWS region for your bucket (env: WALTER_AWS_REGION)')
  .describe('s3-bucket', 'S3 Bucket Name (env: WALTER_S3_BUCKET)')
  .describe('tar-bin', 'Path to tar binary')
  .describe('gpg-bin', 'Path to gpg binary')
  .describe('lzop-bin', 'Path to lzop binary')
  .default('prefix', env.WALTER_PREFIX)
  .default('gpg-id', env.WALTER_GPG_ID)
  .default('aws-key', env.WALTER_AWS_KEY)
  .default('aws-secret', env.WALTER_AWS_SECRET)
  .default('s3-bucket', env.WALTER_S3_BUCKET)
  .default('tar-bin', 'tar')
  .default('gpg-bin', 'gpg')
  .default('lzop-bin', 'lzop')
  .demand('source')
  .demand('destination')
  .demand('prefix')
  .demand('aws-key')
  .demand('aws-secret')
  .demand('s3-bucket')
  .wrap(120)
  .argv;

var destination = path.resolve(argv['destination']);
var pid = path.resolve(destination, 'postmaster.pid');
var source = argv['prefix'] + '/base/' + argv['source'];

module.exports = function() {
  var destExists = fs.existsSync(destination);
  var pidExists = fs.existsSync(pid);

  if(!destExists) mkdirp(destination);

  if(destExists && pidExists)
    return handleError(new Error('Postgres must not be running.'));

  var s3 = createS3Client();
  s3.get(source).on('response', onFetch).end();
};

function onFetch(res) {
  if(res.statusCode < 200 || res.statusCode > 299)
    return handleError(new Error('Error fetching data from s3.'));

  var tar = spawnTar();
  var lzop = spawnLzop();
  var gpg = spawnGpg();

  var pipeline = gpg
    ? [res, gpg, lzop, tar.stdin]
    : [res, lzop, tar.stdin];

  var pipe = es.pipeline.apply(null, pipeline);
  pipe.on('error', handleError);
  tar.on('exit', onTarExit);

  function onTarExit(code) {
    if(code !== 0) return handleError(new Error('Tar did not exit cleanly'));
    console.log('Base backup', source, 'successfully extracted to', destination);
    process.exit();
  }
}

function createS3Client() {
  var options = {
    key: argv['aws-key'],
    secret: argv['aws-secret'],
    bucket: argv['s3-bucket']
  };

  if(argv['aws-region'])
    options.region = argv['aws-region'];

  return new knox.createClient(options);
}

function spawnTar() {
  var args = [
    '-C', destination,
    '-xf', '-'
  ];
  return spawn(argv['tar-bin'], args);
}

function spawnLzop() {
  return es.child(spawn(argv['lzop-bin'], ['-cd']));
}

function spawnGpg() {
  if(!argv['gpg-id']) return false;
  var args = ['-d', '-r', argv['gpg-id']];
  return es.child(spawn(argv['gpg-bin'], args));
}

function handleError(err) {
  throw err;
}