walter/lib/verify.js

Backup utilities for postgresql

var env = process.env;
var fs = require('fs');
var knox = require('knox');
var path = require('path');
var moment = require('moment');
var buffers = require('buffers');
var optimist = require('optimist');
var spawn = require('child_process').spawn;
var textable = require('text-table');
var filesize = require('humanize').filesize;
var winston = require('winston');

var argv = optimist
  .usage('Verify files stored on S3.\nUsage: walter verify')
  .describe('prefix', 'Prefix used to store files on s3 (env: WALTER_PREFIX)')
  .describe('aws-key', 'Your AWS Key (env: WALTER_AWS_KEY)')
  .describe('aws-secret', 'Your AWS Secret (env: WALTER_AWS_SECRET)')
  .describe('aws-region', 'AWS region for your bucket (env: WALTER_AWS_REGION)')
  .describe('s3-bucket', 'S3 Bucket Name (env: WALTER_S3_BUCKET)')
  .describe('default-wal-size', 'Default size of WAL files produced by postgres (bytes) (env: WALTER_PG_WAL_SIZE)')
  .describe('log-file-path', 'Winston file transport path (env: WALTER_LOG_FILE_PATH)')
  .describe('log-file-size', 'Winston file transport size in MB (env: WALTER_LOG_FILE_SIZE)')
  .describe('log-file-limit', 'Winston file transport rotate limit (env: WALTER_LOG_FILE_LIMIT)')
  .describe('labs-api-key', 'Walter Labs API Key (env: WALTER_LABS_API_KEY)')
  .describe('log-http-host', 'Winston HTTP transport host (env: WALTER_LOG_HTTP_HOST)')
  .describe('log-http-port', 'Winston HTTP transport port (env: WALTER_LOG_HTTP_PORT)')
  .describe('log-http-path', 'Winston HTTP transport path (env: WALTER_LOG_HTTP_PATH)')
  .describe('log-http-ssl', 'Winston HTTP transport ssl (env: WALTER_LOG_HTTP_SSL)')
  .default('prefix', env.WALTER_PREFIX)
  .default('aws-key', env.WALTER_AWS_KEY)
  .default('aws-secret', env.WALTER_AWS_SECRET)
  .default('s3-bucket', env.WALTER_S3_BUCKET)
  .default('default-wal-size', env.WALTER_PG_WAL_SIZE || 16777216)
  .default('log-file-path', env.WALTER_LOG_FILE_PATH)
  .default('log-file-size', env.WALTER_LOG_FILE_SIZE)
  .default('log-file-limit', env.WALTER_LOG_FILE_LIMIT)
  .default('labs-api-key', env.WALTER_LABS_API_KEY)
  .default('log-http-host', env.WALTER_LOG_HTTP_HOST)
  .default('log-http-port', env.WALTER_LOG_HTTP_PORT)
  .default('log-http-path', env.WALTER_LOG_HTTP_PATH)
  .default('log-http-ssl', env.WALTER_LOG_HTTP_SSL)
  .demand('prefix')
  .demand('aws-key')
  .demand('aws-secret')
  .demand('s3-bucket')
  .wrap(120)
  .argv;

winston.remove(winston.transports.Console);

if(argv['labs-api-key']) {
  argv['log-http-host'] = 'api.walter-labs.com';
  argv['log-http-port'] = 443;
  argv['log-http-path'] = '/servers/' + argv['labs-api-key'];
  argv['log-http-ssl'] = true;
}

if(argv['log-http-host']) {
  var logHttpOpts = {
    host: argv['log-http-host']
  };

  if(argv['log-http-port'])
    logHttpOpts.port = argv['log-http-port'];

  if(argv['log-http-path'])
    logHttpOpts.path = argv['log-http-path'];

  if(argv['log-http-ssl'] === 'true' || argv['log-http-ssl'] === true)
    logHttpOpts.ssl = true;

  winston.add(winston.transports.Http, logHttpOpts);
}

if(argv['log-file-path']) {
  var logFileOpts = {
    filename: argv['log-file-path']
  };

  if(argv['log-file-size'])
    logFileOpts.maxsize = (argv['log-file-size']*1024*1024);

  if(argv['log-file-limit'])
    logFileOpts.maxFiles = argv['log-file-limit'];

  winston.add(winston.transports.File, logFileOpts);
}

module.exports = function() {
  var s3 = createS3Client();

  console.log(
    'Verifying base backups and wals for %s in %s bucket.',
    argv['prefix'], argv['s3-bucket']
  );

  s3.list({prefix: argv['prefix'] + '/base'}, verifyBases);
  s3.list({prefix: argv['prefix'] + '/wal'}, verifyWals);
};

function verifyBases(err, data) {
  if(err) return handleErr(err);

  var bases = data['Contents'];

  for(var i=0; i<bases.length; i++) {
    var base = bases[i];
    var filename = path.basename(base.Key);

    if(/^[\da-zA-Z]+_[\da-zA-Z]+$/.test(filename)) {

      var detail = filename.split('_');

      winston.info('base-backup-push: %s: verify.', argv['prefix'], {
        type: 'base',
        action: 'completed',
        prefix: argv['prefix'],
        base_offset: detail[0],
        base_file_name: detail[1],
        manifest: filename + '_manifest.json',
        filename: filename,
        started: null,
        size: base.Size,
        completed: base.LastModified,
        verify: true
      });

    }
  }

  console.log('Base backup verification complete.');
}


function verifyWals(err, data) {
  if(err) return handleErr(err);

  var wals = data['Contents'];

  for(var i=0; i<wals.length; i++) {
    var wal = wals[i];
    var filename = path.basename(wal.Key);

    winston.info('wal-push: %s: %s, verify.', argv['prefix'], filename, {
      type: 'wal',
      action: 'completed',
      prefix: argv['prefix'],
      filename: filename,
      size: argv['default-wal-size'],
      compressed: wal.Size,
      started: null,
      completed: wal.LastModified,
      verify: true
    });
  }

  console.log('Wal verification complete.');
}

function handleErr(err) {
  console.error(err);
  process.exit(1);
}

function createS3Client() {
  var options = {
    key: argv['aws-key'],
    secret: argv['aws-secret'],
    bucket: argv['s3-bucket']
  };

  if(argv['aws-region'])
    options.region = argv['aws-region'];

  return new knox.createClient(options);
}