docparse-scraper-nst/lib/processAllBills.js

Node Zombie based scraper to scrape bills from the NStar (supplier code "NST)" website

var inspect = require('eyespect').inspector();
var rk = require('required-keys');
var async = require('async');
var moment = require('moment');
var crypto = require('crypto');

module.exports = function(data, callback) {
  var keys = ['accountIndex', 'numAccounts', 'bro', 'config', 'billDates', 'accountNumber', 'billNumbers'];
  var err = rk.nonNullSync(data, keys);
  if (err) {
    return callback(err);
  }

  var processSingleBill = data.processSingleBill;
  if (!processSingleBill) {
    processSingleBill = require('./processSingleBill');
  }
  var config = data.config;
  var index = -1;
  var accountNumber = data.accountNumber;
  data.numBills = data.billDates.length;


  async.forEachSeries(
    data.billDates,
    function (billDate, cb) {
      index++;
      var formatted = moment(billDate, 'MMM DD, YYYY').format('YYYY MM DD');
      data.billNumber = getBillNumber(accountNumber, formatted);
      data.billDateString = billDate;
      data.billDate = formatted;
      data.billIndex = index;
      processSingleBill(data, function (err, reply) {
        if (err) {
          if (config.get('verbose')) {
            inspect(err, 'error processing single bill');
          }
          return cb(err);
        }
        if (config.get('verbose')) {
          inspect(true, 'processed single bill correctly? ');
        }
        return cb();
      });
    },
    function (err) {
      if (err) {
        inspect(err, 'error processing bills');
        return callback(err);
      }
      callback(err);
    }
  );
};


function getBillNumber(accountNumber, billDate) {
  var concatenated = accountNumber+billDate;
  var billNumber = crypto.createHash('md5').update(concatenated).digest("hex");
  return billNumber;
}