docparse-scraper-nst/lib/scrapeAllAccounts.js

Node Zombie based scraper to scrape bills from the NStar (supplier code "NST)" website

var inspect = require('eyespect').inspector();
var async = require('async');
var rk = require('required-keys');


module.exports = function(data, callback) {
  var keys = ['accountNumbers', 'bro', 'config', 'currentLogin', 'supplierCode'];
  var err = rk.truthySync(data, keys);
  if (err) { return callback(err); }

  var scrapeSingleAccount = data.scrapeSingleAccount;
  if (!scrapeSingleAccount) {
    scrapeSingleAccount = require('./scrapeSingleAccount');
  }
  var accountNumbers = data.accountNumbers;
  var config = data.config;
  var logger = data.logger;
  var startIndex = config.get('startAccountIndex') || 0;
  config.set('startAccountIndex', 0);

  accountNumbers.splice(0, startIndex);
  var index = startIndex-1;
  async.forEachSeries(
    accountNumbers,
    function (accountNumber, cb) {
      index++;
      var accountData = {
        accountNumber: accountNumber,
        numAccounts: accountNumbers.length,
        accountIndex: index,
        config: data.config,
        bro: data.bro,
        currentLogin: data.currentLogin,
        supplierCode: data.supplierCode,
        logger: data.logger
      };

      scrapeSingleAccount(accountData, function (err, reply) {
        if (err) {
          logger.debug('error scraping single account', {
            type: 'scraper',
            supplierCode: 'NST',
            accountNumber: accountNumber,
            error: err
          });
          return cb(err);
        }
        cb();
      });
    }, function (err) {
      if (err) {
        logger.debug('error in scrapeAllAccounts final callback', {
          type: 'scraper',
          supplierCode: 'NST',
          error: err
        });
        return callback(err);
      }
      callback();
    });
};