docparse-scraper-nst/index.js

Node Zombie based scraper to scrape bills from the NStar (supplier code "NST)" website

var Browser = require('zombie');
var uinspect = require('eyespect').inspector();
var fs = require('fs');
var assert = require('assert');

var path = require('path');
var rk = require('required-keys');
module.exports = function(data, cb) {
  var keys = ['logger', 'config'];
  var err = rk.truthySync(data, keys);
  if (err) { return cb(err); }

  // dep injection
  var logins = data.logins;
  if (!logins) {
    logins = require('docparse-fetch-logins-node');
  }

  var processAllLogins = data.processAllLogins;
  if (!processAllLogins) {
    processAllLogins = require('./lib/processAllLogins.js');
  }
  var config = data.config;
  var logger = data.logger;
  var browser = new Browser({
    runScripts: false,
    debug: false
    // maxWait: 1000000,
    // waitFor: 100000,
  });
  // browser.on('error', function(err) {
  //   inspect(err,'browser error');
  //   console.log(err);
  // });
  logger.info('starting scrape', {type: 'scraper', supplierCode: 'NST'});
  var scraperData = {
    supplierCode: 'NST',
    bro: browser,
    config: config,
    logger: logger
  };
  logger.info('getting customer logins', {type: 'scraper', supplierCode: 'NST'});
  logins(scraperData, function (err, loginsReply) {
    if (err) {
      logger.error('error getting customer logins', { error: err, type: 'scraper', supplierCode: 'NST'});
      return cb(err);
    }

    var startIndex = config.get('startLoginIndex') || 0;
    loginsReply.splice(0,startIndex);
    scraperData.numLogins = logins.length;
    scraperData.logins = loginsReply;
    processAllLogins(scraperData, cb);
  });
};