docparse-scraper-nst/lib/downloadBill.js

Node Zombie based scraper to scrape bills from the NStar (supplier code "NST)" website

var ce = require('cloneextend');
module.exports = function download(data, cb) {
  var config = data.config;
  var logger = data.logger;
  var extract = data.extract || require('nst-process-bills');
  var add = data.add || require('docparse-scraper-add-node');
  extract(data, function (err, output) {
    if (err) {
      logger.debug('error extracting data from print-friendly bill', { error: err, type: 'scraper', supplierCode: 'NST'});
      return cb(err);
    }
    if (output.skip) { return cb(); }
    output.loginID = data.currentLogin.loginID;
    output.supplierID = data.currentLogin.supplierID;
    output.customerID = data.currentLogin.customerID;
    var addData = {
      config: config,
      logger: logger,
      payload: output
    };


    var debugData = ce.clone(output);
    debugData.rawHtml = 'truncated';
    if (config.get('verbose')) {
      logger.debug('adding new data', { type: 'scraper', supplierCode: 'NST'});
    }
    add(addData, function (err, reply) {
      if (err) {
        logger.error('error sending scraper data to docparse server', { error: err, type: 'scraper', supplierCode: 'NST'});
        return cb(err);
      }
      if (config.get('verbose')) {
        logger.info('done sending scraper data to docparse server', { type: 'scraper', supplierCode: 'NST'});
      }
      return cb();
    });
  });
};



function logHTML(data, cb) {
  cb();
  // var dir = path.join(__dirname,'../test/data/', data.currentLogin.username, output.accountNumber, output.billNumber);
  // mkdirp.sync(dir);
  // var filePath = path.join(dir, 'data.json');
  // var jsonFile = filed(filePath);
  // var jsonData = ce.clone(output);
  // jsonData.rawHtml = 'truncated';
  // var jsonString = JSON.stringify(jsonData, null, ' ');
  // jsonFile.write(jsonString);
  // var htmlPath = path.join(dir, 'page.html');
  // var htmlFile = filed(htmlPath);
  // htmlFile.write(output.rawHtml);
  // logger.debug('wrote test json data to path', {filePath: filePath, type: 'scraper', supplierCode: 'NST'});
  // logger.debug('wrote test data to path', {filePath: htmlPath, type: 'scraper', supplierCode: 'NST'});
}