docparse-supplier-nge
Version:
process ngrid electric utility bill data for use in the docparse system
502 lines (467 loc) • 14.2 kB
JavaScript
var ce = require('cloneextend');
var inspect = require('eyespect').inspector({maxLength:20000000});
var num = require('num');
var async = require('async');
var moment= require('moment');
var extract = require('regex-extract');
var Invoice = require('docparse-invoice');
var Supplier = require('docparse-supplier');
var path = require('path');
var supplyText = require(path.join(__dirname, 'supplyText'));
var saveUtilityInvoice = require('./saveUtilityInvoice');
var createCompSupplyInvoice = require('./createCompSupplyInvoice');
var getSupplyData = require('./getSupplyData');
module.exports = function(bill, supplier, callback){
var params = {supplier: supplier._id, bill: bill._id};
Invoice.findOne(params, function (err, reply) {
if (err) { return callback(err); }
if (reply) {
return callback(null, bill);
}
var page_index = 0;
var text = bill.text_pages.join(' ');
// strip commas
text = text.replace(/,/g,'');
var data;
var cost_other_total = 0;
var supply_text;
var invoice;
async.series([
// get all the general delivery costs
function extract_delivery_costs(cb) {
extract_costs(text, function (err, reply) {
if (err) { return cb(err); }
data = reply;
cb();
});
},
// get all the supply costs
function extract_supply_costs(cb) {
supply_text = get_supply_text(text);
inspect(supply_text, 'supply text');
if (!supply_text) { return cb(); }
getSupplyData(supply_text, function (err, reply) {
if (err) { return cb(err); }
ce.extend(data, reply);
cb();
});
},
function(cb) {
save_extracted_data(data, cb);
}
], function (err, reply) {
if (err) {
// inspect('********************************************************************************');
// console.log(text);
// inspect('********************************************************************************');
inspect(err, 'err');
inspect(data,' data');
inspect(bill.file_name,' bill_file_name');
return callback(err);
}
callback()
})
});
}
function save_extracted_data(data, callback) {
async.series([
function save_utility_data(cb) {
var utility_data = ce.clone(data);
if (data.is_competitive) {
// competitive supply so remove all supply data before saving the utility invoice
delete utility_data['cost_supply'];
delete utility_data['cost_other_supply'];
}
utility_data.bill = data.bill;
utility_data.cost_other = merge_other_costs(utility_data);
createInvoice(utility_data, cb);
},
function save_comp_data(cb) {
if (!data.is_competitive) { return cb(); }
data.cost_other = merge_other_costs(data);
createCompSupplyInvoice(data, cb);
},
], callback);
}
/**
* @param {Object} data has the following fields set
* @param {Object} data.cost_other_delivery
* @param {Object} data.cost_other_supply
*
* data.cost_other_supply & data.cost_other_delivery both look like
{
total: 31.99
costs: [
{
description: 'Sales Tax',
value: 21.99
},
{
description: 'Sales Tax',
value: 21.99
},
]
}
*/
function merge_other_costs(data) {
var delivery, supply;
var i,len;
var costs = []
var total = num(0);
if (data.hasOwnProperty('cost_other_delivery') && data.cost_other_delivery) {
delivery = data.cost_other_delivery;
}
if (data.hasOwnProperty('cost_other_supply') && data.cost_other_supply) {
supply = data.cost_other_supply;
}
if (delivery) {
total = total.add(delivery.total);
for (i=0, len=delivery.costs.length; i < len; ++i) {
costs.push(delivery.costs[i]);
}
}
if (supply) {
total = total.add(supply.total);
for (i=0, len=delivery.costs.length; i < len; ++i) {
costs.push(delivery.costs[i]);
}
}
var output = {
total: total.toString(),
costs: costs
}
return output;
}
function extract_costs(text, callback) {
var data = {};
async.series([
function extract_use(cb) {
get_use(text, function (err, reply) {
if (err) { return cb(err); }
data.use = reply;
cb();
});
},
// get Demand KW (from Peak KW in text)
function extract_demand(cb) {
get_demand_kw(text, function (err, reply) {
if (err) { return cb(err); }
data.demand_kw = reply;
cb();
});
},
// get read type
function extract_read_type(cb) {
get_read_type(text, function (err, reply) {
if (err) { return cb(err); }
data.read_type = reply;
cb();
});
},
// get cost delivery
function extract_cost_delivery(cb) {
get_cost_delivery(text, function (err, reply) {
if (err) { return cb(err); }
data.cost_delivery = reply;
cb();
});
},
// get cost other delivery
function extract_cost_other_delivery(cb) {
get_cost_other_delivery(text, function (err, reply) {
if (err) { return cb(err); }
data.cost_other_delivery = reply;
cb();
});
},
// get capacity charge
function extract_capacity_charge(cb) {
get_capacity_charge(text, function (err, reply) {
if (err) { return cb(err); }
data.capacity_charge = reply;
cb();
});
},
// get capacity charge (use)
function extract_capacity_charge_use(cb) {
get_capacity_charge_use(text, function (err, reply) {
if (err) { return cb(err); }
data.capacity_charge_use = reply;
cb();
});
},
// get capacity charge adjustments
function extract_capacity_charge_adjustments(cb) {
get_capacity_charge_adjustment(text, function (err, reply) {
if (err) { return cb(err); }
data.capacity_charge_adjustment = reply;
cb();
});
},
], function (err) {
return callback(err, data);
});
}
/**
* NGrid Gas invoices do not have a unique invoice number on them because fu thats why
* To uniquely identify an invoice as paid, we have a multi-tiered approach.
* 1: get an account number, start date, and end date
*
* Text looks like:
surrnuat National Grid
BILLING PERIOD
Jun 1, 2012 to Jun 20. 2012
ACCOUNT NUMBER 9029504022 Aug 1 6, 201 2
l"r\vn _ _
EEM@
$ 428.33
*/
var get_start_and_end_dates_from_text = function(text, callback) {
var err, message,
shortMonths = '(?:JAN|FEB|MAR|APR|MAY|JUN|JUL|AUG|SEP|OCT|NOV|DEC)',
dates = [],
pattern,
matches;
pattern = new RegExp(/BILLING PERIOD([\s\S]+)ACCOUNT/);
matches = text.match(pattern);
if (!matches || matches.length !== 2) {
return callback('failed to get start and end dates, billing period section not found', null);
}
var period_text = matches[1];
period_text = period_text.replace(/\./,',');
var date_pattern = '('+shortMonths+' \\d{1,2}.? \\d{4})';
date_pattern = '('+shortMonths+' \\d{1,2}.? \\d{4})';
var period_pattern = new RegExp(date_pattern + ' to '+ date_pattern, 'i');
var period_matches = period_text.match(period_pattern);
if (!period_matches || period_matches.length !== 3) {
console.log(text);
err = 'failed to get start and end dates, no date matches found'
return callback(err, matches);
}
// matches[1] and matches[2] now look like
// "Nov 1, 2011" and "Dec 25, 2011"
var from_date_string = period_matches[1]
var to_date_string = period_matches[2]
var output = {
from_date: moment(from_date_string, 'MMM D, YYYY'),
to_date: moment(to_date_string, 'MMM D, YYYY')
}
return callback(null, output);
}
/**
* Capacity charge looks like
* "...
????
I have not yet seen a bill that has a capacity charge
..."
*/
var get_capacity_charge = function(text, callback) {
var pattern = /capacity/i;
if (text.match(pattern)) {
inspect(text, 'capacity charge found');
return callback('err, capacity charge found but parsing is not yet implemented');
}
return callback(null, 0);
}
/**
* Capacity charge looks like
* "...
????
I have not yet seen a bill that has a capacity charge
..."
*/
var get_capacity_charge_adjustment = function(text, callback) {
var pattern = /capacity/i;
if (text.match(pattern)) {
inspect('********************************************************************************');
inspect('capacity charge found');
console.log(text);
inspect('********************************************************************************');
return callback('err, capacity charge found but parsing is not yet implemented');
}
return callback(null, 0);
}
/**
* Capacity charge looks like
* "...
????
I have not yet seen a bill that has a capacity charge
..."
*/
var get_capacity_charge_use = function(text, callback) {
var pattern = /capacity/i;
if (text.match(pattern)) {
inspect('********************************************************************************');
inspect('capacity charge found');
console.log(text);
inspect('********************************************************************************');
return callback('err, capacity charge found but parsing is not yet implemented');
}
return callback(null, 0);
}
/**
* Demand looks like
* "...
Jul 11 162000 Demand-kW
Peak 200 432.0 kW
Off Peak 200 298.0 kW
..."
*/
var get_demand_kw = function(text, callback) {
var pattern = /Demand-kW[\s\S]*?Peak.*?(-?\s?[\d\.]+) kW\s*$/im;
extract(text, pattern, function (err, reply) {
if (reply) {
return callback(null, reply);
}
pattern = /Demand-kW[\s\S]*?(-?\s?[\d\.]+) kW\s*$/im
extract(text, pattern, function (err, reply) {
if (reply) {
return callback(null, reply);
}
return callback(err, 0);
});
});
}
/**
* Read type text looks like
* "...
Type of Service Current Reading - Previous Reading = Difference x Multiplier = Total Usage
Loadzone WCMA
Energy 228 Actual 0 Actual 228 200 45600 kWh
Acct No: 01392-15008 Cycle: 16, UMAS
Peak 116 Actual 0 Actual 116 200 23200 kWh
..."
*/
var get_read_type = function(text, callback) {
var pattern = /Type of Service[\s\S]*?Energy\s+\d+\s+([a-z]{4,})\s.*?kWh\s*$/im
extract(text, pattern, function (err, reply) {
if (err) {
return callback(err, null);
}
return callback(null, reply);
});
}
/**
* Use text looks like
* "...
* Total Energy 45600 kWh
* ...
* "
*
*/
/**
* General Credit text looks like
"...
General Credit -0.73
Late Payment Charges 0.03
Total Other Charges/Adjustments - 0.70
*/
var get_general_credit = function(text ,callback) {
text = text.replace(/\$,/g,'');
var pattern = /General Credit.*?(-?\s?[\d\.]+)\s*$/gm;
var cost = num(0);
var cost_found = false;
while(true) {
var matches = pattern.exec(text);
if (!matches || matches.length !== 2) {
break;
}
var match = matches[1].trim();
cost = cost.add(match);
cost_found = true;
}
if (!cost_found) {
return callback(null, null);
}
var output = {
value: cost.toString(),
description: 'General Credit'
};
return callback(null, output);
}
/**
* Go through all the other delivery costs line by line
*/
var get_other_cost_line_item = function(text, callback) {
if (text.match(/Other Charges\/Adjustments/)) {
return callback(null, null);
}
// var items = [
// 'Canceled Electric Charges',
// 'Late Payment',
// 'LPC Strategic Energy',
// 'General Credit',
// 'Enhanced Meter Monthly',
// 'Transfer Credit/Charges',
// 'Market Price Adjustment',
// 'Sales Tax',
// 'LPC Hess Corp',
// 'New Installation Charge',
// 'Refunded Amount',
// 'Debit Transfer Payment',
// 'Deposit Credit',
// 'NextEra Energy Services MA LLC',
// 'Energy Efficiency Project CoPay',
// 'Energy Profiler Online'
// ]
var anchors = items.map(function (item, index, items) {
return '(?:'+item+')';
});
var pattern = anchors.join('|');
var regex = new RegExp(pattern);
if (!text.match(regex)) {
return callback(null);
}
text = text.replace(/\$/g,'');
pattern = /(.*?)\s*(-?\s?[\d\.]+)\s*$/m;
var matches = text.match(pattern);
if (!matches || matches.length !== 3) {
return callback(null, null);
}
var description = matches[1].trim();
var value = matches[2].replace(/\s/g,'');
var output = {
description: description,
value: value,
}
return callback(null, output);
}
/**
* @param {String} text looks like
"
Supply Services
SUPPLIER National Grid
Rate Residential Regular R-1
Billing Period Apr 23, 2012 to May 25, 2012 Number of days in period 32
Basic Service Fixed 0.07056405 x 351 kWh 24.77
SUPPLIER National Grid
Rate Residential Regular R-1
Billing Period May 25, 2012 to Jun 25, 2012 Number of days in period 31
Basic Service Fixed 0.06718 x 340 kWh 22.84
SUPPLIER National Grid
Rate Residential Regular R-1
Billing Period Jun 25, 2012 to Jul 24, 2012 Number of days in period 29
Basic Service Fixed 0.06718 x 319 kWh 21.43
Total Supply Services $ 69.04
"
*/
var get_supply_costs_text = function(text) {
// condense supply text to just the charges
var matches, pattern;
if (text.match(/Basic Service/)) {
pattern =/(Basic Service[\s\S]*?)(?:$|(?:SUPPLIER)|(?:Payment Plans)|(?:Page))/g
}
else if (text.match(/Electricity Supply/)) {
pattern =/(Electricity Supply[\s\S]*?)(?:(?:Total Supply Services)|(?:SUPPLIER)|(?:Payment Plans)|(?:Page))/g
}
matches = text.match(pattern);
if (!matches || matches.length === 0 || !matches[0]) {
pattern =/(Energy Service[\s\S]*?)(?:(?:Total Supply Services)|(?:SUPPLIER)|(?:Payment Plans)|(?:Page))/g
matches = text.match(pattern);
if (!matches) {
return null;
}
}
var output = matches.join(' ');
return output;
}