docparse-supplier-nge
Version:
process ngrid electric utility bill data for use in the docparse system
1,296 lines (1,219 loc) • 37.2 kB
JavaScript
var path = require('path');
var config = require('../../../config');
var logger = require('../../../logger');
var inspect = require('eyes').inspector({maxLength:20000000});
var moment = require('moment');
var ce = require('cloneextend');
var crypto = require('crypto');
var async = require('async');
var fs = require('fs');
var num = require('num');
var helper = require(path.join(config.root_dir,'lib/helper'));
var bill_helper = require(path.join(config.root_dir,'lib/suppliers/bill_helper'));
// Models
var models = require(path.join(config.root_dir,'lib/models/models'));
var Supplier = models.Supplier;
var Bill = models.Bill;
var Invoice = models.Invoice
function generateBillHash(accountNumber, from_date_moment, to_date_moment) {
var concatenated = accountNumber + from_date_moment.format('YYYY MM DD') + to_date_moment.format('YYYY MM DD');
var hash = crypto.createHash("md5")
.update(concatenated)
.digest("hex");
return hash;
}
var bill_process_text = function(bill, supplier, callback){
var params = {supplier: supplier._id, bill: bill._id}
Invoice.findOne(params, function (err, reply) {
if (err) { return callback(err); }
if (reply) {
return callback(null, bill);
}
var page_index = 0;
var text = bill.text_pages.join(' ');
// strip commas
text = text.replace(/,/g,'');
var data;
var cost_other_total = 0;
var supply_text;
var invoice;
async.series([
// get all the general delivery costs
function extract_delivery_costs(cb) {
extract_costs(text, function (err, reply) {
if (err) { return cb(err); }
data = reply
cb();
});
},
// get all the supply costs
function extract_supply_costs(cb) {
supply_text = get_supply_text(text);
if (!supply_text) { return cb(); }
get_supply_data(supply_text, function (err, reply) {
if (err) { return cb(err); }
ce.extend(data, reply);
cb();
});
},
function (cb) {
data.bill_id = bill._id;
data.bill = bill;
var dates = get_start_and_end_dates_from_text(text, function (err, reply) {
if (err) { return cb(err); }
var from_date = moment(reply.from_date);
var to_date = moment(reply.to_date);
bill.from_date = from_date;
bill.to_date = to_date;
data.days = to_date.diff(from_date, 'days');
var bill_number = generateBillHash(bill.account_number, from_date, to_date);
if (bill_number === bill.bill_number) { return cb(); }
remove_bill_if_needed(bill_number, function (err) {
if (err) { return cb(err); }
bill.bill_number = bill_number;
bill.save(cb);
});
});
},
function(cb) {
save_extracted_data(data, cb);
}
], function (err, reply) {
if (err) {
// inspect('********************************************************************************');
// console.log(text);
// inspect('********************************************************************************');
inspect(err, 'err');
inspect(data,' data');
inspect(bill.file_name,' bill_file_name');
return callback(err)
}
callback()
})
});
}
function remove_bill_if_needed(bill_number, callback) {
// see if there is a bill with this file name already
Bill.remove({bill_number: bill_number}, callback);
}
function save_extracted_data(data, callback) {
async.parallel([
function save_utility_data(cb) {
var utility_data = ce.clone(data);
if (data.is_competitive) {
// competitive supply so remove all supply data before saving the utility invoice
delete utility_data['cost_supply'];
delete utility_data['cost_other_supply'];
}
utility_data.bill = data.bill;
utility_data.cost_other = merge_other_costs(utility_data);
save_invoice_data(utility_data,cb);
},
function save_comp_data(cb) {
if (!data.is_competitive) { return cb(); }
data.cost_other = merge_other_costs(data);
save_competitive_supply_invoice_data(data, cb);
},
], callback);
}
/**
* @param {Object} data has the following fields set
* @param {Object} data.cost_other_delivery
* @param {Object} data.cost_other_supply
*
* data.cost_other_supply & data.cost_other_delivery both look like
{
total: 31.99
costs: [
{
description: 'Sales Tax',
value: 21.99
},
{
description: 'Sales Tax',
value: 21.99
},
]
}
*/
function merge_other_costs(data) {
var delivery, supply;
var i,len;
var costs = []
var total = num(0);
if (data.hasOwnProperty('cost_other_delivery') && data.cost_other_delivery) {
delivery = data.cost_other_delivery;
}
if (data.hasOwnProperty('cost_other_supply') && data.cost_other_supply) {
supply = data.cost_other_supply;
}
if (delivery) {
total = total.add(delivery.total);
for (i=0, len=delivery.costs.length; i < len; ++i) {
costs.push(delivery.costs[i]);
}
}
if (supply) {
total = total.add(supply.total);
for (i=0, len=delivery.costs.length; i < len; ++i) {
costs.push(delivery.costs[i]);
}
}
var output = {
total: total.toString(),
costs: costs
}
return output;
}
function extract_costs(text, callback) {
var data = {};
async.series([
function extract_use(cb) {
get_use(text, function (err, reply) {
if (err) { return cb(err); }
data.use = reply;
cb();
});
},
// get Demand KW (from Peak KW in text)
function extract_demand(cb) {
get_demand_kw(text, function (err, reply) {
if (err) { return cb(err); }
data.demand_kw = reply;
cb();
});
},
// get read type
function extract_read_type(cb) {
get_read_type(text, function (err, reply) {
if (err) { return cb(err); }
data.read_type = reply;
cb();
});
},
// get cost delivery
function extract_cost_delivery(cb) {
get_cost_delivery(text, function (err, reply) {
if (err) { return cb(err); }
data.cost_delivery = reply;
cb();
});
},
// get cost other delivery
function extract_cost_other_delivery(cb) {
get_cost_other_delivery(text, function (err, reply) {
if (err) { return cb(err); }
data.cost_other_delivery = reply;
cb();
});
},
// get capacity charge
function extract_capacity_charge(cb) {
get_capacity_charge(text, function (err, reply) {
if (err) { return cb(err); }
data.capacity_charge = reply;
cb();
});
},
// get capacity charge (use)
function extract_capacity_charge_use(cb) {
get_capacity_charge_use(text, function (err, reply) {
if (err) { return cb(err); }
data.capacity_charge_use = reply;
cb();
});
},
// get capacity charge adjustments
function extract_capacity_charge_adjustments(cb) {
get_capacity_charge_adjustment(text, function (err, reply) {
if (err) { return cb(err); }
data.capacity_charge_adjustment = reply;
cb();
});
},
], function (err) {
return callback(err, data);
});
}
/**
* @param {String} text holds the supply text section of the bill
*/
function get_supply_data(text, callback) {
// test if on competitive supply
var data = {};
async.series([
function test_if_comp_supply(cb) {
is_competive_supply(text, function (err, reply) {
if (err) { return cb(err); }
data.is_comp = reply;
if (!data.is_comp) { return cb(); }
get_supplier(text, function (err, reply) {
if (err) { return cb(err); }
data.competitive_supplier = reply;
cb();
});
});
},
// get cost supply
function extract_cost_supply(cb) {
data.text = text;
get_cost_supply(text, function (err, reply) {
if (err) { return cb(err); }
data.cost_supply = reply;
cb();
});
},
// get to pay supply (cost supply + cost other supply
function (cb) {
get_to_pay_supply(text, function (err, reply) {
if (err) { return cb(err); }
data.to_pay_supply = reply;
cb();
});
},
function extract_cost_other_supply(cb) {
get_cost_other_supply(text, function (err, reply) {
if (err) { return cb(err); }
data.cost_other_supply = reply;
cb();
});
},
function confirm_cost_other_supply(cb) {
check_cost_other_supply(data, cb);
}
], function (err) {
if (err) { return callback(err); }
callback(null, data);
});
}
/**
* NGrid Gas invoices do not have a unique invoice number on them because fu thats why
* To uniquely identify an invoice as paid, we have a multi-tiered approach.
* 1: get an account number, start date, and end date
*
* Text looks like:
surrnuat National Grid
BILLING PERIOD
Jun 1, 2012 to Jun 20. 2012
ACCOUNT NUMBER 9029504022 Aug 1 6, 201 2
l"r\vn _ _
EEM@
$ 428.33
*/
var get_start_and_end_dates_from_text = function(text, callback) {
var err, message,
shortMonths = '(?:JAN|FEB|MAR|APR|MAY|JUN|JUL|AUG|SEP|OCT|NOV|DEC)',
dates = [],
pattern,
matches;
pattern = new RegExp(/BILLING PERIOD([\s\S]+)ACCOUNT/);
matches = text.match(pattern);
if (!matches || matches.length !== 2) {
return callback('failed to get start and end dates, billing period section not found', null);
}
var period_text = matches[1];
period_text = period_text.replace(/\./,',');
var date_pattern = '('+shortMonths+' \\d{1,2}.? \\d{4})';
date_pattern = '('+shortMonths+' \\d{1,2}.? \\d{4})';
var period_pattern = new RegExp(date_pattern + ' to '+ date_pattern, 'i');
var period_matches = period_text.match(period_pattern);
if (!period_matches || period_matches.length !== 3) {
console.log(text);
err = 'failed to get start and end dates, no date matches found'
return callback(err, matches);
}
// matches[1] and matches[2] now look like
// "Nov 1, 2011" and "Dec 25, 2011"
var from_date_string = period_matches[1]
var to_date_string = period_matches[2]
var output = {
from_date: moment(from_date_string, 'MMM D, YYYY'),
to_date: moment(to_date_string, 'MMM D, YYYY')
}
return callback(null, output);
}
/**
* Competitive Supply Bills are handled differently than those for standard offer
*
* {Boolean} is_competive
* @return callback(<error if any>, is_supplier)
*/
var is_competive_supply = function(text, callback) {
var is_competive = false;
var pattern = /Supply Services\s*SUPPLIER\s*(.*?)\s*$/im
var matches = text.match(pattern);
if (matches) {
var supplier_name = matches[1];
if (supplier_name !== 'National Grid') {
is_competive = true;
}
}
return callback(null, is_competive);
}
/**
* Find which supplier is suppling for this current bill on competitive supply
* Note that bills on standard offer will not use this function
*/
var get_supplier = function(text, callback) {
var supplier_code;
if (text.match(/Direct Energy Business LLC/i)) {
supplier_code = 'DIR';
}
else if (text.match(/Noble/i)) {
supplier_code = 'NOB';
}
else if (text.match(/NStar/i)) {
supplier_code = 'NST';
}
else if (text.match(/Hess/i)) {
supplier_code = 'HES';
}
else if (text.match(/Liberty Utilities/i)) {
supplier_code = 'LIB';
}
else if (text.match(/NextEra Energy/i)) {
supplier_code = 'NEE';
}
if (!supplier_code) {
console.log(text);
var err = 'unknown competitive supplier'
return callback(err, null);
}
Supplier.findOne({supplier_code: supplier_code}, callback);
}
/**
* Capacity charge looks like
* "...
????
I have not yet seen a bill that has a capacity charge
..."
*/
var get_capacity_charge = function(text, callback) {
var pattern = /capacity/i;
if (text.match(pattern)) {
inspect(text, 'capacity charge found');
return callback('err, capacity charge found but parsing is not yet implemented');
}
return callback(null, 0);
}
/**
* Capacity charge looks like
* "...
????
I have not yet seen a bill that has a capacity charge
..."
*/
var get_capacity_charge_adjustment = function(text, callback) {
var pattern = /capacity/i;
if (text.match(pattern)) {
inspect('********************************************************************************');
inspect('capacity charge found');
console.log(text);
inspect('********************************************************************************');
return callback('err, capacity charge found but parsing is not yet implemented');
}
return callback(null, 0);
}
/**
* Capacity charge looks like
* "...
????
I have not yet seen a bill that has a capacity charge
..."
*/
var get_capacity_charge_use = function(text, callback) {
var pattern = /capacity/i;
if (text.match(pattern)) {
inspect('********************************************************************************');
inspect('capacity charge found');
console.log(text);
inspect('********************************************************************************');
return callback('err, capacity charge found but parsing is not yet implemented');
}
return callback(null, 0);
}
/**
* Demand looks like
* "...
Jul 11 162000 Demand-kW
Peak 200 432.0 kW
Off Peak 200 298.0 kW
..."
*/
var get_demand_kw = function(text, callback) {
var pattern = /Demand-kW[\s\S]*?Peak.*?(-?\s?[\d\.]+) kW\s*$/im;
helper.regex_extract(text, pattern, function (err, reply) {
if (reply) {
return callback(null, reply);
}
pattern = /Demand-kW[\s\S]*?(-?\s?[\d\.]+) kW\s*$/im
helper.regex_extract(text, pattern, function (err, reply) {
if (reply) {
return callback(null, reply);
}
return callback(err, 0);
});
});
}
/**
* Read type text looks like
* "...
Type of Service Current Reading - Previous Reading = Difference x Multiplier = Total Usage
Loadzone WCMA
Energy 228 Actual 0 Actual 228 200 45600 kWh
Acct No: 01392-15008 Cycle: 16, UMAS
Peak 116 Actual 0 Actual 116 200 23200 kWh
..."
*/
var get_read_type = function(text, callback) {
var pattern = /Type of Service[\s\S]*?Energy\s+\d+\s+([a-z]{4,})\s.*?kWh\s*$/im
helper.regex_extract(text, pattern, function (err, reply) {
if (err) {
return callback(err, null);
}
return callback(null, reply);
});
}
/**
* Use text looks like
* "...
* Total Energy 45600 kWh
* ...
* "
*
*/
var get_use = function(text, callback) {
var pattern = /Total Energy\s* (-?\s?[\d\.]+) kWh\s*$/m;
helper.regex_extract(text, pattern, function (err, reply) {
if (err) {
return callback(err, null);
}
if (!reply) {
reply = 0;
}
var output = {
quantity: reply,
units: 'kwh'
}
return callback(null, output);
});
}
/**
* General Credit text looks like
"...
General Credit -0.73
Late Payment Charges 0.03
Total Other Charges/Adjustments - 0.70
*/
var get_general_credit = function(text ,callback) {
text = text.replace(/\$,/g,'');
var pattern = /General Credit.*?(-?\s?[\d\.]+)\s*$/gm;
var cost = num(0);
var cost_found = false;
while(true) {
var matches = pattern.exec(text);
if (!matches || matches.length !== 2) {
break;
}
cost = (cost.add(helper.trim(matches[1])));
cost_found = true;
}
if (!cost_found) {
return callback(null, null);
}
var output = {
value: cost.toString(),
description: 'General Credit'
};
return callback(null, output);
}
/**
* Cost Supply text looks like
"...
Billing Period Jul 13, 2011 to Jul 21, 2011 Number of days in period 8
Basic Service Variable 0.0722 x 45600 kWh 3,292.32
Total Supply Services $ 11,414.00
*/
var get_cost_supply_variable = function(text) {
var pattern = /Basic Service (?:Variable|Fixed).*?(-?\s?[\d\.]+)\s*$/mg;
var cost = num(0);
var cost_found = false
while(true ) {
var matches = pattern.exec(text);
if (!matches || matches.length !== 2) {
break;
}
cost = cost.add(helper.trim(matches[1]));
cost_found = true;
}
if (!cost_found) {
return null;
}
return cost.toString();
}
/**
* Cost Supply Style two text looks like
"...
...
Electricity Supply 0.0681 x 111200 kWh 7572.72
Total Supply Services $ 7572.72'
*/
var get_cost_supply_two = function(text) {
var pattern = /Electricity Supply.*?(-?\s?[\d\.]+)\s*$/gm;
var cost = num(0);
var cost_found = false
while(true) {
var matches = pattern.exec(text);
if (!matches || matches.length !== 2) {
break;
}
cost = (cost.add(helper.trim(matches[1])));
cost_found = true;
}
if (!cost_found) {
return null;
}
return cost.toString();
}
/**
* Cost Supply Style two text looks like
"...
Energy Service 0.07250398 x 6082 kWh 440.97
Total Supply Services $ 440.97'
*/
var get_cost_supply_three = function(text) {
var pattern = /Energy Service.*?(-?\s?[\d\.]+)\s*$/gm;
var cost = num(0);
var cost_found = false
while(true) {
var matches = pattern.exec(text);
if (!matches || matches.length !== 2) {
break;
}
cost = (cost.add(helper.trim(matches[1])));
cost_found = true;
}
if (!cost_found) {
return null;
}
return cost.toString();
}
/**
* Cost Supply Style four text looks like
"...
Energy Service 0.07250398 x 6082 kWh 440.97
Total Supply Services $ 440.97'
*/
var get_cost_supply_four = function(text) {
var pattern = /Energy Charge.*?(-?\s?[\d\.]+)\s*$/gm;
var cost = num(0);
var cost_found = false
while(true) {
var matches = pattern.exec(text);
if (!matches || matches.length !== 2) {
break;
}
cost = (cost.add(helper.trim(matches[1])));
cost_found = true;
}
if (!cost_found) {
return null;
}
return cost.toString();
}
/**
"
* @param {String} text in the supply section text of the bill
*/
var get_cost_supply = function(text, callback) {
text = text.replace(/\$/g,'');
var cost = num(0);
var cost_found = false;
var cost_supply_variable = get_cost_supply_variable(text);
if (cost_supply_variable) {
cost_found = true;
cost = cost.add(cost_supply_variable);
}
var cost_supply_two = get_cost_supply_two(text);
if (cost_supply_two) {
cost_found = true;
cost = cost.add(cost_supply_two);
}
var cost_supply_three = get_cost_supply_three(text);
if (cost_supply_three) {
cost_found = true;
cost = cost.add(cost_supply_three);
}
var cost_supply_four = get_cost_supply_four(text);
if (cost_supply_four) {
cost_found = true;
cost = cost.add(cost_supply_four);
}
if (!cost_found) {
inspect('********************************************************************************');
console.log(text);
var err = 'cost supply not found'
return callback(err, null);
}
var cost_string = cost.toString();
return callback(null, cost_string);
}
/**
* Cost Supply text looks like
"...
Billing Period Jul 13, 2011 to Jul 21, 2011 Number of days in period 8
Basic Service Variable 0.0722 x 45600 kWh 3,292.32
Total Supply Services $ 11,414.00
..."
*/
var get_to_pay_supply = function(text, callback) {
text = text.replace(/\$/g,'');
if (!text.match(/Total Supply Services/)) {
return callback(null, null);
}
var pattern = /Total Supply Services.*?(-?\s?[\d\.]+)\s*$/m;
helper.regex_extract(text, pattern, function (err, reply) {
if (err) { return callback(err, null); }
return callback(null, reply);
});
}
/**
* Cost Delivery looks like
"...
Total Delivery Services $ 7310.30
..."
*/
var get_cost_delivery = function(text, callback) {
text = text.replace(/\$/g,'');
var pattern = /Total Delivery Services\s*(-?\s?[\d\.]+)\s*$/m
helper.regex_extract(text, pattern, function (err, reply) {
if (err) {
return callback(err, null);
}
if (!reply) {
reply = 0;
}
return callback(null, reply);
});
}
/**
* Go through all the other delivery costs line by line
*/
var get_other_cost_line_item = function(text, callback) {
if (text.match(/Other Charges\/Adjustments/)) {
return callback(null, null);
}
var items = [
'Canceled Electric Charges',
'Late Payment',
'LPC Strategic Energy',
'General Credit',
'Enhanced Meter Monthly',
'Transfer Credit/Charges',
'Market Price Adjustment',
'Sales Tax',
'LPC Hess Corp',
'New Installation Charge',
'Refunded Amount',
'Debit Transfer Payment',
'Deposit Credit',
'NextEra Energy Services MA LLC',
'Energy Efficiency Project CoPay',
'Energy Profiler Online'
]
var anchors = items.map(function (item, index, items) {
return '(?:'+item+')';
});
var pattern = anchors.join('|');
var regex = new RegExp(pattern);
if (!text.match(regex)) {
return callback(null);
}
text = text.replace(/\$/g,'');
pattern = /(.*?)\s*(-?\s?[\d\.]+)\s*$/m;
var matches = text.match(pattern);
if (!matches || matches.length !== 3) {
return callback(null, null);
}
var description = matches[1].trim();
var value = matches[2].replace(/\s/g,'');
var output = {
description: description,
value: value,
}
return callback(null, output);
}
var get_supply_text = function(text) {
var pattern = /Supply Services[\s\S]*Total Supply Services.*$/m;
var matches = text.match(pattern);
if (!matches) {
// no supplier charges whatsoever, let alone other charges
return null;
}
return matches[0];
}
var get_other_costs_text = function(text) {
var pattern = /[\s\S]*(Other Charges[\s\S]*Total Other Charges.*$)/m;
var matches = text.match(pattern);
if (!matches || matches.length !== 2 || !matches[1]) {
// no supplier charges whatsoever, let alone other charges
return null;
}
return matches[1].replace(/\$/,'');
}
/**
* @param {String} text looks like
"
Supply Services
SUPPLIER National Grid
Rate Residential Regular R-1
Billing Period Apr 23, 2012 to May 25, 2012 Number of days in period 32
Basic Service Fixed 0.07056405 x 351 kWh 24.77
SUPPLIER National Grid
Rate Residential Regular R-1
Billing Period May 25, 2012 to Jun 25, 2012 Number of days in period 31
Basic Service Fixed 0.06718 x 340 kWh 22.84
SUPPLIER National Grid
Rate Residential Regular R-1
Billing Period Jun 25, 2012 to Jul 24, 2012 Number of days in period 29
Basic Service Fixed 0.06718 x 319 kWh 21.43
Total Supply Services $ 69.04
"
*/
var get_supply_costs_text = function(text) {
// condense supply text to just the charges
var matches, pattern;
if (text.match(/Basic Service/)) {
pattern =/(Basic Service[\s\S]*?)(?:$|(?:SUPPLIER)|(?:Payment Plans)|(?:Page))/g
}
else if (text.match(/Electricity Supply/)) {
pattern =/(Electricity Supply[\s\S]*?)(?:(?:Total Supply Services)|(?:SUPPLIER)|(?:Payment Plans)|(?:Page))/g
}
matches = text.match(pattern);
if (!matches || matches.length === 0 || !matches[0]) {
pattern =/(Energy Service[\s\S]*?)(?:(?:Total Supply Services)|(?:SUPPLIER)|(?:Payment Plans)|(?:Page))/g
matches = text.match(pattern);
if (!matches) {
return null;
}
}
var output = matches.join(' ');
return output;
}
function check_cost_other_supply(data, callback) {
// sum up all other costs
var other_cost_supply = data.cost_other_supply;
var costs = other_cost_supply.costs;
var total = other_cost_supply.total;
var cost_other_num = num(0);
if (data.cost_other_supply.costs.length > 0) {
cost_other_num = sum_other_costs(costs);
}
var cost_other_total = cost_other_num.toString();
var scraped_total_num = num(data.to_pay_supply);
var cost_supply_num = num(data.cost_supply);
var calculated_total_num = cost_other_num.add(cost_supply_num);
var calculated_total = calculated_total_num.toString();
if (!calculated_total_num.eq(scraped_total_num)) {
inspect(data, 'input data');
inspect('supply text');
console.log(data.text);
inspect('********************************************************************************');
return callback('calculated supply total does not match printed total');
}
callback();
}
/**
* @param {String} text the entire supply section of the bill
* @see {Function} get_cost_other_total
*/
var get_cost_other_supply = function(text, callback) {
var anchor;
var anchors = [
'Basic Service Fixed',
'Basic Service Variable',
'Electricity Supply',
'Energy Service',
'Energy Charge',
]
for (var i in anchors) {
var pattern = anchors[i];
var regex = new RegExp(pattern);
if (text.match(regex)) {
anchor = pattern;
}
}
var replace_pattern = new RegExp('[\\s\\S]*?'+anchor +'.*$','m');
text = text.replace(replace_pattern, '');
var costs = [];
var total = num(0);
var lines = text.match(/[^\r\n]+/g);
async.forEach(
lines,
function (line, cb) {
var line_text = line
get_other_cost_line_item(line_text, function (err, reply) {
if (err) { return cb(err); }
if (!reply) { return cb(); }
costs.push(reply);
total = total.add(num(reply.total));
cb();
});
},
function(err) {
if (err) { return callback(err); }
var output = {
total: total.toString(),
costs: costs
};
return callback(null, output);
}
);
}
/**
* First extract the total other cost value printed on the bill. Then iterate
* over all line item other charges and make sure they add up to the printed
* total
*
* @see {Function} get_cost_other_total
*/
var get_cost_other_delivery = function(text, callback) {
var other_costs_text = get_other_costs_text(text);
if (!other_costs_text) {
var no_output = {
total: 0,
costs: []
}
return callback(null, no_output);
}
var total_num;
var total_string;
var costs = [];
async.series([
function(cb) {
get_cost_other_total(other_costs_text, function (err, reply) {
if (err) { return cb(err); }
if (!reply) {
return cb('failed to get total other costs', null);
}
total_string = reply.replace(/\s/g, '');
total_num = num(total_string);
cb();
});
},
// go line by line and get charges
function (cb) {
var lines = other_costs_text.match(/[^\r\n]+/g);
async.forEach(
lines,
function (line, line_cb) {
get_other_cost_line_item(line, function (err, reply) {
if (err) { return line_cb(err); }
if (!reply) { return line_cb(); }
costs.push(reply);
line_cb();
});
},
function (err) {
cb();
}
);
},
function (cb) {
var cost_other_num = num(0);
if (costs.length > 0) {
cost_other_num = sum_other_costs(costs);
}
var cost_other_string = cost_other_num.toString();
if (cost_other_string !== total_string) {
console.log(other_costs_text);
inspect(costs, 'other costs');
inspect(cost_other_string, 'calculated total cost other delivery costs');
inspect(total_string, 'printed total other ');
return cb('error, calculated total other delivery costs do not match printed total other costs');
}
cb();
},
], function(err) {
if (err) { return callback(err); }
var output = {
total: total_string,
costs: costs
};
return callback(null, output);
});
}
/**
* Note that credits are stored as negative costs
* @param {Array} costs contains entries that look like
* costs = [
* {
* description: "Sales Tax",
* value: 1082.41,
* },
* {
* description: "Late Payment Charges ",
* value: 31.59,
* },
* {
* description: "General Credits ",
* value: -2.99,
* },
* ]
* @return {Integer} the summed value, which for the example above should be
* 1111.01 = 1082.41 + 31.59 + (-2.99)
*/
var sum_other_costs = function(costs) {
var sum = num(0);
for (var i in costs) {
var cost = costs[i];
sum = sum.add(num(cost.value));
}
return sum;
}
/**
* cost other text looks like
"...
Other Charges/Adjustments
Sales Tax 6.25 % 1082.24
Total Other Charges/Adjustments $ 1082.24
*/
var get_cost_other_total = function(text, callback) {
var pattern = /Total Other Charges\/Adjustments\s*(-?\s?[\d\.]+)\s*$/m
helper.regex_extract(text, pattern, function (err, reply) {
if (err) {
return callback(err);
}
return callback(null, reply);
});
}
/**
* See /docs/Scrape.md for more details
*
* @param {object} data contains all the fields necessary to begin processing. Note that when this function is called the pdf file has already been saved to a folder on the shared file server. This function just starts a job to OCR the pdf. The following fields should be set in the data object
* supplier_code
* bill_number
* file_name
* bill_date: 'mm/dd/yyy',
* account_number: '<account_number_as_string>'
*
*/
var scraper_process_data = function(data, callback) {
var bill;
// create a new bill document and start a job to extract the data
async.series([
function (cb) {
helper.required_fields(data, ['files', 'files', 'file_name', 'bill_date', 'from_date', 'to_date', 'account_number'], cb);
},
function (cb) {
if (!data.files || !data.files.bill_file_field) {
return cb('file parameter missing from scraper data');
}
data.file = data.files.bill_file_field;
cb();
},
function(cb) {
create_bill_from_scraper_data(data, function (error, reply) {
if (error) { return cb(error); }
if (!reply) {
return cb('no bill returned from create_bill_from_scraper_data');
}
bill = reply;
cb();
});
},
// save the bill pdf data to the gridfs store
function(cb) {
var file_path = data.files.bill_file_field.path;
var save_data = {
file_path: file_path,
file_name: bill.file_name
};
bill_helper.save_bill_pdf_to_database(save_data, cb);
},
function (cb) {
bill_helper.bill_pdf_extract_data(bill, function (err, reply) {
if (err) {
inspect(err, 'error performing pdf extract');
return cb(err);
}
bill = reply;
cb();
});
},
function(cb) {
bill_helper.bill_pdf_process_text(bill, function (err, reply) {
if (err) { return cb(err); }
bill = reply;
cb();
});
},
], function(error) {
if (error) {
logger.error('an error occurred while adding new scraper data', { error: error, data: data, supplier_code: data.supplier_code});
return callback(error, null);
}
callback(null, bill);
});
}
var create_bill_from_scraper_data = function(data, callback) {
var bill = new models.Bill();
var supplier = data.supplier;
bill.supplier_code = supplier.supplier_code;
bill.supplier = supplier._id;
bill.bill_number = data.bill_number;
bill.account_number = data.account_number;
bill.customer = data.customer._id;
bill.login = data.login_id
// dates
var bill_date = moment(data.bill_date, 'YYYY MM DD');
var from_date = moment(data.from_date, 'YYYY MM DD');
var to_date = moment(data.to_date, 'YYYY MM DD');
bill.bill_date = bill_date;
bill.from_date = from_date;
bill.to_date = to_date;
bill.data_type = 'pdf';
bill.file_name = data.supplier_code + '_Electronic_'+data.bill_number + '_'+bill._id+'.pdf';
bill.save(function (err) {
if (err) { return callback(err); }
return callback(null, bill);
});
}
/**
* Save an invoice to the database from the input data
* @param {Bill} data.bill is the bill document which was just created prior
* @see {Function} save_bill_data
*/
var save_invoice_data = function(data, callback) {
var invoice = new Invoice();
var bill = data.bill;
if (!bill) {
return callback('bill missing from data when saving invoice');
}
var supplier = data.supplier;
var to_pay = num(0);
invoice.bill = bill._id;
invoice.customer = data.customer._id;
invoice.login = data.login_id
invoice.bill_number_printed = false;
invoice.bill_number = bill.bill_number;
invoice.file_name = bill.file_name;
invoice.data_type = 'pdf';
invoice.supplier = bill.supplier;
invoice.supplier_code = bill.supplier_code;
invoice.account_number = bill.account_number;
invoice.from_date = bill.from_date;
invoice.to_date = bill.to_date;
if (!data.read_type && bill.read_type) {
data.read_type = bill.read_type
}
invoice.read_type = data.read_type;
invoice.days = data.days;
invoice.use = data.use;
invoice.demand_kw = data.demand_kw;
invoice.invoice_type = {
type: 'utility',
commodity: 'electricity'
}
if (data.cost_supply) {
invoice.cost_supply = data.cost_supply;
to_pay = to_pay.add(data.cost_supply);
}
if (data.cost_delivery) {
invoice.cost_delivery = data.cost_delivery;
to_pay = to_pay.add(data.cost_delivery);
}
to_pay = to_pay.add(data.cost_other.total);
invoice.cost_other = data.cost_other;
invoice.capacity_charge_use = data.capacity_charge_use;
invoice.capacity_charge = data.capacity_charge;
invoice.capacity_charge_adjustment = data.capacity_charge_adjustment;
invoice.date_pay_report = moment().format('MM/DD/YY');
invoice.to_pay = to_pay.toString();
invoice.save(function (err) {
models.Bill.update(
{ _id: bill._id},
{ $addToSet: { invoices: invoice._id}},
function (error, reply) {
if (error) {
logger.error('error when adding invoice document id to bill', {error: error, bill_id: bill._id});
return callback(error);
}
callback(null, invoice);
return;
}
); // end update
});
}
/**
* Save an invoice to the database from the input data.
* This function should only be called for NGrid Electric invoices supplied a
* different competitive supplier
* @param {Bill} data.bill is the bill document which was just created prior
* @param {Supplier} data.competitive_supplier is the supplier to use when creating the invoice
* @see {Function} save_bill_data
*/
var save_competitive_supply_invoice_data = function(data, callback) {
var invoice = new Invoice();
var bill = data.bill;
var supplier = data.supplier;
var to_pay = num(0);
if (!bill) {
return callback('bill missing from data when saving invoice');
}
inspect('saving competitive supply invoices');
invoice.bill = bill._id;
invoice.bill_number = bill.bill_number;
invoice.customer = data.customer._id;
invoice.login = data.login_id;
invoice.billing_supplier_code = 'NGE'
invoice.billing_supplier = data.supplier
invoice.bill_number_printed = false;
invoice.file_name = bill.file_name;
invoice.data_type = 'pdf';
invoice.supplier = data.competitive_supplier._id
invoice.supplier_code = data.competitive_supplier.supplier_code;
invoice.account_number = bill.account_number;
invoice.from_date = bill.from_date;
invoice.to_date = bill.to_date;
if (!data.read_type && bill.read_type) {
data.read_type = bill.read_type
}
invoice.read_type = data.read_type;
invoice.days = data.days;
invoice.use = data.use;
invoice.demand_kw = data.demand_kw;
if (!data.cost_supply) {
return callback('error, cost supply missing from competitive supply invoice data');
}
invoice.invoice_type = {
type: 'supplier',
commodity: 'electricity'
}
invoice.cost_supply = data.cost_supply;
invoice.cost_other = data.cost_other;
to_pay = num(data.cost_supply).add(num(data.cost_other_supply.total));
invoice.capacity_charge_use = data.capacity_charge_use;
invoice.capacity_charge = data.capacity_charge;
invoice.capacity_charge_adjustment = data.capacity_charge_adjustment;
invoice.date_pay_report = moment().format('MM/DD/YY');
invoice.to_pay = to_pay.toString();
invoice.save(function (err) {
models.Bill.update(
{ _id: bill._id},
{ $addToSet: { invoices: invoice._id}},
function (error, reply) {
if (error) {
logger.error('error when adding invoice document id to bill', {error: error, bill_id: bill._id});
return callback(error);
}
callback(null, invoice);
return;
}
); // end update
});
}
exports.create_bill_from_scraper_data = create_bill_from_scraper_data;
exports.generateBillHash = generateBillHash;
exports.bill_process_text = bill_process_text;
exports.scraper_process_data = scraper_process_data;