docparse-supplier-nge
Version:
process ngrid electric utility bill data for use in the docparse system
316 lines (296 loc) • 10.5 kB
JavaScript
var path = require('path');
var config = require('../../../config');
var logger = require('../../../logger');
var eyes = require('eyes');
var inspect = eyes.inspector({maxLength:20000});
var async = require('async');
var _ = require('underscore');
var ce = require('cloneextend');
var crypto = require('crypto');
var moment = require('moment');
var supplier_raw = require(path.join(config.root_dir,'lib/suppliers/raw'));
var models = require(path.join(config.root_dir,'lib/models/models'));
var Supplier = models.Supplier;
var Upload = models.Upload;
var Bill = models.Bill;
function generateBillHash(accountNumber, from_date_moment, to_date_moment) {
var concatenated = accountNumber + from_date_moment.format('YYYY MM DD') + to_date_moment.format('YYYY MM DD');
var hash = crypto.createHash("md5")
.update(concatenated)
.digest("hex");
return hash;
}
var get_bill_with_account_number_and_dates = function (data, callback) {
var message;
var error_data;
if (!data.from_date || !data.to_date) {
return callback('from_date or to_date not set missing');
}
if (!data.account_number) {
return callback('account_number not set in data');
}
var bill_number = generateBillHash(data.account_number, data.from_date, data.to_date);
// see if we have an electronic bill with the generated hash in the database
Bill.findOne({supplier: data.supplier._id, bill_number: bill_number}, function (err, reply) {
if (err) { return callback(err, null); }
if (!reply) {
inspect(data.account_number, 'account_number');
inspect(data.from_date.format('MMM DD YYYY'), 'from_date');
inspect(data.to_date.format('MMM DD YYYY'), 'to_date');
}
data.bill = reply
return callback(null, data);
});
}
var find_account_number_in_text = function(data, cb) {
data.text = data.page.replace(/[oO]/g,'0');
supplier_raw.find_most_common_element_in_upload(data, cb);
}
var find_bill_by_account_number_and_dates = function(data, callback) {
// find the account number on the page using data.inputs and data.page
var dates, bill;
async.series([
function (cb) {
find_account_number_in_text(data, function (err, reply) {
if (err) { return cb(err); }
if (!reply) { return cb(); }
data.account_number = reply;
cb();
});
},
// get from and to dates
function (cb) {
get_start_and_end_dates_from_text(data, function (err, reply) {
if (err) { return cb(err); }
if (!reply) {
inspect('dates not found');
return cb();
}
dates = reply;
data.from_date = dates.from_date;
data.to_date = dates.to_date;
cb();
});
},
function (cb) {
if (!data.from_date || !data.to_date || !data.ccount_number) { return cb(); }
get_bill_with_account_number_and_dates(data, function (err, reply) {
if (err) { return cb(err); }
if (!reply) { return cb('no bill matched ocr text data'); }
data.bill = reply;
cb();
});
},
], function(err) {
if (err) {return callback(err); }
return callback(null, data);
});
}
var get_bill_with_account_number_and_bill_date = function(account_number, bill_date, callback) {
Bill.find({supplier_code: 'NGA', account_number: account_number, bill_date: bill_date}, function (err, reply) {
if (err) { return callback(err); }
if (!reply) { return callback('no bill found', null); }
if (reply.length > 1 ) { return callback('more than 1 bill found', null); }
return callback(null, reply[0]);
});
}
/**
* Bill date looks like "May 13 '11"
*
* Example:
* "...ear are 1 National Grid address on the back must show in return envelope window Write your account number on check.
Service To E Account Number Next Meter Reading Bill Date
45952-16424 Jun 09 '11 May 13 '11
UMASS L.."
*
*/
var get_bill_date_from_text = function(data, callback) {
var err, message,
shortMonths = '(?:JAN|FEB|MAR|APR|MAY|JUN|JUL|AUG|SEP|OCT|NOV|DEC)',
bill_date,
pattern,
matches;
pattern = new RegExp("Bill date[\\s\\S]*?("+shortMonths+" \\d{2} '\\d{2})$",'igm');
matches = pattern.exec(data.page);
if (!matches || matches.length !== 2) {
err = 'no bill date found'
return callback(err, null);
}
bill_date = moment(matches[1], 'MMM DD \'YY');
return callback(null, bill_date);
}
var get_bill_by_account_number_and_bill_date;
function add_account_numbers_to_data(data, cb) {
Bill.find({supplier: data.supplier._id}).distinct('account_number', function (err, reply) {
if (err) { return cb(err); }
if (!reply || reply.length === 0) {
return cb('no account numbers found to try and match to bill ocr text');
}
// transform from objects into an array
data.inputs = reply;
cb(null, data);
})
}
var add_supplier_to_data = function(data, cb) {
Supplier.findOne({supplier_code: data.upload.supplier_code}, function (err, reply) {
if (err) { return cb(err); }
if (!reply) { return cb('supplier not found with supplier code: ' + data.upload.supplier_code); }
data.supplier = reply;
cb(null, data);
});
}
var handle_warning = function(data, cb) {
var message = 'error parsing raw ocr text';
var error_data = {
upload: data.upload,
message: message,
error: data.error,
page_index: data.page_index
};
// ignore summary bills
if (data.page.match(/SUMMARY OF CHARGES BY ACCOUNT/)) {
return cb();
}
supplier_raw.add_warning_to_upload(error_data, cb);
}
var process_all_pages = function(data, cb) {
var upload = data.upload;
data.bills = [];
data.page_index = -1;
async.forEachSeries(
upload.text_pages,
function (page, page_cb) {
data.page = page.replace(/-/g,'');
data.page_index++;
find_bill_by_account_number_and_dates(data, function (err, reply) {
if (err) {
inspect(err, 'find bill error ');
return page_cb(err);
}
if (!reply.bill) {
inspect({ account_number: reply.account_number, from_date: reply.from_date, to_date: reply.to_date}, 'no bill found');
// find out what the problem was
if (!data.account_number) {
data.error = 'account number not found';
}
else if (!data.from_date || !data.to_date) {
data.error = 'from and to dates not found in text';
}
else {
data.error = 'no matching bill found with account_number: ' + data.account_number + ', from date: ' + data.from_date.format('MMM DD, YYYY') + ', to date: ' + data.to_date.format('MMM DD, YYYY');
}
return handle_warning(data, page_cb);
}
data.bills.push(reply._id);
return page_cb();
});
}, function (err) {
if (err) { return cb(err); }
cb(null, data);
});
}
var upload_process_text = function(upload, callback) {
var data = {
upload: upload
}
inspect('ngrid electric raw parse begin');
async.waterfall([
// Get all NGrid Electric Account numbers
function (cb) {
add_supplier_to_data(data, cb);
},
add_account_numbers_to_data,
process_all_pages
], function (err, data) {
if (err) { return callback(err); }
if (data.bills && data.bills.length > 0) {
inspect(data.bills, 'data.bills');
supplier_raw.set_upload_matches(upload, data.bills, function (error, reply) {
return callback(error, reply);
});
}
else {
return callback(null, upload);
}
});
}
/**
* NGrid Gas invoices do not have a unique invoice number on them because fu thats why
* To uniquely identify an invoice as paid, we have a multi-tiered approach.
* @param {Object} data has the following fields set
* @param {String} data.page the raw ocr text of the given page
* 1: get an account number, start date, and end date
*
* Text looks like:
surrnuat National Grid
BILLING PERIOD
Jun 1, 2012 to Jun 20. 2012
ACCOUNT NUMBER 9029504022 Aug 1 6, 201 2
l"r\vn _ _
EEM@
$ 428.33
*/
var get_start_and_end_dates_from_text = function(data, callback) {
var err, message,
shortMonths = '(?:JAN|FEB|MAR|APR|MAY|JUN|JUL|AUG|SEP|OCT|NOV|DEC)',
dates = [],
pattern,
matches;
var text = data.page;
text = text.replace(/2\s?[oO0]\s?([1li])\s?(\w)/g, '20$1$2');
var get_date_from_string = function(input) {
var pattern = new RegExp('('+shortMonths+').*(\\w{1,2}). ?(\\w{4})', 'i');
var matches = input.match(pattern);
matches.shift();
var month = matches[0];
var day = matches[1]
var year = matches[2]
var date_string = matches.join(' ').replace(/[oO]/g,'0');
var date = moment(date_string, 'MMM D YYYY');
return date;
}
var get_dates_in_text = function(input) {
var date_pattern = '('+shortMonths+' ?\\w{1,2}. ?\\w{4})';
date_pattern = '('+shortMonths+' ?\\w{1,2}. ?\\w{4})';
var regex = new RegExp(date_pattern + '.*'+ date_pattern, 'i');
var period_matches = input.match(regex);
if (!period_matches || period_matches.length < 3) {
err = 'failed to get start and end dates'
return null;
}
// matches[1] and matches[2] now look like
// "Nov 1, 2011" and "Dec 25, 2011"
var from_date_string = period_matches[1];
var to_date_string = period_matches[2];
var from_date = get_date_from_string(from_date_string);
var to_date = get_date_from_string(to_date_string);
var output = {
from_date: from_date,
to_date: to_date
}
return output;
}
text = text.replace(/\./g,',');
pattern = new RegExp(/BILLING PERIOD([\s\S]+)ACCOUNT/i);
matches = text.match(pattern);
if (matches && matches.length == 2 && matches[1]) {
var period_text = matches[1];
dates = get_dates_in_text(period_text);
if (dates) {
return callback(null, dates);
}
}
dates = get_dates_in_text(text);
if (dates) {
return callback(null, dates);
}
inspect('********************************************************************************');
inspect('dates not found in following text');
console.log(text)
inspect('********************************************************************************');
callback(null, null);
}
exports.generateBillHash = generateBillHash;
exports.upload_process_text = upload_process_text;
exports.get_start_and_end_dates_from_text = get_start_and_end_dates_from_text;
exports.get_bill_date_from_text = get_bill_date_from_text;