aps-data-api
Version:
package for data extraction from APS company for omnimetic project
646 lines (645 loc) • 32.6 kB
JavaScript
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.APSWorkflow = void 0;
const axios_1 = __importDefault(require("axios"));
const config_1 = __importDefault(require("config"));
const fs_1 = __importDefault(require("fs"));
const path_1 = __importDefault(require("path"));
const pdf_js_extract_1 = require("pdf.js-extract");
const puppeteer_1 = __importDefault(require("puppeteer"));
const request_1 = __importDefault(require("request"));
const tunnel_1 = __importDefault(require("tunnel"));
const typings_1 = require("../../typings");
const models_1 = require("../../models");
const services_1 = require("../../services");
const helpers_1 = require("../../helpers");
class APSWorkflow extends services_1.Workflow {
constructor() {
super();
const workFlowPath = path_1.default.resolve(__dirname);
const workFlowConfigFilePath = path_1.default.join(workFlowPath, 'workflow.json');
this.config = this.readConfig(workFlowConfigFilePath);
}
initializeBrowser() {
return __awaiter(this, void 0, void 0, function* () {
this.browser = yield puppeteer_1.default.launch({
args: [
'--enable-features=NetworkService',
'--disable-setuid-sandbox',
'--no-sandbox',
],
});
});
}
login(payload, tunnelingAgentOptions) {
return __awaiter(this, void 0, void 0, function* () {
const { companyUrl, dataExtractionError, passwordCssId, usernameCssId } = this.config.login;
try {
this.logger.info('[APS] Login to APS process start');
if (!this.browser) {
yield this.initializeBrowser();
}
this.page = yield this.browser.newPage();
const page = this.page;
page.setDefaultTimeout(config_1.default.get('puppeteer.defaultTimeout'));
yield page.setRequestInterception(true);
// Uncomment below code to see browser console logs
// page.on('console', (message) => {
// this.logger.info(
// `[APS] Browser Console: ${message.type()}, ${message.text()}`,
// );
// });
this.setPageOnEvent(page, tunnelingAgentOptions);
const { password, username } = payload;
yield page.goto(companyUrl, {
waitUntil: 'networkidle0',
});
yield page.waitForSelector(usernameCssId);
yield this.typeValueForField(usernameCssId, username);
yield this.typeValueForField(passwordCssId, password);
yield this.click('Enter');
this.logger.info('[APS] Login to APS process end');
}
catch (err) {
yield this.browser.close();
return Promise.reject(new typings_1.Errors.DataExtractionError(`${dataExtractionError}: ${err}`));
}
});
}
getServiceAccounts() {
return __awaiter(this, void 0, void 0, function* () {
const page = this.page;
const { addressDataSelector, customerProfileUrl, dataExtractionError } = this.config.getServiceAccounts;
const userDetails = yield this.getUserDetails();
try {
const { id, name, premiseDetailsList, legalNames, meterNumber } = userDetails;
yield page.goto(customerProfileUrl);
yield page.waitForSelector(addressDataSelector);
const addressData = yield page.$eval(addressDataSelector, (element) => {
return element.textContent;
});
// addressData = 'Mailing address416115 N JW Marriot RD # 120-156, SCOTTSDALE,
// ARIZONA,85641, USA'
const address = addressData.slice(15).split(',');
const city = address[1].trim();
const postalCode = address[3].trim();
const state = address[2].trim();
const street = address[0];
const serviceAccounts = [];
for (const premiseDetails of premiseDetailsList) {
const premiseId = yield premiseDetails.premiseID;
serviceAccounts.push(new services_1.UtilityServiceAccountResponse({
id: premiseId,
address: {
city,
postalCode,
state,
street,
},
meterNumber,
legalNames,
}));
}
const response = new services_1.UtilityUserAccountResponse({
id,
name,
serviceAccounts,
});
return Promise.resolve(response);
}
catch (err) {
return Promise.reject(new typings_1.Errors.DataExtractionError(`${dataExtractionError}: ${err}`));
}
finally {
yield this.browser.close();
}
});
}
getMonthlyUtilityData() {
return __awaiter(this, void 0, void 0, function* () {
const page = this.page;
const { billDetailsUrl, dataExtractionError } = this.config.getMonthlyUtilityData;
const userDetails = yield this.getUserDetails();
try {
const { premiseDetailsList } = userDetails;
const externalServiceAccountId = yield premiseDetailsList[0].premiseID;
const billsDataObject = yield this.getBillsData();
const { billsData, billsDateString } = billsDataObject;
let i = -1;
const userMonthlyUtilityDataList = [];
for (const bill of billsData) {
try {
i += 1;
if (billsDateString[i] !== billsDateString[i - 1]) {
const relativeDownloadFilePath = `service_acccounts/${externalServiceAccountId}/bills/${billsDateString[i]}`;
const downloadPath = path_1.default.resolve(__dirname, `/tmp/downloads/${relativeDownloadFilePath}`);
const client = yield page.target().createCDPSession();
yield client.send('Page.setDownloadBehavior', {
downloadPath,
behavior: 'allow',
});
yield bill.click();
yield page.waitForResponse((request) => request.url().startsWith(billDetailsUrl));
this.logger.info(`[APS] Downloading pdf ${i}`);
yield new Promise((resolve) => setTimeout(resolve, config_1.default.get('apsDownloadPDF.timeOut')));
const monthlyData = yield this.extractPdfData(downloadPath, billsDateString[i], externalServiceAccountId);
userMonthlyUtilityDataList.push(monthlyData);
fs_1.default.unlinkSync(`${downloadPath}/bill.pdf`);
}
}
catch (err) {
this.logger.error(`[APS] Downloading PDF error: ${err.message}`);
}
}
return Promise.resolve(userMonthlyUtilityDataList);
}
catch (err) {
return Promise.reject(new typings_1.Errors.DataExtractionError(`${dataExtractionError}: ${err.message}`));
}
finally {
yield this.browser.close();
}
});
}
getReqHeaderAndBody() {
return __awaiter(this, void 0, void 0, function* () {
const page = this.page;
const { utilityDataUrl, dataExtractionError, intervalUsageUrl } = this.config.hourlyUtilityDataConfig;
try {
this.logger.info('[APS] Fetching request headers and body start');
let headers = {}, requestBody;
// cookies: string;
yield page.setRequestInterception(true);
page.on('response', (response) => __awaiter(this, void 0, void 0, function* () {
if (response.url().startsWith(intervalUsageUrl)) {
//cookies = response.headers()['set-cookie'];
headers = response.request().headers();
requestBody = response.request().postData();
}
}));
yield page.waitForNavigation({ waitUntil: 'networkidle2' });
yield page.goto(utilityDataUrl, { waitUntil: 'networkidle0' });
let newHeaders = {};
const client = yield page.target().createCDPSession();
let cookies = (yield client.send('Network.getAllCookies')).cookies;
let cookieString = '';
cookies.forEach((cookie) => {
cookieString += cookie.name + '=' + cookie.value + '; ';
});
newHeaders['authorization'] = headers['authorization'];
newHeaders['ocp-apim-subscription-key'] =
headers['ocp-apim-subscription-key'];
newHeaders['Cookie'] = cookieString;
this.logger.info('[APS] Fetching request headers and body end');
return { headers: newHeaders, requestBody };
}
catch (err) {
return Promise.reject(new typings_1.Errors.DataExtractionError(`${dataExtractionError}: ${err}`));
}
finally {
yield this.browser.close();
}
});
}
makeBatchRequest(headers, body, nextDate, finishDate) {
return __awaiter(this, void 0, void 0, function* () {
const { batchSize, batchesPerLoop, dataExtractionError, requestUrl } = this.config.hourlyUtilityDataConfig;
try {
this.logger.info('[APS] Batch requests function start');
let batchRequestConfigsArray = [];
let nextDateForExtraction = new Date(nextDate);
nextDateForExtraction.setDate(nextDate.getDate() - batchSize * batchesPerLoop);
for (let i = 0; i < batchesPerLoop; i++) {
let batchRequestConfigs = [];
for (let j = batchSize * i + 1; j <= batchSize * i + batchSize; j++) {
let prevDate = new Date(nextDate);
prevDate.setDate(nextDate.getDate() - j);
const prevDateString = helpers_1.DateHelper.formatDate(prevDate);
let configRequestBody = JSON.parse(body);
if (prevDateString === configRequestBody.ratePlan[0].startDate ||
prevDateString === helpers_1.DateHelper.formatDate(finishDate)) {
// Checks if the loop has reached the begining date of utility service start
nextDateForExtraction = null;
break;
}
configRequestBody['billCycleStartDate'] = prevDateString;
configRequestBody['billCycleEndDate'] = prevDateString;
const config = {
method: 'post',
url: requestUrl,
headers: headers,
data: configRequestBody,
};
batchRequestConfigs.push(config);
}
batchRequestConfigsArray.push(batchRequestConfigs);
if (nextDateForExtraction === null)
break;
}
let batchResponse = [];
let k = batchRequestConfigsArray.length;
while (k >= 1) {
const axiosApiResponseArray = yield this.axiosBatchApiRequest(batchRequestConfigsArray[k - 1], 0);
if (axiosApiResponseArray === undefined)
continue;
const hourlyDataFormatted = yield this.formatHourlyData(axiosApiResponseArray);
batchResponse = batchResponse.concat(...hourlyDataFormatted);
k--;
}
this.logger.info('[APS] Batch requests function end');
return { batchResponse, nextBatchDate: nextDateForExtraction };
}
catch (err) {
return Promise.reject(new typings_1.Errors.DataExtractionError(`${dataExtractionError}: ${err}`));
}
});
}
axiosBatchApiRequest(batchRequestConfigs, retryCount) {
return __awaiter(this, void 0, void 0, function* () {
try {
const res = yield axios_1.default.all(batchRequestConfigs.map((requestConfigs, index) => __awaiter(this, void 0, void 0, function* () {
return {
response: yield (0, axios_1.default)(requestConfigs),
date: batchRequestConfigs[index].data.billCycleStartDate,
};
})));
this.logger.info('[APS] Axios batch request sent');
return res;
}
catch (err) {
if (retryCount === 2) {
this.logger.error('[APS] Axios request error');
return Promise.reject(err);
}
yield this.axiosBatchApiRequest(batchRequestConfigs, retryCount + 1);
}
});
}
formatHourlyData(axiosApiResponseArray) {
return __awaiter(this, void 0, void 0, function* () {
const res = axiosApiResponseArray.map((axiosApiResponse) => axiosApiResponse.response.data.summarizedUsageDataResponse.dailyRatePlanUsage.map((hourlyData) => {
return new models_1.HourlyUtilityData(Object.assign(Object.assign({}, hourlyData), { date: axiosApiResponse.date }));
}));
return res;
});
}
getUserDetails() {
return __awaiter(this, void 0, void 0, function* () {
const page = this.page;
const { accessResult, loginXhrUrl } = this.config.login;
const { userDetailsUrl, dataExtractionError } = this.config.getServiceAccounts;
try {
const loginXhr = yield page.waitForResponse((request) => request.url().startsWith(loginXhrUrl));
try {
const res = yield loginXhr.json();
if (res.isLoginSuccess === false && res.error === accessResult) {
yield this.browser.close();
return Promise.reject(new typings_1.Errors.AuthenticationError(accessResult));
}
}
catch (err) {
this.logger.error(`[APS] Login XHR JSON error: ${err}`);
}
const userDetailsHTTPResponse = yield page.waitForResponse(userDetailsUrl);
const userDetails = yield userDetailsHTTPResponse.json();
// userDetails = {
// Details: {
// profileData: {
// MainPersonName: 'Holmes,Sherlock',
// },
// AccountDetails: {
// getAccountDetailsResponse: {
// getAccountDetailsRes: {
// getPersonDetails: {
// accountID: '6076a3eb3d755200159dc776',
// },
// getSASPListByAccountID: {
// premiseDetailsList: [
// {
// premiseID: '6076a3eb3d755200159dc908',
// },
// ],
// },
// },
// },
// },
// },
// };
let name = userDetails.Details.profileData.MainPersonName.split(',');
name = name[1].concat(' ', name[0]);
const accountDetails = yield userDetails.Details.AccountDetails
.getAccountDetailsResponse.getAccountDetailsRes.getPersonDetails;
const id = yield accountDetails.accountID;
const premiseDetailsList = yield userDetails.Details.AccountDetails
.getAccountDetailsResponse.getAccountDetailsRes.getSASPListByAccountID
.premiseDetailsList;
const legalNames = accountDetails.results
.filter((tag, index, array) => array.findIndex((t) => t.personName === tag.personName && t.personID === tag.personID) === index)
.map((object) => object.personName.split(',').length == 2
? `${object.personName.split(',')[1]} ${object.personName.split(',')[0]}`
: object.personName);
const meterNumbersList = premiseDetailsList[0].sASPDetails
.filter((listItem) => listItem.sATypeDesc !== '' && listItem.sARatePlancCode !== '')
.map((user) => user.meterBadgeNumber);
const meterNumber = meterNumbersList[meterNumbersList.length - 1];
return { id, name, premiseDetailsList, legalNames, meterNumber };
}
catch (err) {
return Promise.reject(new typings_1.Errors.DataExtractionError(`${dataExtractionError}: ${err}`));
}
});
}
getBillsData() {
return __awaiter(this, void 0, void 0, function* () {
const page = this.page;
const { billDetailsPage, billDetailsPageSelector, billsDataSelector, billsDateStringSelector, selectStringSelector, selectValueStringSelector, } = this.config.getMonthlyUtilityData;
yield page.goto(billDetailsPage, { waitUntil: 'networkidle0' });
yield page.waitForSelector(billDetailsPageSelector);
const selectValueString = yield page.$eval(selectValueStringSelector, (ele) => {
return ele.textContent;
});
const selectValueInt = parseInt(selectValueString.split(' ')[7], 10);
const selectValueRaw = Math.ceil(selectValueInt / 10) * 10;
const selectValue = selectValueRaw.toString();
yield page.select(selectStringSelector, selectValue);
const billsData = yield page.$$(billsDataSelector);
const billsDateStringRaw = yield page.$$eval(billsDateStringSelector, (nodes) => nodes.map((n) => n.textContent));
// tslint:disable-next-line: max-line-length
// billsDateStringRaw = [' Sep 08, 2021 ', ' Aug 09, 2021 ', ' Jul 08, 2021 ', ' Jun 08, 2021 ', ' May 06, 2021 ']
const billsDateString = [];
for (const date of billsDateStringRaw) {
const monthYear = date.split(' ');
const monthAndYear = monthYear[1].concat('_').concat(monthYear[3]);
billsDateString.push(monthAndYear);
}
return { billsData, billsDateString };
});
}
extractPdfData(downloadPath, billsDateString, externalServiceAccountId) {
return __awaiter(this, void 0, void 0, function* () {
const { dataExtractionError } = this.config.getMonthlyUtilityData;
try {
const pdfExtract = new pdf_js_extract_1.PDFExtract();
const extractedBillData = yield pdfExtract.extract(`${downloadPath}/bill.pdf`);
this.logger.info(`[APS] Extracted data for pdf ${downloadPath}/bill.pdf`);
const pageContent = extractedBillData.pages[2].content;
// pageContent = [
// {
// x: 37.20000000000002,
// y: 520.3199999999998,
// str: 'Cost of electricity with taxes and fees',
// dir: 'ltr',
// width: 150.00000000000003,
// height: 10.32,
// fontName: 'g_d0_f375R',
// },
// {
// x: 273.12,
// y: 520.3199999999998,
// str: '$77.46',
// dir: 'ltr',
// width: 27.84,
// height: 10.32,
// fontName: 'g_d0_f375R',
// },
// {
// x: 314.4,
// y: 196.80000000000018,
// str: 'Total electricity you used, in kWh',
// dir: 'ltr',
// width: 144.48,
// height: 10.799999999999999,
// fontName: 'g_d0_f6R',
// },
// {
// x: 566.88,
// y: 197.5200000000002,
// str: '448',
// dir: 'ltr',
// width: 16.56,
// height: 11.76,
// fontName: 'g_d0_f918R',
// },
// ];
const totalUsageObject = this.extractfromYCoordinates(pageContent, {
y1: 197.52,
y2: 234.96,
});
const totalUsage = this.convertToString(totalUsageObject);
// extracting usage Amount value
const usageAmountObject = this.extractFromXCoordinates(pageContent, {
x1: 273.12,
x2: 268.08,
});
const usageAmount = usageAmountObject[usageAmountObject.length - 1].str.slice(1);
// extracting onPeakUsage value
const onPeakUsageObject = this.extractfromYCoordinates(pageContent, {
y1: 246.96,
y2: null,
});
const onPeakUsageInKW = this.convertToString(onPeakUsageObject);
// extracting offPeakUsage value
const offPeakUsageObject = this.extractfromYCoordinates(pageContent, {
y1: 331.92,
y2: null,
});
const offPeakUsageInKW = this.convertToString(offPeakUsageObject);
const monthAndYear = billsDateString;
const rawMonth = monthAndYear.slice(0, 3);
// tslint:disable-next-line: radix
const rawYear = parseInt(monthAndYear.slice(4));
const date = new Date(`${rawMonth},${rawYear}`);
date.setMonth(date.getMonth() - 1);
const month = date.toLocaleString('default', { month: 'long' });
const year = date.getFullYear();
const amountInCents = Math.round(parseFloat(usageAmount) * 100);
const energyConsumptionInWatts = totalUsage
? this.convertToWatts(totalUsage)
: totalUsage;
const offPeakUsageInWatts = offPeakUsageInKW
? this.convertToWatts(offPeakUsageInKW)
: offPeakUsageInKW;
const onPeakUsageInWatts = onPeakUsageInKW
? this.convertToWatts(onPeakUsageInKW)
: onPeakUsageInKW;
// tslint:disable-next-line: radix
const monthlyData = new models_1.MonthlyUtilityData({
amountInCents,
energyConsumptionInWatts,
month,
offPeakUsageInWatts,
onPeakUsageInWatts,
year,
serviceAccountId: externalServiceAccountId,
});
return monthlyData;
}
catch (err) {
return Promise.reject(new typings_1.Errors.DataExtractionError(`${dataExtractionError}: ${err}`));
}
});
}
getUtilityBillPDFs(serviceAccountId, upload) {
return __awaiter(this, void 0, void 0, function* () {
const page = this.page;
const { billDetailsUrl } = this.config.getMonthlyUtilityData;
try {
yield page.waitForNavigation();
const billsDataObject = yield this.getBillsData();
const { billsData, billsDateString } = billsDataObject;
let index = -1;
for (const bill of billsData) {
try {
index += 1;
if (billsDateString[index] !== billsDateString[index - 1]) {
const relativeDownloadFilePath = `service_acccounts/${serviceAccountId}/${helpers_1.DateHelper.formatBillDate(billsDateString[index])}`;
const downloadPath = path_1.default.resolve(__dirname, `/tmp/bills/${relativeDownloadFilePath}`); // /tmp/bills/service_accounts/634900de7f2d174aacfd29c3/2022_1
const client = yield page.target().createCDPSession();
yield client.send('Page.setDownloadBehavior', {
downloadPath,
behavior: 'allow',
});
yield page.evaluate((b) => b.click(), bill);
yield page.waitForResponse((res) => res.url().startsWith(billDetailsUrl));
this.logger.info(`[APS] Downloading pdf ${index}`);
yield new Promise((resolve) => setTimeout(resolve, config_1.default.get('apsDownloadPDF.timeOut')));
const uploadPath = `bills/${relativeDownloadFilePath}`;
yield upload({
filename: 'bill.pdf',
mimetype: 'application/pdf',
path: downloadPath,
}, uploadPath);
fs_1.default.unlinkSync(`${downloadPath}/bill.pdf`);
}
}
catch (err) {
this.logger.error(`[APS]: Bill pdf error: ${err.message}`);
}
}
yield this.browser.close();
}
catch (err) {
this.logger.info(`Downloading PDF bills err: ${err.message}`);
this.browser.close();
}
});
}
setPageOnEvent(page, tunnelingAgentOptions) {
const { loginXhrUrl } = this.config.login;
let cookies = [];
let cookiesFlag;
const logger = this.logger;
// Uncomment below code to see network logs
// page.on('response', async (interceptedResponse) => {
// this.logger.info(
// `[SRP] Response per Request: { statusCode: ${interceptedResponse.status()},
// statusText: ${interceptedResponse.statusText()},
// method: ${interceptedResponse.request().method()},
// url: ${interceptedResponse.url()},}`,
// );
// });
page.on('request', (interceptedRequest) => __awaiter(this, void 0, void 0, function* () {
if (interceptedRequest.url() === loginXhrUrl) {
const tunnelingAgent = tunnel_1.default.httpsOverHttp(tunnelingAgentOptions);
const interceptedRequestHeaders = interceptedRequest.headers();
// Replacing HeadlessChrome with just Chrome. Because HeadlessChrome is resulting as an invalid user-agent header on hitting request.
interceptedRequestHeaders['user-agent'] = interceptedRequestHeaders['user-agent'].replace('Headless', '');
const options = {
uri: interceptedRequest.url(),
method: interceptedRequest.method(),
headers: interceptedRequestHeaders,
agent: tunnelingAgent,
body: interceptedRequest.postData(),
};
(0, request_1.default)(options, function (err, res, body) {
return __awaiter(this, void 0, void 0, function* () {
if (err) {
logger.error(`[APS] Page ON event request error: ${err.message}`);
return;
}
const setCookies = res.headers['set-cookie'];
for (let i = 0; setCookies && i < setCookies.length; i++) {
const value = setCookies[i];
// Here the string "value" is in this form - "<name>=<value>; ..."
const cookie = value.split('; ')[0].split('=');
// Considering only the non utm cookies. All the utm cookies will be in the end of setCookies.
if (cookie[0].startsWith('___utm')) {
break;
}
cookies.push({ name: cookie[0], value: cookie[1] });
}
try {
yield page.setCookie(...cookies);
}
catch (err) {
logger.error(`[APS] Page set cookie error: ${err.message}`);
}
cookiesFlag = true;
interceptedRequest.respond({
status: res.statusCode,
contentType: res.headers['content-type'],
headers: res.headers,
body: body,
});
});
});
}
else {
if (cookiesFlag) {
try {
yield page.setCookie(...cookies);
}
catch (err) {
logger.error(`[APS] Page set cookie error: ${err.message}`);
}
}
interceptedRequest.continue();
}
}));
}
extractfromYCoordinates(pageContent, coordinates) {
const textValueObject = pageContent.filter((el) => {
return (Math.round(el.y * 100) / 100 === coordinates.y1 ||
Math.round(el.y * 100) / 100 === coordinates.y2);
});
return textValueObject;
}
extractFromXCoordinates(pageContent, coordinates) {
const textValueObject = pageContent.filter((el) => {
return (Math.round(el.x * 100) / 100 === coordinates.x1 ||
Math.round(el.x * 100) / 100 === coordinates.x2);
});
return textValueObject;
}
convertToString(object) {
try {
let resultantStringValue = object[object.length - 1].str;
return resultantStringValue;
}
catch (err) {
return null;
}
}
convertToWatts(data) {
try {
const value = parseInt(data) * 1000;
return value;
}
catch (err) {
return null;
}
}
}
exports.APSWorkflow = APSWorkflow;