@accounter/modern-poalim-scraper
Version:
Modern scraper for Israeli banks (Hapoalim, Isracard, Max)
472 lines • 19.9 kB
JavaScript
import { addMonths, format, isAfter, max as maxDate, startOfMonth, subYears } from 'date-fns';
import { fetchGetWithinPage } from '../utils/fetch.js';
import { maxCategoriesSchema } from './types/max/get-categories.js';
import { maxTransactionsSchema, } from './types/max/get-transactions-for-month.js';
var ScraperErrorTypes;
(function (ScraperErrorTypes) {
ScraperErrorTypes["TwoFactorRetrieverMissing"] = "TWO_FACTOR_RETRIEVER_MISSING";
ScraperErrorTypes["InvalidPassword"] = "INVALID_PASSWORD";
ScraperErrorTypes["ChangePassword"] = "CHANGE_PASSWORD";
ScraperErrorTypes["AccountBlocked"] = "ACCOUNT_BLOCKED";
})(ScraperErrorTypes || (ScraperErrorTypes = {}));
var LoginBaseResults;
(function (LoginBaseResults) {
LoginBaseResults["Success"] = "SUCCESS";
LoginBaseResults["UnknownError"] = "UNKNOWN_ERROR";
})(LoginBaseResults || (LoginBaseResults = {}));
const LoginResults = {
...ScraperErrorTypes,
...LoginBaseResults,
};
var TransactionTypes;
(function (TransactionTypes) {
TransactionTypes["Normal"] = "normal";
TransactionTypes["Installments"] = "installments";
})(TransactionTypes || (TransactionTypes = {}));
function getInstallmentsInfo(comments) {
if (!comments) {
return undefined;
}
const matches = comments.match(/\d+/g);
if (!matches || matches.length < 2) {
return undefined;
}
return {
number: parseInt(matches[0], 10),
total: parseInt(matches[1], 10),
};
}
var MaxPlanName;
(function (MaxPlanName) {
MaxPlanName["Normal"] = "\u05E8\u05D2\u05D9\u05DC\u05D4";
MaxPlanName["ImmediateCharge"] = "\u05D7\u05D9\u05D5\u05D1 \u05E2\u05E1\u05E7\u05D5\u05EA \u05DE\u05D9\u05D9\u05D3\u05D9";
MaxPlanName["InternetShopping"] = "\u05D0\u05D9\u05E0\u05D8\u05E8\u05E0\u05D8/\u05D7\u05D5\"\u05DC";
MaxPlanName["Installments"] = "\u05EA\u05E9\u05DC\u05D5\u05DE\u05D9\u05DD";
MaxPlanName["MonthlyCharge"] = "\u05D7\u05D9\u05D5\u05D1 \u05D7\u05D5\u05D3\u05E9\u05D9";
MaxPlanName["OneMonthPostponed"] = "\u05D3\u05D7\u05D5\u05D9 \u05D7\u05D5\u05D3\u05E9";
MaxPlanName["MonthlyPostponed"] = "\u05D3\u05D7\u05D5\u05D9 \u05DC\u05D7\u05D9\u05D5\u05D1 \u05D4\u05D7\u05D5\u05D3\u05E9\u05D9";
MaxPlanName["MonthlyPayment"] = "\u05EA\u05E9\u05DC\u05D5\u05DD \u05D7\u05D5\u05D3\u05E9\u05D9";
MaxPlanName["FuturePurchaseFinancing"] = "\u05DE\u05D9\u05DE\u05D5\u05DF \u05DC\u05E8\u05DB\u05D9\u05E9\u05D4 \u05E2\u05EA\u05D9\u05D3\u05D9\u05EA";
MaxPlanName["MonthlyPostponedInstallments"] = "\u05D3\u05D7\u05D5\u05D9 \u05D7\u05D5\u05D3\u05E9 \u05EA\u05E9\u05DC\u05D5\u05DE\u05D9\u05DD";
MaxPlanName["ThirtyDaysPlus"] = "\u05E2\u05E1\u05E7\u05EA 30 \u05E4\u05DC\u05D5\u05E1";
MaxPlanName["TwoMonthsPostponed"] = "\u05D3\u05D7\u05D5\u05D9 \u05D7\u05D5\u05D3\u05E9\u05D9\u05D9\u05DD";
MaxPlanName["TwoMonthsPostponed2"] = "\u05D3\u05D7\u05D5\u05D9 2 \u05D7' \u05EA\u05E9\u05DC\u05D5\u05DE\u05D9\u05DD";
MaxPlanName["MonthlyChargePlusInterest"] = "\u05D7\u05D5\u05D3\u05E9\u05D9 + \u05E8\u05D9\u05D1\u05D9\u05EA";
MaxPlanName["Credit"] = "\u05E7\u05E8\u05D3\u05D9\u05D8";
MaxPlanName["CreditOutsideTheLimit"] = "\u05E7\u05E8\u05D3\u05D9\u05D8-\u05DE\u05D7\u05D5\u05E5 \u05DC\u05DE\u05E1\u05D2\u05E8\u05EA";
MaxPlanName["AccumulatingBasket"] = "\u05E1\u05DC \u05DE\u05E6\u05D8\u05D1\u05E8";
MaxPlanName["PostponedTransactionInstallments"] = "\u05E4\u05E8\u05D9\u05E1\u05EA \u05D4\u05E2\u05E1\u05E7\u05D4 \u05D4\u05D3\u05D7\u05D5\u05D9\u05D4";
MaxPlanName["ReplacementCard"] = "\u05DB\u05E8\u05D8\u05D9\u05E1 \u05D7\u05DC\u05D9\u05E4\u05D9";
MaxPlanName["EarlyRepayment"] = "\u05E4\u05E8\u05E2\u05D5\u05DF \u05DE\u05D5\u05E7\u05D3\u05DD";
MaxPlanName["MonthlyCardFee"] = "\u05D3\u05DE\u05D9 \u05DB\u05E8\u05D8\u05D9\u05E1";
MaxPlanName["CurrencyPocket"] = "\u05D7\u05D9\u05D5\u05D1 \u05D0\u05E8\u05E0\u05E7 \u05DE\u05D8\u05D7";
})(MaxPlanName || (MaxPlanName = {}));
function getTransactionType(planName, planTypeId) {
const cleanedUpTxnTypeStr = planName.replace('\t', ' ').trim();
switch (cleanedUpTxnTypeStr) {
case MaxPlanName.ImmediateCharge:
case MaxPlanName.Normal:
case MaxPlanName.MonthlyCharge:
case MaxPlanName.OneMonthPostponed:
case MaxPlanName.MonthlyPostponed:
case MaxPlanName.FuturePurchaseFinancing:
case MaxPlanName.MonthlyPayment:
case MaxPlanName.MonthlyPostponedInstallments:
case MaxPlanName.ThirtyDaysPlus:
case MaxPlanName.TwoMonthsPostponed:
case MaxPlanName.TwoMonthsPostponed2:
case MaxPlanName.AccumulatingBasket:
case MaxPlanName.InternetShopping:
case MaxPlanName.MonthlyChargePlusInterest:
case MaxPlanName.PostponedTransactionInstallments:
case MaxPlanName.ReplacementCard:
case MaxPlanName.EarlyRepayment:
case MaxPlanName.MonthlyCardFee:
case MaxPlanName.CurrencyPocket:
return TransactionTypes.Normal;
case MaxPlanName.Installments:
case MaxPlanName.Credit:
case MaxPlanName.CreditOutsideTheLimit:
return TransactionTypes.Installments;
default: {
/* empty */
}
}
switch (planTypeId) {
case 2:
case 3:
return TransactionTypes.Installments;
case 5:
return TransactionTypes.Normal;
default:
throw new Error(`Unknown transaction type ${cleanedUpTxnTypeStr}`);
}
}
function fixInstallments(txns) {
return txns.map((txn) => {
const clonedTxn = { ...txn };
const type = getTransactionType(clonedTxn.planName, clonedTxn.planTypeId);
const installments = getInstallmentsInfo(clonedTxn.comments);
if (type === TransactionTypes.Installments && installments && installments.number > 1) {
const date = new Date(clonedTxn.purchaseDate);
const actualDate = addMonths(date, installments.number - 1);
clonedTxn.purchaseDate = actualDate.toISOString();
}
return clonedTxn;
});
}
function sortTransactionsByDate(txns) {
return txns.sort((a, b) => a.purchaseDate.localeCompare(b.purchaseDate));
}
async function waitForNavigation(pageOrFrame, options) {
await pageOrFrame.waitForNavigation(options);
}
function getCurrentUrl(pageOrFrame, clientSide = false) {
if (clientSide) {
return pageOrFrame.evaluate(() => window.location.href);
}
return pageOrFrame.url();
}
const BASE_API_ACTIONS_URL = 'https://onlinelcapi.max.co.il';
const BASE_WELCOME_URL = 'https://www.max.co.il';
const LOGIN_URL = `${BASE_WELCOME_URL}/homepage/welcome`;
const PASSWORD_EXPIRED_URL = `${BASE_WELCOME_URL}/renew-password`;
const SUCCESS_URL = `${BASE_WELCOME_URL}/homepage/personal`;
const INVALID_DETAILS_SELECTOR = '#popupWrongDetails';
const LOGIN_ERROR_SELECTOR = '#popupCardHoldersLoginError';
const categories = new Map();
async function fillInput(pageOrFrame, inputSelector, inputValue) {
await pageOrFrame.$eval(inputSelector, (input) => {
const inputElement = input;
// @ts-expect-error force value
inputElement.value = '';
});
await pageOrFrame.type(inputSelector, inputValue);
}
async function clickButton(page, buttonSelector) {
await page.$eval(buttonSelector, el => el.click());
}
async function elementPresentOnPage(pageOrFrame, selector) {
return (await pageOrFrame.$(selector)) !== null;
}
async function waitUntilElementFound(page, elementSelector, onlyVisible = false) {
await page.waitForSelector(elementSelector, { visible: onlyVisible });
}
function getAllMonthDates(options) {
const defaultStartDate = subYears(new Date(), 3);
const startDate = options.startDate
? maxDate([defaultStartDate, options.startDate])
: defaultStartDate;
let monthDate = startOfMonth(startDate);
const allMonths = [];
let lastMonth = startOfMonth(new Date());
const { futureMonthsToScrape = 2 } = options;
if (futureMonthsToScrape && futureMonthsToScrape > 0) {
lastMonth = addMonths(lastMonth, futureMonthsToScrape);
}
while (isAfter(lastMonth, monthDate)) {
allMonths.push(monthDate);
monthDate = addMonths(monthDate, 1);
}
return allMonths;
}
function timeoutPromise(ms, promise, description) {
const timeout = new Promise((__, reject) => {
const id = setTimeout(() => {
clearTimeout(id);
reject(new Error(description));
}, ms);
});
return Promise.race([
promise,
// casting to avoid type error- safe since this promise will always reject
timeout,
]);
}
/**
* Wait until a promise resolves with a truthy value or reject after a timeout
*/
function waitUntil(asyncTest, description = '', timeout = 10_000, interval = 100) {
const promise = new Promise((resolve, reject) => {
function wait() {
asyncTest()
.then(value => {
if (value) {
resolve(value);
}
else {
setTimeout(wait, interval);
}
})
.catch(() => {
reject();
});
}
wait();
});
return timeoutPromise(timeout, promise, description);
}
async function waitForRedirect(pageOrFrame, timeout = 20_000, clientSide = false, ignoreList = []) {
const initial = await getCurrentUrl(pageOrFrame, clientSide);
await waitUntil(async () => {
const current = await getCurrentUrl(pageOrFrame, clientSide);
return current !== initial && !ignoreList.includes(current);
}, `waiting for redirect from ${initial}`, timeout, 1000);
}
function redirectOrDialog(page) {
return Promise.race([
waitForRedirect(page, 20_000, false, [BASE_WELCOME_URL, `${BASE_WELCOME_URL}/`]),
waitUntilElementFound(page, INVALID_DETAILS_SELECTOR, true),
waitUntilElementFound(page, LOGIN_ERROR_SELECTOR, true),
]);
}
function getTransactionsUrl(date) {
const stringDate = `${format(date, 'yyyy-MM')}-01`;
/**
* url explanation:
* userIndex: -1 for all account owners
* cardIndex: -1 for all cards under the account
* all other query params are static, beside the date which changes for request per month
*/
return `${BASE_API_ACTIONS_URL}/api/registered/transactionDetails/getTransactionsAndGraphs?filterData={"userIndex":-1,"cardIndex":-1,"monthView":true,"date":"${stringDate}","dates":{"startDate":"0","endDate":"0"},"bankAccount":{"bankAccountIndex":-1,"cards":null}}&firstCallCardIndex=-1`;
}
async function loadCategories(page) {
console.debug('Loading categories');
const res = await fetchGetWithinPage(page, `${BASE_API_ACTIONS_URL}/api/contents/getCategories`);
const parsedResponse = maxCategoriesSchema.safeParse(res);
if (!parsedResponse.success) {
throw new Error('Failed to parse response', parsedResponse.error);
}
const { result } = parsedResponse.data;
if (result) {
console.debug(`${result.length} categories loaded`);
result?.map(({ id, name }) => categories.set(id, name));
}
}
async function fetchTransactionsForMonth(page, monthDate) {
const url = getTransactionsUrl(monthDate);
const res = await fetchGetWithinPage(page, url);
const parsedResponse = maxTransactionsSchema.safeParse(res);
if (!parsedResponse.success) {
console.error('failed to parse response', parsedResponse.error);
throw new Error('Failed to parse response', parsedResponse.error);
}
const { result } = parsedResponse.data;
const transactionsByAccount = {};
if (!result)
return transactionsByAccount;
result.transactions
// Filter out non-transactions without a plan type, e.g. summary rows
.filter(transaction => !!transaction.planName)
.map(transaction => {
transactionsByAccount[transaction.shortCardNumber] ||= [];
transactionsByAccount[transaction.shortCardNumber]?.push(transaction);
});
return transactionsByAccount;
}
function addResult(allResults, result) {
const clonedResults = { ...allResults };
Object.keys(result).map(accountNumber => {
clonedResults[accountNumber] ||= [];
clonedResults[accountNumber].push(...(result[accountNumber] ?? []));
});
return clonedResults;
}
function prepareTransactions(txns) {
let clonedTxns = Array.from(txns);
clonedTxns = fixInstallments(clonedTxns);
clonedTxns = sortTransactionsByDate(clonedTxns);
return clonedTxns;
}
async function fetchTransactions(page, options) {
const allMonths = getAllMonthDates(options);
await loadCategories(page);
let allResults = {};
await Promise.all(allMonths.map(async (month) => await fetchTransactionsForMonth(page, month).then(result => {
allResults = addResult(allResults, result);
})));
Object.entries(allResults).map(([accountNumber, originalTxns]) => {
let txns = originalTxns;
txns = prepareTransactions(txns);
allResults[accountNumber] = txns;
});
const accounts = Object.entries(allResults).map(([accountNumber, txns]) => {
return {
accountNumber,
txns,
};
});
return accounts;
}
function getPossibleLoginResults(page) {
const urls = {};
urls[LoginResults.Success] = [SUCCESS_URL];
urls[LoginResults.ChangePassword] = [PASSWORD_EXPIRED_URL];
urls[LoginResults.InvalidPassword] = [
async () => {
return elementPresentOnPage(page, INVALID_DETAILS_SELECTOR);
},
];
urls[LoginResults.UnknownError] = [
async () => {
return elementPresentOnPage(page, LOGIN_ERROR_SELECTOR);
},
];
return urls;
}
function createLoginFields(credentials) {
return [
{ selector: '#user-name', value: credentials.username },
{ selector: '#password', value: credentials.password },
];
}
const OK_STATUS = 200;
async function getKeyByValue(object, value, page) {
const keys = Object.keys(object);
for (const key of keys) {
const conditions = object[key];
for (const condition of conditions ?? []) {
let result;
if (condition instanceof RegExp) {
result = condition.test(value);
}
else if (typeof condition === 'function') {
result = await condition({ page, value });
}
else {
result = value.toLowerCase() === condition.toLowerCase();
}
if (result) {
return Promise.resolve(key);
}
}
}
return Promise.resolve(LoginResults.UnknownError);
}
async function getLoginOptions(page, credentials) {
return {
loginUrl: LOGIN_URL,
fields: createLoginFields(credentials),
submitButtonSelector: '#login-password > app-user-login-form > form > div.button-wrapper > button',
preAction: async () => {
if (await elementPresentOnPage(page, '#closePopup')) {
await clickButton(page, '#closePopup');
}
await clickButton(page, '.personal-area > a.go-to-personal-area');
await waitUntilElementFound(page, '#private', true);
await clickButton(page, '#private');
await waitUntilElementFound(page, '#login-password-link', true);
await clickButton(page, '#login-password-link');
await waitUntilElementFound(page, '#login-password.tab-pane.active app-user-login-form', true);
},
checkReadiness: async () => {
await waitUntilElementFound(page, '.personal-area > a.go-to-personal-area', true);
},
postAction: async () => redirectOrDialog(page),
possibleResults: getPossibleLoginResults(page),
waitUntil: 'domcontentloaded',
};
}
function handleLoginResult(loginResult) {
switch (loginResult) {
case LoginResults.Success:
return true;
case LoginResults.InvalidPassword:
case LoginResults.UnknownError:
throw new Error(`Login failed with ${loginResult} error`);
case LoginResults.ChangePassword:
throw new Error('Change password');
default:
throw new Error(`unexpected login result "${loginResult}"`);
}
}
async function internalFillInputs(pageOrFrame, fields) {
const modified = [...fields];
const input = modified.shift();
if (!input) {
return;
}
await fillInput(pageOrFrame, input.selector, input.value);
if (modified.length) {
await internalFillInputs(pageOrFrame, modified);
}
}
async function navigateTo(url, page, waitUntilA = 'load') {
const options = {
waitUntil: waitUntilA,
};
const response = await page.goto(url, options);
// note: response will be null when navigating to same url while changing the hash part. the condition below will always accept null as valid result.
if (response !== null && (response === undefined || response.status() !== OK_STATUS)) {
throw new Error(`Error while trying to navigate to url ${url}`);
}
}
async function login(page, credentials) {
if (!credentials || !page) {
throw new Error('missing credentials or page');
}
console.debug('execute login process');
const loginOptions = await getLoginOptions(page, credentials);
if (loginOptions.userAgent) {
console.debug('set custom user agent provided in options');
await page.setUserAgent(loginOptions.userAgent);
}
console.debug('navigate to login url');
await navigateTo(loginOptions.loginUrl, page, loginOptions.waitUntil);
if (loginOptions.checkReadiness) {
console.debug("execute 'checkReadiness' interceptor provided in login options");
await loginOptions.checkReadiness();
}
else if (typeof loginOptions.submitButtonSelector === 'string') {
console.debug('wait until submit button is available');
await waitUntilElementFound(page, loginOptions.submitButtonSelector);
}
let loginFrameOrPage = page;
if (loginOptions.preAction) {
console.debug("execute 'preAction' interceptor provided in login options");
loginFrameOrPage = (await loginOptions.preAction()) || page;
}
console.debug('fill login components input with relevant values');
await internalFillInputs(loginFrameOrPage, loginOptions.fields);
console.debug('click on login submit button');
if (typeof loginOptions.submitButtonSelector === 'string') {
await clickButton(loginFrameOrPage, loginOptions.submitButtonSelector);
}
else {
await loginOptions.submitButtonSelector();
}
if (loginOptions.postAction) {
console.debug("execute 'postAction' interceptor provided in login options");
await loginOptions.postAction();
}
else {
console.debug('wait for page navigation');
await waitForNavigation(page);
}
console.debug('check login result');
const current = await getCurrentUrl(page, true);
const loginResult = await getKeyByValue(loginOptions.possibleResults, current, page);
console.debug(`handle login results ${loginResult}`);
return handleLoginResult(loginResult);
}
export async function max(page, credentials, options = {}) {
try {
await login(page, credentials);
}
catch (e) {
console.error(e);
throw new Error(e.message, { cause: e });
}
return {
getTransactions: async () => {
try {
return await fetchTransactions(page, options);
}
catch (e) {
console.error(e);
throw new Error(e.message, { cause: e });
}
},
};
}
//# sourceMappingURL=max.js.map