@accounter/modern-poalim-scraper
Version:
Modern scraper for Israeli banks (Hapoalim, Isracard, Max)
406 lines • 19.7 kB
JavaScript
import { closeSync, openSync } from 'node:fs';
import { createInterface } from 'node:readline/promises';
import { ReadStream, WriteStream } from 'node:tty';
import { captureMytradeSession, fetchGetWithinPage, fetchPoalimMytradeWithinPage, fetchPoalimXSRFWithinPage, } from '../utils/fetch.js';
import { HapoalimAccountDataSchema, } from '../zod-schemas/hapoalim-account-data-schema.js';
import { HapoalimDepositsSchema, } from '../zod-schemas/hapoalim-deposits-schema.js';
import { HapoalimForeignDepositsSchema, } from '../zod-schemas/hapoalim-foreign-deposits-schema.js';
import { HapoalimForeignTransactionsBusinessSchema, } from '../zod-schemas/hapoalim-foreign-transactions-business-schema.js';
import { HapoalimForeignTransactionsPersonalSchema, } from '../zod-schemas/hapoalim-foreign-transactions-personal-schema.js';
import { HapoalimILSTransactionsSchema, } from '../zod-schemas/hapoalim-ils-checking-transactions-schema.js';
import { HapoalimSecuritiesSchema, } from '../zod-schemas/hapoalim-securities-schema.js';
import { SwiftTransactionSchema, } from '../zod-schemas/swift-transaction-schema.js';
import { SwiftTransactionsSchema, } from '../zod-schemas/swift-transactions-schema.js';
/**
* Reads a line from the user.
*
* `process.stdin` is only usable when the process was started with an
* interactive stdin. Launched from an IDE run/debug console, from a wrapper
* that pipes stdin, or with stdin redirected, nothing you type ever reaches
* this process and the prompt hangs forever. In that case read the
* controlling terminal (`/dev/tty`) directly instead.
*/
async function promptInput(message) {
let input = process.stdin;
let output = process.stdout;
const fds = [];
if (!process.stdin.isTTY) {
try {
const inFd = openSync('/dev/tty', 'r');
const outFd = openSync('/dev/tty', 'w');
fds.push(inFd, outFd);
input = new ReadStream(inFd);
output = new WriteStream(outFd);
console.warn('stdin is not a TTY — reading the prompt from /dev/tty instead.');
}
catch {
console.warn('stdin is not a TTY and /dev/tty is unavailable. ' +
'Run this script directly in a terminal, or pass an `otpCallback`.');
}
}
const rl = createInterface({ input, output, terminal: true });
try {
return (await rl.question(`${message} `)).trim();
}
finally {
rl.close();
for (const fd of fds) {
closeSync(fd);
}
}
}
async function businessLogin(credentials, page, otpCallback) {
const BASE_URL = 'https://biz2.bankhapoalim.co.il/ng-portals/auth/he/biz-login/authenticate';
await page.goto(BASE_URL);
await page.waitForSelector('.submit-btn');
await page.type('#user-code', credentials.userCode);
await page.type('#password', credentials.password);
await page.click('.submit-btn');
// wait for the OTP screen before asking for the code, so the input is there
// by the time we type it in
await page.waitForSelector('input[formcontrolname="code"]', { timeout: 30_000 });
let otp;
if (otpCallback) {
otp = await otpCallback();
}
else {
otp = await promptInput('Enter the code you got in SMS:');
}
await page.type('input[formcontrolname="code"]', otp);
await Promise.all([
page.waitForNavigation(),
page.keyboard.press('Enter'),
page.click('button.next'),
]);
}
async function personalLogin(credentials, page) {
const BASE_URL = 'https://login.bankhapoalim.co.il/ng-portals/auth/he/';
await page.goto(BASE_URL);
const userCode = credentials.userCode;
const password = credentials.password;
await page.waitForSelector('.login-btn');
await page.type('#userCode', userCode);
await page.type('#password', password);
await page.click('.login-btn');
await page.waitForNavigation();
return 0;
}
async function replacePassword(previousCredentials, page, otpCallback) {
await page.waitForSelector('#buttonAction');
let newPassword;
if (otpCallback) {
newPassword = await otpCallback();
}
else {
newPassword = await promptInput('Enter your new wanted password:');
}
await page.type('[name="oldpassword"]', previousCredentials.password);
await page.type('[name="newpassword"]', newPassword);
await page.type('[name="newpassword2"]', newPassword);
await Promise.all([page.waitForNavigation(), page.keyboard.press('Enter')]);
await page.waitForSelector('#linkToHomePage');
await Promise.all([
page.waitForNavigation(),
page.keyboard.press('Enter'),
page.click('#linkToHomePage'),
]);
return 0;
}
export async function hapoalim(page, credentials, options) {
if (options?.isBusiness) {
await businessLogin(credentials, page, options.otpCallback);
}
else {
await personalLogin(credentials, page);
}
const result = await page.evaluate(() => {
if (window?.bnhpApp?.restContext) {
return window.bnhpApp.restContext;
}
return 'nothing';
});
// Example replace password url:
// https://biz2.bankhapoalim.co.il/ABOUTTOEXPIRE/START?flow=ABOUTTOEXPIRE&state=START&expiredDate=11122020
if (result === 'nothing') {
if (page.url().search('ABOUTTOEXPIRE') === -1) {
return 'Unknown Error';
}
await replacePassword(credentials, page, options?.otpCallback);
}
const apiSiteUrl = `https://${options?.isBusiness ? 'biz2' : 'login'}.bankhapoalim.co.il/${result.slice(1)}`;
const now = new Date();
const startMonth = options?.duration ?? 12;
const startDate = new Date(now.getFullYear(), now.getMonth() - startMonth, now.getDate() + 1);
const startDateString = startDate.toISOString().substr(0, 10).replace(/-/g, '');
const endDateString = new Date().toISOString().substr(0, 10).replace(/-/g, '');
// TODO: https://www.npmjs.com/package/node-fetch-cookies
return {
getAccountsData: async () => {
const accountDataUrl = `${apiSiteUrl}/general/accounts`;
const getAccountsFunction = fetchGetWithinPage(page, accountDataUrl);
if (options?.validateSchema) {
const data = await getAccountsFunction;
const validation = HapoalimAccountDataSchema.safeParse(data);
return {
data: validation.data ?? null,
isValid: validation.success,
errors: validation.success ? null : validation.error.issues,
};
}
return { data: await getAccountsFunction, isValid: null };
},
getILSTransactions: async (account) => {
const fullAccountNumber = `${account.bankNumber}-${account.branchNumber}-${account.accountNumber}`;
const ILSCheckingTransactionsUrl = `${apiSiteUrl}/current-account/transactions?accountId=${fullAccountNumber}&numItemsPerPage=200&retrievalEndDate=${endDateString}&retrievalStartDate=${startDateString}&sortCode=1`;
const getIlsTransactionsFunction = fetchPoalimXSRFWithinPage(page, ILSCheckingTransactionsUrl, '/current-account/transactions');
if (options?.validateSchema || options?.getTransactionsDetails) {
const data = await getIlsTransactionsFunction;
if (options?.getTransactionsDetails && data != null) {
for (const transaction of data?.transactions ?? []) {
if (transaction.pfmDetails) {
/* let a = */ await fetchPoalimXSRFWithinPage(page, ILSCheckingTransactionsUrl, transaction.pfmDetails);
// TODO: create schema and make this attribute string / object for inputing data
}
if (transaction.details) {
/*let b = */ await fetchPoalimXSRFWithinPage(page, ILSCheckingTransactionsUrl, transaction.details);
// TODO: create schema and make this attribute string / object for inputing data
}
}
if (!options?.validateSchema) {
return { data, isValid: null };
}
}
if (!data) {
console.log(`No ILS data found for account ${fullAccountNumber}`);
return {
data,
isValid: true,
};
}
const validation = HapoalimILSTransactionsSchema.safeParse(data);
return {
data: validation.data ?? null,
isValid: validation.success,
errors: validation.success ? null : validation.error.issues,
};
}
return { data: await getIlsTransactionsFunction, isValid: null };
},
getForeignTransactions: async (account, isBusiness = true) => {
/**
* The URL includes optional "type" query param.
* Setting it to "business" changes the response type.
* Since accounter uses the business-specific fields,
* the non-business option is commented out
* */
const fullAccountNumber = `${account.bankNumber}-${account.branchNumber}-${account.accountNumber}`;
const foreignTransactionsUrl = `${apiSiteUrl}/foreign-currency/transactions?accountId=${fullAccountNumber}${isBusiness ? '&type=business' : ''}&view=details&retrievalEndDate=${endDateString}&retrievalStartDate=${startDateString}¤cyCodeList=19,27,51,100,140,248&detailedAccountTypeCodeList=142&lang=he`;
const getForeignTransactionsFunction = fetchGetWithinPage(page, foreignTransactionsUrl);
if (options?.validateSchema) {
const data = await getForeignTransactionsFunction;
if (data &&
data['messageCode'] === 0 &&
data['severity'] === 'E') {
return {
data,
errors: 'Data seems unreachable. Is the account active?',
isValid: false,
};
}
if (isBusiness) {
const validation = HapoalimForeignTransactionsBusinessSchema.safeParse(data);
return {
data: validation.data ?? null,
isValid: validation.success,
errors: validation.success ? null : validation.error.issues,
};
}
const validation = HapoalimForeignTransactionsPersonalSchema.safeParse(data);
return {
data: validation.data ?? null,
isValid: validation.success,
errors: validation.success ? null : validation.error.issues,
};
}
return { data: await getForeignTransactionsFunction, isValid: null };
},
getForeignSwiftTransactions: async (account) => {
const fullAccountNumber = `${account.bankNumber}-${account.branchNumber}-${account.accountNumber}`;
const foreignSwiftTransactionsUrl = `${apiSiteUrl}/foreign-trade/swiftTransactions?accountId=${fullAccountNumber}&endDate=${endDateString}&startDate=20190501`;
const getForeignSwiftTransactionsFunction = fetchGetWithinPage(page, foreignSwiftTransactionsUrl);
if (options?.validateSchema) {
const data = await getForeignSwiftTransactionsFunction;
if (data &&
data['messageCode'] === 0 &&
data['severity'] === 'E') {
return {
data,
errors: 'Data seems unreachable. Is the account active?',
isValid: false,
};
}
const validation = SwiftTransactionsSchema.safeParse(data);
return {
data: validation.data ?? null,
isValid: validation.success,
errors: validation.error,
};
}
return { data: await getForeignSwiftTransactionsFunction, isValid: null };
},
getForeignSwiftTransaction: async (account, transferCatenatedId, dataOriginCode) => {
const fullAccountNumber = `${account.bankNumber}-${account.branchNumber}-${account.accountNumber}`;
const foreignSwiftTransactionUrl = `${apiSiteUrl}/foreign-trade/swiftTransactions/${transferCatenatedId}?accountId=${fullAccountNumber}&dataOriginCode=${dataOriginCode}&lang=he`;
const getForeignSwiftTransactionFunction = fetchGetWithinPage(page, foreignSwiftTransactionUrl);
if (options?.validateSchema) {
const data = await getForeignSwiftTransactionFunction;
if (data &&
data['messageCode'] === 0 &&
data['severity'] === 'E') {
return {
data,
errors: 'Data seems unreachable. Is the account active?',
isValid: false,
};
}
const validation = SwiftTransactionSchema.safeParse(data);
return {
data,
isValid: validation.success,
errors: validation.error,
};
}
return { data: await getForeignSwiftTransactionFunction, isValid: null };
},
getDeposits: async (account) => {
const fullAccountNumber = `${account.bankNumber}-${account.branchNumber}-${account.accountNumber}`;
const depositsUrl = `${apiSiteUrl}/deposits-and-savings/deposits?accountId=${fullAccountNumber}&view=details&lang=he`;
const getDepositsFunction = fetchGetWithinPage(page, depositsUrl);
if (options?.validateSchema) {
const data = await getDepositsFunction;
if (!data) {
console.log(`No deposits data found for account ${fullAccountNumber}`);
return {
data,
isValid: true,
};
}
const validation = HapoalimDepositsSchema.safeParse(data);
return {
data: validation.data ?? null,
isValid: validation.success,
errors: validation.success ? null : validation.error.issues,
};
}
return { data: await getDepositsFunction, isValid: null };
},
getForeignDeposits: async (account) => {
const fullAccountNumber = `${account.bankNumber}-${account.branchNumber}-${account.accountNumber}`;
const depositsUrl = `${apiSiteUrl}/foreign-currency/revaluedDeposit?accountId=${fullAccountNumber}&lang=he`;
const getDepositsFunction = fetchGetWithinPage(page, depositsUrl);
if (options?.validateSchema) {
const data = await getDepositsFunction;
const validation = HapoalimForeignDepositsSchema.safeParse(data);
return {
data: validation.data ?? null,
isValid: validation.success,
errors: validation.success ? null : validation.error.issues,
};
}
return { data: await getDepositsFunction, isValid: null };
},
/**
* Static securities reference info from the "mytrade" portfolio app.
* Live balances and open orders in the same response are ignored.
*
* Unlike every other Poalim endpoint this one addresses the account as
* `branch-account` (no bank number), and is only served to callers running
* on the mytrade SPA's own page — so the request is issued from a
* short-lived sibling tab sharing the logged-in browser context.
*/
getSecurities: async (account) => {
const tradeAccountNumber = `${account.branchNumber}-${account.accountNumber}`;
const fields = [
'EngName',
'EngSymbol',
'HebName',
'HebSymbol',
'Symbol',
'ExpirationDate',
'ItemType',
'StockType',
'IsEtf',
'IsForeign',
'CurrencyCode',
'Exchange',
'CreationEquityNum',
'EquityType',
'ContractType',
'AllowedOrderDirection',
'EquitySubType',
].join(',');
const securitiesUrl = `${apiSiteUrl}/mytrade/api/v2/json2/account/view?account=${tradeAccountNumber}&fields=${fields}`;
const tradePage = await page.browser().newPage();
let data;
try {
// The session key is minted server-side when the SPA boots, so listen
// for it before navigating and reuse it — see captureMytradeSession.
const sessionPromise = captureMytradeSession(tradePage);
// Same origin as every other call; only the REST context is dropped,
// since the SPA is served from the site root rather than under it.
await tradePage.goto(`${new URL(apiSiteUrl).origin}/mytrade/app`, {
waitUntil: 'networkidle2',
});
const mytradeSession = await sessionPromise;
data = await fetchPoalimMytradeWithinPage(tradePage, securitiesUrl, mytradeSession);
}
finally {
await tradePage.close();
}
if (!options?.validateSchema) {
return { data, isValid: null };
}
if (!data) {
console.log(`No securities data found for account ${tradeAccountNumber}`);
return { data, isValid: true };
}
if (data['messageCode'] === 0 &&
data['severity'] === 'E') {
return {
data,
errors: 'Data seems unreachable. Is the account active?',
isValid: false,
};
}
// e.g. { Exception: { "-ExceptionType": "InvalidSessionException", Message, SessionKey } }
const exception = data['Exception'];
if (exception) {
return {
data,
errors: `Poalim mytrade error${exception['-ExceptionType'] ? ` (${exception['-ExceptionType']})` : ''}: ${exception['Message'] ?? 'unknown'}`,
isValid: false,
};
}
const validation = HapoalimSecuritiesSchema.safeParse(data);
return {
data: validation.data ?? null,
isValid: validation.success,
errors: validation.success ? null : validation.error.issues,
};
},
};
}
export class HapoalimOptions {
validateSchema = false;
isBusiness = true;
duration = 12;
getTransactionsDetails = false;
otpCallback = undefined;
}
class HapoalimPersonalCredentials {
userCode = '';
password = '';
}
class HapoalimBusinessCredentials extends HapoalimPersonalCredentials {
}
//# sourceMappingURL=hapoalim.js.map