n26-puppeteer-scraper/lib/n26-read-transactions.js

n26 scraper: a tool to download account informations and transactions archive

// const megaScraper = require('mega-scraper')
// const pRetry = require('p-retry')
const debug = require('debug')
const log = debug('n26:read-transactions')

module.exports = n26ReadTransactions

async function n26ReadTransactions (page, options = {}) {
  await page.goto('https://app.n26.com/transactions')
  log('navigated to https://app.n26.com/transactions')

  const untilPage = options.untilPage || 1
  const txs = []

  for (let currentPage = 1; currentPage <= untilPage; currentPage++) {
    log('reading transactions on page', currentPage, untilPage)
    const transactionsTexts = await page.evaluate(
      () => [...document.querySelectorAll('main ul li')].map(elem => elem.innerText)
    )
    const transactionsLines = transactionsTexts
      .map(line => line.split('\n').filter(Boolean))
      .filter(Boolean)

    const newTxs = transactionsLines.map(l => ({
      title: l[0],
      dateString: l[1],
      type: l[2],
      amountString: l[3]
    }))
      .filter(t => t.title)

    txs.push(...newTxs)
    log(`txs.length ${txs.length} (+${newTxs.length} on page ${currentPage}/${untilPage})`)
    if (currentPage <= untilPage + 1) {
      log('clicking next page', currentPage, untilPage)
      log('clearing txs')
      await page.evaluate(async () => {
        const els = document.querySelectorAll('main ul li')
        if (els.length === 0) console.log(`empty txs!`)

        window.scrollTo(0, document.body.scrollHeight)

        for (const el of els) {
          el && el.getAttribute('title') !== 'Next' && el.scrollIntoView()
          await new Promise(resolve => setTimeout(resolve, 20))
          el && el.getAttribute('title') !== 'Next' && el.remove()
        }
      })

      log('clicking next button')
      await page.evaluate(() => {
        const el = document.querySelector('[title="Next"]')
        !el && console.log('could not find next button!')
        el && el.click()
        // el && el.scrollIntoView(true)
        window.scrollTo(0, document.body.scrollHeight)
      })

      if (currentPage <= untilPage) {
        let hasNewTxs
        const start = Date.now()
        do {
          hasNewTxs = await page.evaluate(() => !!document.querySelector('main ul li'))
          await new Promise(resolve => setTimeout(resolve, 200))
          hasNewTxs && log('loaded new transactions')
        } while (start > Date.now() - 1000 * 5 && !hasNewTxs)
        if (!hasNewTxs) log('seems like the transactions take longer too load than expected..')
      }
    }

    log('read txs', txs.length, txs[0] && txs[0].dateString, txs[txs.length - 1] && txs[txs.length - 1].dateString)
  }

  return txs
}