@nqminds/fin-genie-gl-reconciliator/lib/gl-reconciliator.js

A databot for general ledger reconciliation for FinGenie

"use strict";

const FinGenieDatabot = require("@nqminds/fin-genie-databot");
const {chunk} = require("lodash");
const split = require("split");

const verifyTransactionComplete = require("./verify-transaction-complete");
const pipelines = require("./transaction-matching-pipelines");
const formatTransaction = require("./format-transaction");

class GlReconciliator extends FinGenieDatabot {
  constructor(input, output, context) {
    super(input, output, context);
    this.progress = 0;
  }
  async main() {
    try {
      const glDatasetId = this.getDatasetId("generalLedger", this.input.auditId);
      this.stage = "Getting aggregated general ledger data";
      const aggregatedTransactions = await Promise.all([
        this.getAggregatedData(glDatasetId, pipelines.jrn),
        this.getAggregatedData(glDatasetId, pipelines.po),
        this.getAggregatedData(glDatasetId, pipelines.inv),
      ]);
      this.updateProgress(60);
      this.stage = "Building grouped transactions";
      const completeTransactions = aggregatedTransactions.flat().filter(verifyTransactionComplete);
      const nEntries = completeTransactions.reduce((sum, {glEntries}) => sum += glEntries.length, 0);
      this.output.debug(`${nEntries} entries grouped into ${completeTransactions.length} transactions`);
      this.stage = "Uploading transaction data";
      const formattedTransactions = completeTransactions.map(formatTransaction);
      await this.uploadTransactionDataset(formattedTransactions);
      this.updateProgress(40);
      return Promise.resolve();
    } catch (err) {
      await this.reportBug(err);
      this.output.result({error: err, stage: this.stage});
      return Promise.resolve();
    }
  }

  async getAggregatedData(glDatasetId, pipeline) {
    const {body: dataStream} = await this.api.getAggregateDataStream(
      glDatasetId, pipeline, true,
    );
    return new Promise((resolve, reject) => {
      const data = [];

      dataStream.pipe(split(JSON.parse, null, {trailing: true}))
        .on("data", async(document) => data.push(document))
        .on("end", () => resolve(data))
        .on("error", (err) => {
          // TDX sends a new line even if the dataset is empty
          if (err.message === "Unexpected end of JSON input" && data.length === 0) {
            resolve(data);
          } else {
            reject(err);
          }
        });
    });
  }

  uploadTransactionDataset(transactions) {
    this.output.debug("Uploading to transactions dataset");
    const transactionsDatasetId = this.getDatasetId("transactions", this.input.auditId);
    return this.uploadAndImportJson(transactions, transactionsDatasetId, this.input.auditId);
  }

  async updateGeneralLedger(transactions) {
    this.output.debug("Updating general ledger");
    const progressIncrement = 50 / transactions.length; // Allocate 50% of total run time to general ledger update
    const maxBatchSize = 100; // Avoid exceeding http request size limit

    for (const {id, glEntries} of transactions) {
      const batches = chunk(glEntries, maxBatchSize);
      // eslint-disable-next-line no-await-in-loop
      await Promise.all(batches.map((batch) => this.api.updateDataByQuery(
        this.getDatasetId("generalLedger", this.input.auditId),
        {id: {$in: batch}}, {transaction: id},
      )));
      this.updateProgress(progressIncrement);
    }
  }

  updateProgress(progressIncrement) {
    const oldProgress = this.progress;
    this.progress += progressIncrement;
    if (Math.round(oldProgress * 10) !== Math.round(this.progress * 10)) {
      this.output.progress(this.progress);
      this.output.debug(`${this.progress}% complete`);
    }
  }
}

module.exports = GlReconciliator;