@nqminds/fin-genie-file-analyser/lib/file-analyser.js

A databot for analysing and tagging FinGenie files

"use strict";

const FinGenieDatabot = require("@nqminds/fin-genie-databot");
const {
  auditResourceTag, analysedResourceTag, auditResourceTypes, analysisErrorTag, extractFileType, fileTypes,
} = require("@nqminds/fin-genie-constants");
const {schemas} = require("@nqminds/fin-genie-schemas");
const isPlainObject = require("lodash.isplainobject");
const csv = require("fast-csv");
const fsAsync = require("fs").promises;
const {
  inferSchemaMapping,
  findMatchingTag,
  findExistingMatch,
} = require("./matching-functions");

/**
 * @typedef TdxResource
 * @property {string} id
 */

const auditResourceTags = Object.values(auditResourceTypes).map(({tag}) => tag);
const matchNumberRegex = /(($|[^\w])[0-9]+($|[^\w]))/; /* matches a number with a non-alphanumeric character,
or no character, on either side */

class FileAnalyser extends FinGenieDatabot {
  constructor(input, output, context) {
    input.pollPeriod = context.packageParams.pollPeriod;
    input.resourceQuery = JSON.parse(context.packageParams.resourceQuery);
    super(input, output, context);
    this.inProgress = [];
  }

  async main() {
    await this.poll();
  }

  async poll() {
    await this.asyncDelay(this.input.pollPeriod);
    await this.process();
    return this.poll();
  }

  async process() {
    this.output.debug("Checking for new resources");
    const [existingMappings, unmappedResources] = await this.loadData();
    this.output.debug(`Found ${unmappedResources.length} new resources`);
    this.inProgress.push(...unmappedResources.map(({id}) => id));
    for (const resource of unmappedResources) {
      this.processResource(resource, existingMappings);
    }
    this.output.debug(`${this.inProgress.length} files in progress`);
  }

  async processResource(resource, existingMappings) {
    try {
      this.output.debug(`Processing resource ${resource.name} - ${resource.id}`);

      if (extractFileType(resource.store) === fileTypes.evidence) {
        await this.handleEvidence(resource);
      } else {
        this.output.debug(`loading resource ${resource.name}`);
        const content = await this.getExampleContent(resource);
        this.output.debug(`loaded resource ${resource.name}`);

        const [parent] = resource.parents;
        const {data: mappingTemplate} = await this.api.getData(this.getDatasetId("mapping", parent));
        const exampleContentNameMap = {};
        Object.keys(content).forEach((key) => {
          const match = key.match(matchNumberRegex);
          if (match) {
            const matchNumber = match[0].trim();
            const matchingMapping = mappingTemplate.find(({ac}) => ac === matchNumber);
            if (matchingMapping && matchingMapping.acDsc) {
              const acDsc = matchingMapping.acDsc;
              const keyName = key.replace(matchNumberRegex, (a, b) => `${b} (${acDsc})`);
              exampleContentNameMap[key] = keyName;
            }
          }
        });
        validateContent(content);

        const resourceUpdate = buildResourceUpdate(content, exampleContentNameMap);
        const existingMapping = findExistingMatch(content, existingMappings);

        let resourceTag = getResourceTag(resource); // Identify the resource type
        if (resourceTag === auditResourceTypes.unknown.tag) {
          if (existingMapping) {
            resourceTag = existingMapping.tag;
            resourceUpdate.meta.tagConfirmed = true;
          } else {
            resourceTag = findMatchingTag(content);
          }
        } else { // The user assigned the tag
          resourceUpdate.meta.tagConfirmed = true;
        }
        if (!resourceTag) {
          throw new Error("resourceTag is undefined for null");
        }
        resourceUpdate.tags.push(resourceTag);

        if (resourceTag === auditResourceTypes.unknown.tag ||
          resourceTag === auditResourceTypes.evdn.tag) { // Early exit as file can't be correctly identified
          await this.api.updateResource(resource.id, resourceUpdate);
        } else {
          const auditResource = getAuditResourceByTag(resourceTag);

          if (!resource.meta.schemaMapping.length) { // Only run if this hasn't been mapped yet
            if (existingMapping) {
              resourceUpdate.meta.schemaMapping = existingMapping.schemaMapping;
              resourceUpdate.meta.mappingConfirmed = true;
            } else {
              resourceUpdate.meta.schemaMapping = inferSchemaMapping(content, schemas[auditResource.schema]);
            }
          }
          // Don't return this in order to catch any api errors
          await this.api.updateResource(resource.id, resourceUpdate);
        }
      }
    } catch (err) {
      this.output.debug(err);
      await this.api.updateResource(resource.id, {
        tags: [auditResourceTag, analysedResourceTag, analysisErrorTag],
        meta: {...resource.meta, error: err.message}},
      );
    }
    this.inProgress = this.inProgress.filter((id) => id !== resource.id);
    return Promise.resolve();
  }

  async getExampleContent(resource) {
    const csvStream = await this.getResourceCsvStream(resource);
    let {path} = csvStream;
    return new Promise((resolve, reject) => {
      csvStream.pipe(csv.parse({headers: true}))
        .on("data", (exampleContent) => {
          csvStream.pause();
          csvStream.destroy();
          if (path) {
            fsAsync.unlink(path); // Fire and forget to avoid issues with rapid calls to data
            path = ""; // Prevent unlink firing multiple times
          }
          resolve(exampleContent);
        })
        .on("error", (error) => reject(error));
    });
  }

  handleEvidence(resource) {
    const newTags = resource.tags.filter((currentTag) => !auditResourceTags.includes(currentTag));
    newTags.push(auditResourceTypes.evdn.tag);
    newTags.push(analysedResourceTag);
    return this.api.updateResource(resource.id, {tags: newTags});
  }

  async loadData() {
    const timeLimitedQuery = { // Applies a time delay after upload detected to ensure file is fully uploaded
      ...this.input.resourceQuery,
      modified: {$lt: new Date(Date.now() - this.input.pollPeriod).toISOString()},
    };
    const [{data: existingMappings}, unmappedResources] = await Promise.all([
      this.api.getData(this.getDatasetId("fileMappings"), null, null, {sort: {createdOn: -1}}),
      this.api.getResources(timeLimitedQuery),
    ]);
    // TODO: There's a subtle race condition here whereby a resource currently being analysed completes
    // analysis between the above api calls and the filter call below and is processed twice
    return [
      existingMappings,
      unmappedResources.filter(({id, modified}) => {
        return !this.inProgress.includes(id) && (Date.now() - new Date(modified).valueOf()) > 10000;
      }),
    ];
  }
}

function buildResourceUpdate(exampleContent, exampleContentNameMap) {
  return {
    tags: [auditResourceTag, analysedResourceTag],
    meta: {
      schemaMapping: [],
      mappingConfirmed: false,
      tagConfirmed: false,
      exampleContent: JSON.stringify(exampleContent),
      exampleContentNameMap,
    },
  };
}

function getAuditResourceByTag(resourceTag) {
  for (const resource in auditResourceTypes) {
    if (auditResourceTypes[resource].tag === resourceTag) {
      return auditResourceTypes[resource];
    }
  }
  return auditResourceTypes.unknown;
}

function getResourceTag(resource) {
  const resourceTag = auditResourceTags.find((tag) => resource.tags.includes(tag));
  return resourceTag || auditResourceTypes.unknown.tag;
}

function validateContent(content) {
  if (isPlainObject(content) && content !== null) {
    return true;
  } else {
    throw new Error("Could not parse content");
  }
}

module.exports = FileAnalyser;