@nqminds/fin-genie-file-analyser/lib/matching-functions.js

A databot for analysing and tagging FinGenie files

"use strict";

const distance = require("jaro-winkler");
const {auditResourceTypes} = require("@nqminds/fin-genie-constants");

const STRING_DIST_THRESHOLD = 0.75; // This was picked through trial and error

/**
 * Checks to see if a file with content matching the provided content has been seen before
 *
 * @param {object} exampleContent - a json example of the content of the resource
 * @param {object} existingMappings - obtained from the TDX, includes information about schema etc
 * @returns {object} returns a fileMapping document if found
 */
function findExistingMatch(exampleContent, existingMappings) {
  const inputKeys = Object.keys(exampleContent);

  return existingMappings.find((fileMapping) => {
    const equalKeyNumber = fileMapping.columnHeadings.length === inputKeys.length;
    const allInputKeysPresent = inputKeys.every((key) => fileMapping.columnHeadings.includes(key));
    return equalKeyNumber && allInputKeysPresent;
  });
}

/**
 * Matches a resource with an audit resource type
 *
 * @param {object} exampleContent - a json example of the content of the resource
 * @returns {string} the tag of the audit resource that most closely matches the provided content
 */
function findMatchingTag(exampleContent) {
  const lcInputKeys = Object.keys(exampleContent).map((key) => key.toLowerCase());
  const auditResourceScores = Object.keys(auditResourceTypes).map((auditResourceType) => {
    return calculateAuditResourceScore(auditResourceTypes[auditResourceType], lcInputKeys);
  });
  let maxScore = 0;
  let matchedResourceTag = auditResourceTypes.unknown.tag;
  auditResourceScores.forEach(({tag, score}) => {
    if (score > maxScore) {
      matchedResourceTag = tag;
      maxScore = score;
    }
  });
  return matchedResourceTag;
}

function calculateAuditResourceScore(auditResource, inputKeys) {
  let score = 0;
  auditResource.keywords.forEach((keyword) => {
    const exactMatch = inputKeys.includes(keyword);
    if (exactMatch) {
      score += 2;
    } else {
      const fuzzyMatch = inputKeys.find((inputKey) => {
        return distance(inputKey, keyword) > STRING_DIST_THRESHOLD;
      });
      if (fuzzyMatch) {
        score += 1;
      }
    }
  });
  return {tag: auditResource.tag, score};
}

function isValueOfType(value, type) {
  switch (type) {
    case "string":
      return true; // All js objects can be represented by strings
    case "number":
      const number = Number(value); // Ensures value can be sensibly parsed to a number
      return !Number.isNaN(number);
    default:
      return false;
  }
}

/**
 * A field mapping
 *
 * @typedef {object} FieldMapping
 * @property {string|null} inputKey
 * @property {string} outputKey
 */

/**
 * Attempts to infer the mapping between
 *
 * @param {object} content - a json example of the content of the resource
 * @param {object} schema - the schema that this resource has been identified as corresponding to
 * @returns {FieldMapping[]} the mapping to convert from input resource to the schema
 */
function inferSchemaMapping(content, schema) {
  const {schema: {dataSchema: {properties: schemaProperties}}} = schema;
  const schemaMapping = [];
  // for each of the schema fields, find the most
  // similar key that also has data that could be considered to match
  for (const property in schemaProperties) {
    const potentialMatches = [];
    for (const key in content) {
      if (isValueOfType(content[key], schemaProperties[property].type)) {
        potentialMatches.push(key);
      }
    }
    let bestMatch = null;
    let bestScore = STRING_DIST_THRESHOLD;
    potentialMatches.forEach((key) => {
      const description = schemaProperties[property].description;
      const score = distance(key, description ? description : "", {caseSensitive: false});
      if (score > bestScore) {
        bestScore = score;
        bestMatch = key;
      }
    });
    schemaMapping.push({inputKey: bestMatch, outputKey: property});
  }
  return schemaMapping;
}

module.exports = {
  inferSchemaMapping,
  findMatchingTag,
  findExistingMatch,
};