globalstorage/lib/pg.ddl.js

Global Storage is a Global Distributed Data Warehouse

'use strict';

const { iter } = require('@metarhia/common');
const { extractDecorator } = require('metaschema');

const {
  escapeString,
  escapeIdentifier,
  validateIdentifier,
  PREDEFINED_DOMAINS,
  IGNORED_DOMAINS,
  classMapping,
} = require('./pg.utils');

const { getCategoryType, isGlobalCategory } = require('./schema.utils');

const { manyToManyTableName } = require('./ddl.utils.js');

const pad = (name, length, symbol = ' ') =>
  name + symbol.repeat(length - name.length);

const padProperty = (name, length) => pad(`"${name}"`, length + 2);

const verticalPad = text => (text ? `\n\n${text}` : '');

const createComment = string => {
  const separator = `-- ${string} `;
  return '\n' + pad(separator, 80, '-') + '\n\n';
};

// Generates SQL to define an Enum
//   domainName - <string>
//   values - <Array>, values of an Enum
//
// Returns: <Object>
//   type - <string>
//   sql - <string>
const createEnum = (domainName, values) => {
  validateIdentifier(domainName, 'enum');
  const type = escapeIdentifier(domainName);

  const sql =
    createComment(`Enum: ${type}`) +
    `CREATE TYPE ${type} AS ENUM (\n` +
    `  ${values.map(escapeString).join(',\n  ')}\n);`;
  return { type, sql };
};

const typeFromDomain = {
  bigint: () => 'bigint',
  number: domain => (domain.subtype === 'int' ? 'integer' : 'double precision'),
  string: () => 'text',
  function: () => 'text',
  boolean: () => 'boolean',
  object: (domain, domainKind) => {
    const type = classMapping[domain.class];
    if (!type) {
      throw new Error(
        `Unsupported domain class '${domain.class}' in domain '${domainKind}'`
      );
    }
    return type;
  },
};

// Generates SQL to define a domain
//   domainName - <string>
//   domain - <Object>
//
// Returns: <Object>
//   type - <string>
//   sql - <string>, optional
//
// Throws: <Error>, if `domainName` is not supported
//     or `domain` has too many values
const generateType = (domainName, domain) => {
  const domainKind = extractDecorator(domain);
  if (domainKind === 'Object') {
    const type =
      PREDEFINED_DOMAINS[domainName] ||
      typeFromDomain[domain.type](domain, domainName);
    return { type };
  }
  if (domainKind === 'Enum') {
    return createEnum(domainName, domain.values);
  }
  if (domainKind === 'Flags') {
    if (domain.values.length <= 16) {
      return { type: 'smallint' };
    }
    if (domain.values.length <= 32) {
      return { type: 'integer' };
    }
    if (domain.values.length <= 64) {
      return { type: 'bigint' };
    }
    throw new Error(
      `Too many flags in ${domainName}, must not be bigger than 64`
    );
  }
  throw new Error(`Unsupported domain: ${domainName}`);
};

// Generates Map of domain name to SQL definition of that domain
//   domains - <Map>, domain definitions
//
// Returns: <Object>
//   types - <Map>, domain -> SQL type definition
//   typesSQL - <string>
const generateTypes = domains => {
  const sqlQueries = [];
  const types = iter(domains)
    .filter(([domainName]) => !IGNORED_DOMAINS.includes(domainName))
    .map(([domainName, domain]) => {
      const { type, sql } = generateType(domainName, domain);
      if (sql) {
        sqlQueries.push(sql);
      }
      return [domainName, type];
    })
    .collectTo(Map);
  return { types, typesSQL: sqlQueries.join('\n') };
};

// Determines if a property should have a FOREIGN KEY constraint
//   categories - <Map>, categories generated by metaschema
//   from - <string>, name of a source category
//
// Returns: <boolean>
const requiresForeignKey = () => true;

// Categorizes schema entries into indexes, unique, links, and properties
//   category - <Object>, category definition
//   categoryName - <string>, name of a category
//   categories - <Map>, categories generated by metaschema
//
// Returns: <Object>
//   indexes - <Object[]>, with structure { name, property }
//   unique - <Object[]>, with structure { name, property }
//   links - <Object[]>, with structure { name, property, required }
//   properties - <Object[]>, with structure { name, property }
const categorizeEntries = (category, categoryName, categories) => {
  const indexes = [];
  const unique = [];
  const links = [];
  const properties = [];
  iter(Object.keys(category))
    .filter(name => typeof category[name] !== 'function')
    .each(name => {
      validateIdentifier(name, 'entry', `${categoryName}.`);

      const property = category[name];
      const type = extractDecorator(property);
      if (type === 'Include') {
        return;
      }
      const entry = { name, property };
      if (property.index) {
        indexes.push(entry);
      } else if (property.unique) {
        unique.push(entry);
      }
      if (property.domain) {
        properties.push(entry);
      } else if (property.category) {
        entry.foreignKey = requiresForeignKey(categories, categoryName);

        entry.destination = isGlobalCategory(property.definition)
          ? 'Identifier'
          : property.category;

        if (type !== 'Many') {
          properties.push(entry);
        }

        if (type === 'Many' || entry.foreignKey) {
          links.push(entry);
        }
      } else if (type === 'Index') {
        indexes.push(entry);
      } else if (type === 'Unique') {
        unique.push(entry);
      }
    });

  return { indexes, unique, links, properties };
};

// Calculates max length of properties in a category
//   properties - <Object[]>, with structure { name }
//
// Returns: <number>
const getMaxPropLength = properties =>
  iter(properties)
    .map(property => property.name.length)
    .reduce(Math.max);

// Generates SQL to define properties in a table
//   category - <string>, category name
//   properties - <Array>, properties in a category
//   types - <Map>, domain -> SQL type definition
//   maxPropLength - <number>, length of properties to be padded to
//
// Returns: <string>
const generateProperties = (category, properties, types, maxPropLength) =>
  properties.map(({ name, property }) => {
    const type = property.domain ? types.get(property.domain) : 'bigint';
    let sql = `${padProperty(name, maxPropLength)} ${type}`;
    if (property.required) {
      sql += ' NOT NULL';
    }
    if (property.default !== undefined) {
      sql += ` DEFAULT ${escapeString(property.default.toString())}`;
    }
    return sql;
  });

const ON_DELETE_RESTRICT_TYPES = new Set([
  'Object',
  'Catalog',
  'Subsystem',
  'Hierarchy',
]);

// Generates SQL to define a link in a table
//   from - <string>, name of a source category
//   to - <string>, name of a destination category
//   name - <string>, name of a link
//   type - <string>, type of a link
//   destination - <string>, name of a table to create `FOREIGN KEY` to
//
// Returns: <string>
const generateLink = (from, to, name, type, destination) => {
  const constraint = `fk${from}${name}`;
  validateIdentifier(constraint, 'constraint');
  let sql =
    `ALTER TABLE ${escapeIdentifier(from)} ` +
    `ADD CONSTRAINT ${escapeIdentifier(constraint)} ` +
    `FOREIGN KEY (${escapeIdentifier(name)}) `;

  sql +=
    `REFERENCES "${destination}" (${escapeIdentifier('Id')}) ` +
    'ON UPDATE RESTRICT ON DELETE ';

  if (ON_DELETE_RESTRICT_TYPES.has(type)) {
    sql += 'RESTRICT';
  } else if (type === 'Master') {
    sql += 'CASCADE';
  } else {
    throw new Error(`${type} decorator is not supported`);
  }

  return sql + ';';
};

// Generates SQL to define many to many relationship
//   from - <string>, name of a source category
//   to - <string>, name of a destination category
//   name - <string>, name of a link
//   categories - <Map>, categories generated by metaschema
//   destination - <string>, name of a table to create the link to
//
// Returns: <string>
const generateManyToMany = (from, to, name, categories, destination) => {
  const propLength = from.length > name.length ? from.length : name.length;
  const tableName = manyToManyTableName(from, to, name);
  const table =
    `\nCREATE TABLE ${escapeIdentifier(tableName)} (\n` +
    `  ${padProperty(from, propLength)} bigint NOT NULL,\n` +
    `  ${padProperty(name, propLength)} bigint NOT NULL\n);`;

  const lines = [table];

  if (requiresForeignKey(categories, from)) {
    lines.push(generateLink(tableName, from, from, 'Master', from));
    lines.push(generateLink(tableName, to, name, 'Master', destination));
  }

  const primary = `pk${name}${from}${to}`;
  validateIdentifier(primary, 'constraint');

  lines.push(
    `ALTER TABLE ${escapeIdentifier(tableName)} ` +
      `ADD CONSTRAINT ${escapeIdentifier(primary)} ` +
      `PRIMARY KEY (${escapeIdentifier(from)}, ${escapeIdentifier(name)});`
  );
  return lines.join('\n');
};

// Generates SQL to define links in a table
//   links - <Array>, links
//   categories - <Map>, categories generated by metaschema
//
// Returns: <string>
const generateLinks = (links, categories) =>
  links
    .sort(({ link: l1 }, { link: l2 }) => {
      const d1 = extractDecorator(l1);
      const d2 = extractDecorator(l2);
      if (d1 !== 'Many' && d2 === 'Many') return -1;
      if (d1 === 'Many' && d2 !== 'Many') return 1;
      return 0;
    })
    .map(({ from, to, name, link, destination }) => {
      const type = extractDecorator(link);
      return type === 'Many'
        ? generateManyToMany(from, to, name, categories, destination)
        : generateLink(from, to, name, type, destination);
    })
    .join('\n');

// Generates SQL to define indexes in a table
//   indexes - <Array>, entries that require an index
//   table - <string>, name of a table
//
// Returns: <string>
const generateIndexes = (indexes, table) =>
  indexes
    .map(({ name, property: index }) => {
      const type = extractDecorator(index);
      const prop =
        type === 'Index'
          ? index.fields.map(escapeIdentifier).join(', ')
          : escapeIdentifier(name);

      const indexName = `idx${table}${name}`;
      validateIdentifier(name, 'index');

      return (
        `CREATE INDEX ${escapeIdentifier(indexName)} ` +
        `on ${escapeIdentifier(table)} (${prop});`
      );
    })
    .join('\n');

// Generates SQL to define UNIQUE constraints in a table
//   unique - <Array>, entries that require UNIQUE constraint
//   table - <string>, name of a table
//
// Returns: <string>
const generateUnique = (unique, table) =>
  unique
    .map(({ name, property: index }) => {
      const type = extractDecorator(index);
      const prop =
        type === 'Unique'
          ? index.fields.map(escapeIdentifier).join(', ')
          : escapeIdentifier(name);

      const constraint = `ak${table}${name}`;
      validateIdentifier(constraint, 'constraint');

      return (
        `ALTER TABLE ${escapeIdentifier(table)} ` +
        `ADD CONSTRAINT ${escapeIdentifier(constraint)} UNIQUE (${prop});`
      );
    })
    .join('\n');

// Generates SQL to define id in a table
//   type - <string>, type of a category
//   length - <number>, length to pad id to
//
// Returns: <string>
const generateId = (type, length) =>
  padProperty('Id', length) + (type === 'Global' ? ' bigint' : ' bigserial');

// Categorizes links into resolvable and unresolvable.
//   category - <string>, name of category to resolve links to and from
//   unresolvedLinks - <Map>, unresolved links
//   links - <Array>, links to be categorized
//   existingTables - <Array>, names of defined tables
//
// Returns: [ <Array>, <Array> ]
//   unresolved - <Array>, links that can not be resolved at the moment
//   resolvableLinks - <Array>, links that can be resolved
const getResolvableLinks = (
  category,
  unresolvedLinks,
  links,
  existingTables
) => {
  const resolvableLinks = unresolvedLinks.get(category) || [];
  const unresolved = links.filter(({ name, property: link, destination }) => {
    if (destination === category || existingTables.includes(destination)) {
      resolvableLinks.push({
        from: category,
        to: link.category,
        name,
        link,
        destination,
      });
      return false;
    }
    return true;
  });

  return [unresolved, resolvableLinks];
};

// Generates SQL to define a table.
//   name - <string>, name of a category
//   category - <Object>, category definition
//   type - <string>, type of a category
//   types - <Map>, domain -> SQL type definition
//   unresolvedLinks - <Map>, unresolved links
//   existingTables - <Array>, names of defined tables
//   categories - <Map>, categories generated by metaschema
//
// Returns: <Object>
//   sql - <string>
//   unresolved - <Array>
const generateTable = (
  name,
  category,
  type,
  types,
  unresolvedLinks,
  existingTables,
  categories
) => {
  validateIdentifier(name, 'table');

  const { indexes, unique, links, properties } = categorizeEntries(
    category,
    name,
    categories
  );

  const maxPropLength = getMaxPropLength(properties);
  const props = generateProperties(name, properties, types, maxPropLength);
  props.unshift(generateId(type, maxPropLength));

  const [unresolved, resolvableLinks] = getResolvableLinks(
    name,
    unresolvedLinks,
    links,
    existingTables
  );

  const pk = `pk${name}Id`;
  validateIdentifier(pk, 'constraint');

  const primaryKey =
    `ALTER TABLE ${escapeIdentifier(name)} ` +
    `ADD CONSTRAINT ${escapeIdentifier(pk)} ` +
    `PRIMARY KEY (${escapeIdentifier('Id')});`;

  const sql =
    createComment(`Category: ${name}`) +
    `CREATE TABLE ${escapeIdentifier(name)} (\n  ${props.join(',\n  ')}\n);` +
    [
      generateIndexes(indexes, name),
      generateUnique(unique, name),
      primaryKey,
      generateLinks(resolvableLinks, categories),
    ]
      .map(verticalPad)
      .join('');

  return {
    sql,
    unresolved,
  };
};

// Adds new unresolved links to an existing map
//   links - <Object[]>, with structure { name, property, required }
//   unresolved - <Map>, unresolved links -> destination
//   category - <string>, category name
const addUnresolved = (links, unresolved, category) => {
  links.forEach(({ name, property: link, destination }) => {
    const existingLinks = unresolved.get(destination);
    const newLink = {
      from: category,
      to: link.category,
      name,
      link,
      destination,
    };
    if (existingLinks) {
      existingLinks.push(newLink);
    } else {
      unresolved.set(destination, [newLink]);
    }
  });
};

// Generates SQL to define tables based on a schema.
//   categories - <Map>, categories generated by metaschema
//   types - <Map>, domain -> SQL type definition
//
// Returns: <string>
const generateTables = (categories, types) => {
  const unresolvedLinks = new Map(); /* links: table => [{ name, link }] */
  const existingTables = [];
  return iter(categories)
    .map(([name, category]) => {
      const type = getCategoryType(category.definition);
      return {
        name,
        category: category.definition,
        type,
      };
    })
    .filter(({ type }) => type !== 'Ignore')
    .map(({ name, category, type }) => {
      const { sql, unresolved } = generateTable(
        name,
        category,
        type,
        types,
        unresolvedLinks,
        existingTables,
        categories
      );
      if (unresolved) {
        addUnresolved(unresolved, unresolvedLinks, name);
      }
      existingTables.push(name);
      return sql;
    })
    .toArray()
    .join('\n');
};

const createHistorySchema = (schema, domains, categories) => {
  const history = Object.assign({}, schema);
  const dateTime = domains.get('DateTime');
  history._Creation = {
    domain: 'DateTime',
    required: true,
    index: true,
    definition: dateTime,
  };
  history._Effective = {
    domain: 'DateTime',
    required: true,
    index: true,
    definition: dateTime,
  };
  history._Cancel = {
    domain: 'DateTime',
    index: true,
    definition: dateTime,
  };
  history._HistoryStatus = {
    domain: 'HistoryStatus',
    required: true,
    definition: dateTime,
  };
  history._Identifier = {
    category: 'Identifier',
    required: true,
    index: true,
    definition: categories.get('Identifier').definition,
  };
  return history;
};

const preprocessSchema = (categories, domains) => {
  const processed = new Map();
  for (const [name, category] of categories) {
    processed.set(name, category);
    const type = extractDecorator(category.definition);
    if (type === 'History') {
      const historyName = `${name}History`;
      processed.set(historyName, {
        name: historyName,
        definition: createHistorySchema(
          category.definition,
          domains,
          categories
        ),
      });
    }
  }
  return processed;
};

// Generate SQL to define extensions being used.
//   names <string[]> extension names
// Returns: <string>
const generateExtensions = names =>
  names.map(extName => `CREATE EXTENSION IF NOT EXISTS ${extName};\n`).join('');

// Generates SQL to define a postgres database structure based on a schema.
//   schema - <Metaschema>
//
// Returns: <string>
const generateDDL = schema => {
  const { types, typesSQL } = generateTypes(schema.domains);
  const extensionsSQL = generateExtensions(['pgcrypto']);
  const tablesSQL = generateTables(
    preprocessSchema(schema.categories, schema.domains),
    types
  );

  return `${extensionsSQL}\n${typesSQL}\n${tablesSQL}`;
};

module.exports = {
  generateDDL,
  generateExtensions,
  preprocessSchema,
  createHistorySchema,
  generateTables,
  generateTable,
  addUnresolved,
  getResolvableLinks,
  generateId,
  generateUnique,
  generateIndexes,
  generateLinks,
  generateManyToMany,
  generateLink,
  generateProperties,
  getMaxPropLength,
  categorizeEntries,
  generateTypes,
  generateType,
  createEnum,
  createComment,
  verticalPad,
  padProperty,
  pad,
};