@krlwlfrt/xsdco/src/extract.ts

XSD converter

/*
 * Copyright (C) 2019, 2020 Karl-Philipp Wulfert
 * This program is free software: you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation, version 3.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program. If not, see <https://www.gnu.org/licenses/>.
 */

import {
  Attribute,
  Entity,
  Property,
  ThingWithNameAndNamespace,
  Type,
} from '@krlwlfrt/tsg';
import {existsSync} from 'fs';
import {readFile} from 'fs/promises';
import {dirname, resolve} from 'path';
import {exit} from 'process';
import {asyncParseString} from './async';
import {logger} from './common';
import {
  XSD,
  XSDAnnotation,
  XSDAttribute,
  XSDComplexType,
  XSDElement,
  XSDElementRef,
  XSDExtension,
  XSDGroup,
  XSDGroupRef,
  XSDImport,
  XSDRestriction,
  XSDRestrictionValue,
  XSDSchema,
  XSDSequence,
  XSDSimpleContent,
  XSDSimpleType,
} from './types';

/**
 * Map from names to their respecitve XSD <attribute>s
 */
const attributes: Record<string, Property> = {};
/**
 * Map from names to their respective XSD <group>s
 */
const groups: Record<string, Entity> = {};

/**
 * Handle a complex type
 * @param complexType Complex type to handle
 * @param xsdNamespace XSD namespace for complex type
 * @param targetNamespace Target namespace for complex type
 * @returns An array of types
 */
function handleComplexType(
  complexType: XSDComplexType,
  xsdNamespace: string,
  targetNamespace: string,
): Type[] {
  const types: Type[] = [];

  const description = getDescription(complexType, xsdNamespace);

  if (Array.isArray(complexType[`${xsdNamespace}simpleContent` as keyof XSDComplexType])) {
    const simpleContent = (complexType[`${xsdNamespace}simpleContent` as keyof XSDComplexType] as XSDSimpleContent[])[0];
    const extension = (simpleContent[`${xsdNamespace}extension` as keyof XSDSimpleContent] as XSDExtension[])[0];

    types.push({
      description,
      name: complexType.$.name,
      namespace: targetNamespace,
      type: getSplitType(extension.$.base, targetNamespace),
    });
  } else {
    const entity: Entity = {
      description,
      name: complexType.$.name,
      namespace: targetNamespace,
      properties: [],
    };

    if (Array.isArray(complexType[`${xsdNamespace}sequence` as keyof XSDComplexType])) {
      const sequences = complexType[`${xsdNamespace}sequence` as keyof XSDComplexType] as XSDSequence[];

      if (sequences.length > 1) {
        logger.error(`Complex type '${complexType.$.name}' has ${sequences.length} sequences!`);
      }

      handleSequence(
        sequences[0],
        entity,
        xsdNamespace,
        targetNamespace,
        types,
      );
    }

    if (Array.isArray(complexType[`${xsdNamespace}attribute` as keyof XSDComplexType])) {
      for (const attribute of complexType[`${xsdNamespace}attribute` as keyof XSDComplexType] as XSDAttribute[]) {
        if (typeof attribute.$.ref === 'string') {
          if (attribute.$.ref in attributes) {
            const referencedProperty = attributes[attribute.$.ref];
            entity.properties.push(referencedProperty);
          } else {
            logger.error(`Referenced attribute ${attribute.$.ref} does not exist.`);
          }
        } else {
          entity.properties.push({
            description,
            name: `$${attribute.$.name}`,
            namespace: targetNamespace,
            required: attribute.$.use === 'required',
            type: getSplitType(attribute.$.type, targetNamespace),
          });
        }
      }
    }

    if (Array.isArray(complexType[`${xsdNamespace}group` as keyof XSDComplexType])) {
      for (const group of complexType[`${xsdNamespace}group` as keyof XSDComplexType] as XSDGroupRef[]) {
        if (typeof group.$.ref === 'string') {
          if (group.$.ref in groups) {
            entity.properties.push(...groups[group.$.ref].properties);
            logger.info(`Adding properties from group '${group.$.ref}' to entity '${entity.name}'.`);
          } else {
            logger.error(`Referenced group ${group.$.ref} does not exist.`);
          }
        }
      }
    }

    types.push(entity);

    const unhandledKeys = Object.keys(complexType).filter((key) => {
      return ![
        '$',
        `${xsdNamespace}sequence`,
        `${xsdNamespace}attribute`,
        `${xsdNamespace}annotation`,
        `${xsdNamespace}group`,
      ].includes(key);
    });

    if (unhandledKeys.length > 0) {
      logger.error(
        `Complex type '${complexType.$.name}' has unhandled keys (${unhandledKeys.join(', ')}).`,
      );
    }
  }

  return types;
}

/**
 * Get description of an element
 * @param element Element to get description of
 * @param xsdNamespace XSD namespace to use
 * @returns Description of an element
 */

/**
 * Get description from element
 * @param element Element to get description from
 * @param xsdNamespace XSD namespace to uses
 * @returns Returns a description
 */
function getDescription(
  element: XSDElement,
  xsdNamespace: string,
): string | undefined {
  const annotations = element[`${xsdNamespace}annotation` as keyof XSDElement] as XSDAnnotation[];

  if (!Array.isArray(annotations) || annotations.length === 0) {
    return;
  }

  const description = annotations[0][`${xsdNamespace}documentation` as keyof XSDAnnotation];

  if (!Array.isArray(description) || description.length === 0) {
    return;
  }

  return description[0];
}

/**
 * Get split type from XSD type denotation
 * @param type XSD type denotation
 * @param targetNamespace Target namespace for the type, if it has no namespace
 * @returns Thing with namespace and name
 */
function getSplitType(
  type: string,
  targetNamespace: string,
): ThingWithNameAndNamespace {
  if (typeof type === 'undefined') {
    logger.warn(`Can not get split type for ${type} and ${targetNamespace}`);
  }

  const typeParts = type.split(':');

  const splitType = {
    name: typeParts[0],
    namespace: targetNamespace,
  };

  if (typeParts.length === 2) {
    splitType.namespace = typeParts[0];
    splitType.name = typeParts[1];
  }

  return splitType;
}

/**
 * Get attributes from an XSD restriction
 * @param restriction XSD restriction to get attributes from
 * @param xsdNamespace XSD namespace for tags
 * @param targetNamespace Target namespace for attributes
 * @returns An array of attributes
 */
function handleRestriction(
  restriction: XSDRestriction,
  xsdNamespace: string,
  targetNamespace: string,
): Attribute[] {
  const attributes: Attribute[] = [];
  let unhandledKeys: string[] = [];

  for (const key of Object.keys(restriction)) {
    if (['pattern', 'maxLength', 'minLength', 'totalDigits', 'fractionDigits'].map((key) => `${xsdNamespace}${key}`).includes(key)) {
      if (Array.isArray(restriction[`${xsdNamespace}${key}` as keyof XSDRestriction])) {
        let value: string | number = (restriction[`${xsdNamespace}pattern` as keyof XSDRestriction] as XSDRestrictionValue[])[0].$.value;

        if (['maxLength', 'minLength', 'totalDigits', 'fractionDigits'].map((key) => `${xsdNamespace}${key}`).includes(key)) {
          value = parseInt(value, 10);
        }

        attributes.push({
          name: key,
          namespace: targetNamespace,
          value,
        });
      }
    } else {
      unhandledKeys.push(key);
    }
  }

  unhandledKeys = unhandledKeys.filter((key) => key !== '$');

  if (unhandledKeys.length > 0) {
    logger.error(`Restriction has unhandled keys ${unhandledKeys.join(', ')}.`);
  }

  return attributes;
}

/**
 * Parse a list of XSD sequences
 * @param sequence XSD sequences to parse
 * @param entity Entity to attribute sequence to
 * @param xsdNamespace XSD namespace for tags
 * @param targetNamespace Target namespace for properties, attributes, nested entities, ...
 * @param types An array of types
 */
function handleSequence(
  sequence: XSDSequence,
  entity: Entity,
  xsdNamespace: string,
  targetNamespace: string,
  types: Type[],
): void {
  if (Array.isArray(sequence[`${xsdNamespace}element` as keyof XSDSequence])) {
    const elements = sequence[`${xsdNamespace}element` as keyof XSDSequence] as (XSDElement | XSDElementRef)[];

    for (const iterator of elements) {
      const element: XSDElement = iterator as XSDElement;
      const elementRef: XSDElementRef = iterator as XSDElementRef;

      if (typeof element.$.type === 'string') {
        const type = getSplitType(element.$.type, targetNamespace);

        entity.properties.push({
          description: getDescription(element as XSDElement, xsdNamespace),
          multiple: element.$.maxOccurs === 'unbounded',
          name: element.$.name,
          namespace: targetNamespace,
          required: element.$.minOccurs !== '0',
          type,
        });
      } else if (typeof elementRef.$.ref === 'string') {
        const splitType = getSplitType(elementRef.$.ref, targetNamespace);
        const property = types.find(
          (type) =>
            type.name === splitType.name &&
            type.namespace === splitType.namespace,
        ) as Property | undefined;
        if (typeof property === 'undefined') {
          logger.error(`Could not find referenced property ${elementRef.$.ref}.`);
        } else {
          entity.properties.push(property);
        }
      } else if (Array.isArray(element[`${xsdNamespace}simpleType` as keyof XSDElement] as XSDSimpleType[])) {
        const simpleType = (element[`${xsdNamespace}simpleType` as keyof XSDElement] as XSDSimpleType[])[0];
        const restriction = (simpleType[`${xsdNamespace}restriction` as keyof XSDSimpleType] as XSDRestriction[])[0];

        const type = getSplitType(restriction.$.base, targetNamespace);

        entity.properties.push({
          attributes: handleRestriction(
            restriction,
            xsdNamespace,
            targetNamespace,
          ),
          description: getDescription(element, xsdNamespace),
          multiple: element.$.maxOccurs === 'unbounded',
          name: element.$.name,
          namespace: targetNamespace,
          required: element.$.minOccurs !== '0',
          type,
        });
      } else if (Array.isArray(element[`${xsdNamespace}complexType` as keyof XSDElement])) {
        // TODO: replace with handleComplexType

        const complexType = (element[`${xsdNamespace}complexType` as keyof XSDElement] as XSDComplexType[])[0];

        if (Array.isArray(complexType[`${xsdNamespace}sequence` as keyof XSDComplexType])) {
          const nestedEntity: Entity = {
            name: `${entity.name}__${element.$.name}`,
            namespace: targetNamespace,
            properties: [],
          };

          entity.properties.push({
            multiple: element.$.maxOccurs === 'unbounded',
            name: element.$.name,
            namespace: targetNamespace,
            required: element.$.minOccurs !== '0',
            type: {
              name: `${entity.name}__${element.$.name}`,
              namespace: targetNamespace,
            },
          });

          handleSequence(
            (complexType[`${xsdNamespace}sequence` as keyof XSDComplexType] as XSDSequence[])[0],
            nestedEntity,
            xsdNamespace,
            targetNamespace,
            types,
          );

          types.push(nestedEntity);
        } else if (Array.isArray(complexType[`${xsdNamespace}simpleContent` as keyof XSDComplexType])) {
          const simpleContent = (complexType[`${xsdNamespace}simpleContent` as keyof XSDComplexType] as XSDSimpleContent[])[0];
          const extension = simpleContent[`${xsdNamespace}extension` as keyof XSDSimpleContent][0];

          const type = getSplitType(extension.$.base, targetNamespace);

          entity.properties.push({
            description: getDescription(element, xsdNamespace),
            multiple: element.$.maxOccurs === 'unbounded',
            name: element.$.name,
            namespace: targetNamespace,
            required: element.$.minOccurs !== '0',
            type,
          });
        } else {
          logger.info(`Skipping element ${element.$.name} on complex type ${entity.name}.`);
        }
      } else {
        logger.info(`Skipping element ${element.$.name} on complex type ${entity.name}.`);
      }
    }
  }

  if (Array.isArray(sequence[`${xsdNamespace}group` as keyof XSDSequence])) {
    for (const group of sequence[`${xsdNamespace}group` as keyof XSDSequence] as XSDGroupRef[]) {
      if (typeof group.$.ref === 'string') {
        if (group.$.ref in groups) {
          entity.properties.push(...groups[group.$.ref].properties);
          logger.info(`Adding properties from group ${group.$.ref} to entity ${entity.name}.`);
        } else {
          logger.error(`Referenced group ${group.$.ref} does not exist.`);
        }
      }
    }
  }

  const unhandledKeys = Object.keys(sequence).filter((key) => {
    return !['$', `${xsdNamespace}element`, `${xsdNamespace}group`].includes(
      key,
    );
  });

  if (unhandledKeys.length > 0) {
    logger.warn(`Sequence has unhandled keys (${unhandledKeys.join(', ')}).`);
  }
}

/**
 * Extract data description from an an XSD
 * @param content Content of the XSD
 * @param intendedTargetNamespace Intended target namespace for the XSD
 * @param path Path to the original file to resolve imports
 * @returns A promise that resolves with an array of types
 */
export async function extract(
  // eslint-disable-next-line @typescript-eslint/ban-ts-comment
  // @ts-ignore
  content: any,
  intendedTargetNamespace?: string,
  path?: string,
): Promise<Type[]> {
  const xsd: XSD = await asyncParseString(content.toString());

  const keyParts = Object.keys(xsd)[0].split(':');
  let assumedXsdNamespace: 'xsd:' = null as unknown as 'xsd:';
  if (keyParts.length > 1) {
    assumedXsdNamespace = keyParts[0] as 'xsd:';
  }

  logger.info(`Assuming namespace for schema to be '${assumedXsdNamespace}'.`);

  const assumedXsdUri =
    xsd[`${assumedXsdNamespace}:schema` as keyof XSD].$[`xmlns:${assumedXsdNamespace}`];
  if (assumedXsdUri !== 'http://www.w3.org/2001/XMLSchema') {
    logger.warn(`Assumed namespace for schema has wrong URI '${assumedXsdUri}'.`);
  }

  const xsdNamespace: 'xsd:' =
    assumedXsdNamespace !== null
      ? (`${assumedXsdNamespace}:` as 'xsd:')
      : ('' as 'xsd:');
  const xsdSchema = xsd[`${assumedXsdNamespace}:schema` as keyof XSD];

  const targetNamespaceURI = xsdSchema.$.targetNamespace;

  let targetNamespace;

  logger.info(`URI for target namespace is '${targetNamespaceURI}'.`);

  const uriToNamespace: Record<string, string> = {};

  for (const key in xsdSchema.$) {
    if (!{}.hasOwnProperty.call(xsdSchema.$, key)) {
      continue;
    }

    if (!/^xmlns/.test(key)) {
      continue;
    }

    const namespace = key.split(':')[1];

    if (xsdSchema.$[key] === targetNamespaceURI) {
      targetNamespace = namespace;
    }

    uriToNamespace[xsdSchema.$[key]] = namespace;
  }

  logger.info(`Detected target namespace is '${typeof targetNamespace === 'undefined' ? 'ROOT' : targetNamespace}'.`);

  if (typeof intendedTargetNamespace === 'string' && intendedTargetNamespace !== targetNamespace) {
    targetNamespace = intendedTargetNamespace;

    logger.info(`Using intended target namespace '${intendedTargetNamespace}' as target namespace.`);
  }

  if (typeof targetNamespace === 'undefined') {
    targetNamespace = '';
  }

  const types: Type[] = [];

  for (const key in xsdSchema) {
    if (!{}.hasOwnProperty.call(xsdSchema, key)) {
      continue;
    }

    if ([`${xsdNamespace}include`, `${xsdNamespace}import`].includes(key) && Array.isArray(xsdSchema[key as keyof XSDSchema])) {
      for (const importDeclaration of xsdSchema[key as keyof XSDSchema] as XSDImport[]) {
        if (typeof path === 'undefined') {
          logger.error(`Can not resolve ${key}s without a path.`);
          continue;
        }

        if (typeof importDeclaration.$.schemaLocation === 'undefined') {
          logger.error(`Can not import schema referenced by '${importDeclaration.$.namespace}' because it is missing a 'schemaLocation'.`);
          continue;
        }

        const importPath = resolve(dirname(path.toString()), importDeclaration.$.schemaLocation);
        let importTargetNamespace = targetNamespace;

        if (Object.keys(uriToNamespace).includes(importDeclaration.$.namespace)) {
          importTargetNamespace = uriToNamespace[importDeclaration.$.namespace];
        }

        if (existsSync(importPath)) {
          logger.info(`Additionally parsing '${importPath}' for '${importDeclaration.$.namespace}'.`);

          types.push(...(await extractFromFile(importPath, importTargetNamespace)));
        } else {
          logger.error(`Could not additionally parse ${importPath} for ${importTargetNamespace}. File does not exist.`);
        }
      }
    } else if (key === `${xsdNamespace}simpleType` && Array.isArray(xsdSchema[`${xsdNamespace}simpleType`])) {
      for (const simpleType of xsdSchema[`${xsdNamespace}simpleType`] as XSDSimpleType[]) {
        const restriction = (simpleType[`${xsdNamespace}restriction`] as XSDRestriction[])[0];

        const type: Type = {
          attributes: handleRestriction(
            restriction,
            xsdNamespace,
            targetNamespace,
          ),
          name: simpleType.$.name,
          namespace: targetNamespace,
          type: getSplitType(restriction.$.base, targetNamespace),
        };

        types.push(type);
      }
    } else if (key === `${xsdNamespace}element` && Array.isArray(xsdSchema[`${xsdNamespace}element` as keyof XSDSchema])) {
      for (const element of xsdSchema[`${xsdNamespace}element` as keyof XSDSchema] as XSDElement[]) {
        if (Array.isArray(element[`${xsdNamespace}complexType` as keyof XSDElement])) {
          const complexType = (element[`${xsdNamespace}complexType` as keyof XSDElement] as XSDComplexType[])[0];

          if (Array.isArray(complexType[`${xsdNamespace}sequence` as keyof XSDComplexType])) {
            const entity: Entity = {
              name: element.$.name,
              namespace: targetNamespace,
              properties: [],
            };

            const sequences = complexType[`${xsdNamespace}sequence` as keyof XSDComplexType] as XSDSequence[];

            if (sequences.length > 1) {
              logger.error(`ComplexType '${complexType.$.name}' on Element '${element.$.name}' has multiple sequences!`);
            }

            handleSequence(
              sequences[0],
              entity,
              xsdNamespace,
              targetNamespace,
              types,
            );

            types.push(entity);
          }
        } else if (typeof element.$.name === 'string') {
          types.push({
            name: element.$.name,
            namespace: targetNamespace,
            // eslint-disable-next-line @typescript-eslint/ban-ts-comment
            // @ts-ignore
            type: getSplitType(element.$.type, targetNamespace),
          });
        }
      }
    } else if (key === `${xsdNamespace}complexType` && Array.isArray(xsd[`${xsdNamespace}schema`][`${xsdNamespace}complexType`])) {
      for (const complexType of xsd[`${xsdNamespace}schema`][`${xsdNamespace}complexType`] as XSDComplexType[]) {
        types.push(...handleComplexType(complexType, xsdNamespace, targetNamespace));
      }
    } else if (key === `${xsdNamespace}group` && Array.isArray(xsdSchema[`${xsdNamespace}group` as keyof XSDSchema])) {
      for (const group of xsdSchema[`${xsdNamespace}group` as keyof XSDSchema] as unknown as XSDGroup[]) {
        if (Array.isArray(group[`${xsdNamespace}sequence` as keyof XSDGroup])) {
          const entity: Entity = {
            name: group.$.name,
            properties: [],
          };

          handleSequence(
            (group[`${xsdNamespace}sequence` as keyof XSDGroup] as XSDSequence[])[0],
            entity,
            xsdNamespace,
            targetNamespace,
            types,
          );

          groups[`${targetNamespace}:${group.$.name}`] = entity;
          groups[`${group.$.name}`] = entity;
        }
      }
    } else if (key === `${xsdNamespace}attribute` && Array.isArray(xsdSchema[`${xsdNamespace}attribute` as keyof XSDSchema])) {
      for (const attribute of xsdSchema[`${xsdNamespace}attribute` as keyof XSDSchema] as unknown as XSDAttribute[]) {
        attributes[attribute.$.name] = {
          // description,
          name: `$${attribute.$.name}`,
          namespace: targetNamespace,
          required: attribute.$.use === 'required',
          type: getSplitType(attribute.$.type, targetNamespace),
        };
      }
    } else if (key !== '$') {
      logger.error(`Unhandled key ${key} in schema.`);
    }
  }

  return types;
}

/**
 * Extract data description from an XSD file
 * @param path Path to XSD
 * @param intendedTargetNamespace Intended target namespace for the XSD
 * @returns A promise that resolves with an array of types
 */
export async function extractFromFile(
  path: string,
  intendedTargetNamespace?: string,
): Promise<Type[]> {
  if (!existsSync(path)) {
    logger.error(`Can not extract types from '${path}' because it does not exist.`);
    exit(1);
  }

  const content = await readFile(path);

  return extract(content, intendedTargetNamespace, path);
}