arkit/src/parser.ts

Visualises JavaScript, TypeScript and Flow codebases as meaningful and committable architecture diagrams

import { EOL } from "os";
import * as path from "path";
import * as fs from "fs";
import {
  ExportDeclarationStructure,
  ImportDeclarationStructure,
  Node,
  SourceFile,
  Statement,
} from "ts-morph";
import {
  find,
  debug,
  info,
  trace,
  warn,
  getAllStatements,
  error,
} from "./utils";
import { ConfigBase, Exports, File, Files, Imports } from "./types";
import ProgressBar from "progress";
import { FileSystem } from "./filesystem";

const QUOTES = `(?:'|")`;
const TEXT_INSIDE_QUOTES = `${QUOTES}([^'"]+)${QUOTES}`;
const TEXT_INSIDE_QUOTES_RE = new RegExp(TEXT_INSIDE_QUOTES);
const REQUIRE_RE = new RegExp(
  `require\\(${TEXT_INSIDE_QUOTES}\\)(?:\\.(\\w+))?`,
);
const EXPORTS_RE = /(?:module\.)?exports(?:\.(\w+))?\s*=/g;

export class Parser {
  private readonly fs: FileSystem;

  constructor(config: ConfigBase) {
    this.fs = new FileSystem(config);
  }

  parse(): Files {
    const files: Files = {};
    const progress = new ProgressBar("Parsing :bar", {
      clear: true,
      total: this.fs.folderPaths.length + this.fs.filePaths.length,
      width: process.stdout.columns,
    });

    info("Parsing", progress.total, "files");

    this.fs.folderPaths.forEach((fullPath) => {
      files[fullPath] = { exports: [], imports: {} };
      progress.tick();
    });

    this.fs.filePaths.forEach((fullPath) => {
      try {
        files[fullPath] = this.parseFile(fullPath);
      } catch (e: any) {
        error(`Error parsing ${fullPath}: ${e.message}`);
        trace(e);
      }

      progress.tick();
    });

    progress.terminate();

    return files;
  }

  private createSourceFile(fullPath: string): SourceFile {
    if (fullPath.endsWith(".vue")) {
      const content = fs.readFileSync(fullPath, "utf-8");
      const scriptMatch = content.match(/<script[^>]*>([\s\S]*?)<\/script>/);
      const scriptContent = scriptMatch ? scriptMatch[1] : "";
      return this.fs.project.createSourceFile(fullPath, scriptContent, {
        overwrite: true,
      });
    }
    return this.fs.project.addSourceFileAtPath(fullPath);
  }

  private parseFile(fullPath: string): File {
    trace(`Parsing ${fullPath}`);

    const sourceFile = this.createSourceFile(fullPath);
    const rootStatements = sourceFile.getStatements();
    const allStatements = getAllStatements(rootStatements);

    debug(fullPath, allStatements.length, "statements");
    const exports = this.getExports(sourceFile, rootStatements);
    const imports = this.getImports(sourceFile, allStatements);
    debug(
      "-",
      Object.keys(exports).length,
      "exports",
      Object.keys(imports).length,
      "imports",
    );

    this.fs.project.removeSourceFile(sourceFile);
    return { exports, imports };
  }

  private getImports(sourceFile: SourceFile, statements: Statement[]): Imports {
    return statements.reduce((imports, statement) => {
      let sourceFileImports: string[] | undefined;

      if (Node.isImportTypeNode(statement)) {
        try {
          const argument = statement.getArgument();
          const moduleSpecifier = eval(argument.getText());
          sourceFileImports = this.addModule(
            imports,
            moduleSpecifier,
            sourceFile,
          );

          const namedImport = statement.getQualifier();

          if (sourceFileImports && namedImport) {
            sourceFileImports.push(namedImport.getText());
          }
        } catch (e) {
          warn(e);
        }
      } else if (
        Node.isVariableStatement(statement) ||
        Node.isExpressionStatement(statement)
      ) {
        const text = statement.getText();
        const [match, moduleSpecifier, namedImport] = Array.from(
          REQUIRE_RE.exec(text) || [],
        );

        if (moduleSpecifier) {
          sourceFileImports = this.addModule(
            imports,
            moduleSpecifier,
            sourceFile,
          );

          if (sourceFileImports && namedImport) {
            sourceFileImports.push(namedImport);
          }
        }
      } else if (
        Node.isImportDeclaration(statement) ||
        Node.isExportDeclaration(statement)
      ) {
        let moduleSpecifier: string | undefined;
        let structure:
          | ImportDeclarationStructure
          | ExportDeclarationStructure
          | undefined;

        try {
          structure = statement.getStructure() as
            | ImportDeclarationStructure
            | ExportDeclarationStructure;
          moduleSpecifier = structure.moduleSpecifier as string | undefined;
        } catch (e) {
          warn(e);
          const brokenLineNumber = statement.getStartLineNumber();
          const brokenLine = sourceFile.getFullText().split(EOL)[
            brokenLineNumber - 1
          ];
          const moduleSpecifierMatch = TEXT_INSIDE_QUOTES_RE.exec(brokenLine);

          if (moduleSpecifierMatch) {
            moduleSpecifier = moduleSpecifierMatch[1];
          }
        }

        if (moduleSpecifier) {
          sourceFileImports = this.addModule(
            imports,
            moduleSpecifier,
            sourceFile,
          );
        }

        if (
          sourceFileImports &&
          structure &&
          Node.isImportDeclaration(statement)
        ) {
          const importStructure = structure as ImportDeclarationStructure;

          if (importStructure.namespaceImport) {
            sourceFileImports.push(importStructure.namespaceImport);
          }

          if (importStructure.defaultImport) {
            sourceFileImports.push(importStructure.defaultImport);
          }

          if (importStructure.namedImports instanceof Array) {
            sourceFileImports.push(
              ...importStructure.namedImports.map((namedImport) =>
                typeof namedImport === "string"
                  ? namedImport
                  : namedImport.name,
              ),
            );
          }

          if (!sourceFileImports.length && !importStructure.namedImports) {
            warn("IMPORT", sourceFile.getBaseName(), structure);
          }
        }
      }

      return imports;
    }, {} as Imports);
  }

  private getExports(sourceFile: SourceFile, statements: Statement[]): Exports {
    const exports = statements.reduce((exports, statement) => {
      const hasExport =
        Node.isExportable(statement) &&
        statement.getText().trimStart().startsWith("export");

      if (hasExport) {
        if (Node.isVariableStatement(statement)) {
          try {
            const structure = statement.getStructure();

            exports.push(
              ...structure.declarations.map((declaration) =>
                String(declaration.name),
              ),
            );
          } catch (e) {
            warn(e);
            warn("isVariableStatement", statement.getText());
          }
        } else if (
          Node.isInterfaceDeclaration(statement) ||
          Node.isClassDeclaration(statement) ||
          Node.isEnumDeclaration(statement) ||
          Node.isTypeAliasDeclaration(statement)
        ) {
          try {
            const structure = statement.getStructure();

            if (structure.name) {
              exports.push(String(structure.name));
            }
          } catch (e) {
            warn(e);
            warn("isInterfaceDeclaration, ...", statement.getText());
          }
        } else if (Node.isFunctionDeclaration(statement)) {
          try {
            const structure = statement.getStructure();
            trace("EXPORT", sourceFile.getBaseName(), structure);
          } catch (e) {
            warn(e);
            warn("isFunctionDeclaration", statement.getText());
          }
        } else {
          warn("EXPORT Unknown type", sourceFile.getBaseName(), statement);
        }
      }

      return exports;
    }, [] as Exports);

    // Fallback: detect CommonJS exports (module.exports = ... / exports.foo = ...)
    if (!exports.length) {
      const text = sourceFile.getFullText();
      let match: RegExpExecArray | null;
      EXPORTS_RE.lastIndex = 0;

      while ((match = EXPORTS_RE.exec(text)) !== null) {
        const namedExport = match[1];
        if (namedExport) {
          exports.push(namedExport);
        } else {
          exports.push(
            path.basename(
              sourceFile.getFilePath(),
              path.extname(sourceFile.getFilePath()),
            ),
          );
        }
      }
    }

    return exports;
  }

  private addModule(
    imports: Imports,
    moduleSpecifier: string,
    sourceFile: SourceFile,
  ): string[] | undefined {
    const modulePath = this.fs.getModulePath(moduleSpecifier, sourceFile);

    if (modulePath) {
      const folder = find(modulePath, this.fs.folderPaths);
      const realModulePath = folder || modulePath;

      if (!imports[realModulePath]) {
        imports[realModulePath] = [];
      }

      return imports[realModulePath];
    } else {
      trace("Import not found", sourceFile.getBaseName(), moduleSpecifier);
    }
  }
}