@mintlify/scraping/src/openapi/common.ts

Scrape documentation frameworks to Mintlify docs

import {
  getOpenApiTitleAndDescription,
  optionallyAddLeadingSlash,
  slugToTitle,
} from '@mintlify/common';
import type { DecoratedNavigationPage, NavigationEntry } from '@mintlify/models';
import { outputFile } from 'fs-extra';
import fse from 'fs-extra';
import fs from 'fs/promises';
import yaml from 'js-yaml';
import { OpenAPI, OpenAPIV3 } from 'openapi-types';
import path, { join, parse, resolve } from 'path';

import { fetchOpenApi } from '../utils/network.js';

export const getOpenApiDefinition = async (
  pathOrDocumentOrUrl: string | OpenAPI.Document | URL
): Promise<{ document: OpenAPI.Document; isUrl: boolean }> => {
  if (typeof pathOrDocumentOrUrl === 'string') {
    if (pathOrDocumentOrUrl.startsWith('http://')) {
      // This is an invalid location either for a file or a URL
      throw new Error('Only HTTPS URLs are supported. Please provide an HTTPS URL');
    } else {
      try {
        const url = new URL(pathOrDocumentOrUrl);
        pathOrDocumentOrUrl = url;
      } catch {
        const pathname = path.join(process.cwd(), pathOrDocumentOrUrl.toString());
        const file = await fs.readFile(pathname, 'utf-8');
        pathOrDocumentOrUrl = yaml.load(file) as OpenAPI.Document;
      }
    }
  }
  const isUrl = pathOrDocumentOrUrl instanceof URL;
  if (pathOrDocumentOrUrl instanceof URL) {
    if (pathOrDocumentOrUrl.protocol !== 'https:') {
      throw new Error('Only HTTPS URLs are supported. Please provide an HTTPS URL');
    }
    pathOrDocumentOrUrl = await fetchOpenApi(pathOrDocumentOrUrl);
  }

  return { document: pathOrDocumentOrUrl, isUrl };
};

// returns a filename that is unique within the given array of pages
export const generateUniqueFilenameWithoutExtension = (
  pages: NavigationEntry[],
  base: string
): string => {
  let filename = base;
  if (pages.includes(filename)) {
    let extension = 1;
    filename = `${base}-${extension}`;
    while (pages.includes(filename)) {
      extension += 1;
      filename = `${base}-${extension}`;
    }
  }
  return filename;
};

export const createOpenApiFrontmatter = async (
  filename: string,
  openApiMetaTag: string,
  version?: string
) => {
  const data = `---
openapi: ${openApiMetaTag}${version ? `\nversion: ${version}` : ''}
---`;

  await outputFile(filename, data);
};

export const prepareStringToBeValidFilename = (str?: string) =>
  str
    ? str
        .replaceAll(' ', '-')
        .replace(/\{.*?\}/g, '-') // remove path parameters
        .replace(/^-/, '')
        .replace(/-$/, '')
        .replace(/[{}(),.'\n\/]/g, '') // remove special characters
        .replaceAll(/--/g, '-') // replace double hyphens
        .toLowerCase()
    : undefined;

export type GenerateOpenApiPagesOptions = {
  openApiFilePath?: string;
  version?: string;
  writeFiles?: boolean;
  outDir?: string;
  outDirBasePath?: string;
  overwrite?: boolean;
};

export type OpenApiPageGenerationResult<N, DN> = {
  nav: N;
  decoratedNav: DN;
  spec: OpenAPI.Document;
  pagesAcc: Record<string, DecoratedNavigationPage>;
  isUrl: boolean;
};

export function processOpenApiPath<N, DN>(
  path: string,
  pathItemObject: OpenAPIV3.PathItemObject,
  schema: OpenAPI.Document,
  nav: N,
  decoratedNav: DN,
  writePromises: Promise<void>[],
  pagesAcc: Record<string, DecoratedNavigationPage>,
  options: GenerateOpenApiPagesOptions,
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
  findNavGroup: (nav: any, groupName?: string) => any
) {
  const openApiFilePathFromRoot = options.openApiFilePath
    ? optionallyAddLeadingSlash(options.openApiFilePath)
    : undefined;

  Object.values(OpenAPIV3.HttpMethods).forEach((method) => {
    if (method in pathItemObject) {
      const operation = pathItemObject[method];
      const groupName = operation?.tags?.[0];
      const title =
        prepareStringToBeValidFilename(operation?.summary) ??
        `${method}-${prepareStringToBeValidFilename(path)}`;
      const folder = prepareStringToBeValidFilename(groupName) ?? '';
      const base = join(options.outDir ?? '', folder, title);

      const navGroup = findNavGroup(nav, groupName);
      const decoratedNavGroup = findNavGroup(decoratedNav, groupName);

      const filenameWithoutExtension = generateUniqueFilenameWithoutExtension(navGroup, base);
      const openapiMetaTag = `${
        openApiFilePathFromRoot ? `${openApiFilePathFromRoot} ` : ''
      }${method} ${path}`;
      const { title: titleTag, description } = getOpenApiTitleAndDescription(
        [
          {
            filename: options.openApiFilePath
              ? parse(options.openApiFilePath).name
              : 'filler-filename',
            spec: schema,
            originalFileLocation: options.openApiFilePath,
          },
        ],
        openapiMetaTag
      );
      navGroup.push(filenameWithoutExtension);
      const page: DecoratedNavigationPage = {
        openapi: openapiMetaTag,
        href: resolve('/', filenameWithoutExtension),
        title: titleTag ?? slugToTitle(filenameWithoutExtension),
        description,
        version: options.version,
      };
      decoratedNavGroup.push(page);
      pagesAcc[filenameWithoutExtension] = page;
      const targetPath = options.outDirBasePath
        ? join(options.outDirBasePath, `${filenameWithoutExtension}.mdx`)
        : `${filenameWithoutExtension}.mdx`;
      if (options.writeFiles && (!fse.pathExistsSync(targetPath) || options.overwrite)) {
        writePromises.push(createOpenApiFrontmatter(targetPath, openapiMetaTag, options.version));
      }
    }
  });
}

export const DEFAULT_API_GROUP_NAME = 'API Reference';
export const DEFAULT_WEBHOOK_GROUP_NAME = 'Webhooks';

export function processOpenApiWebhook<N, DN>(
  webhook: string,
  webhookObject: OpenAPIV3.PathItemObject,
  _schema: OpenAPI.Document,
  nav: N,
  decoratedNav: DN,
  writePromises: Promise<void>[],
  pagesAcc: Record<string, DecoratedNavigationPage>,
  options: GenerateOpenApiPagesOptions,
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
  findNavGroup: (nav: any, groupName?: string) => any
) {
  const openApiFilePathFromRoot = options.openApiFilePath
    ? optionallyAddLeadingSlash(options.openApiFilePath)
    : undefined;

  Object.values(OpenAPIV3.HttpMethods).forEach((method) => {
    if (method in webhookObject) {
      const operation = webhookObject[method];
      const groupName = operation?.tags?.[0] ?? DEFAULT_WEBHOOK_GROUP_NAME;
      const title =
        prepareStringToBeValidFilename(operation?.summary) ??
        `${prepareStringToBeValidFilename(webhook)}`;
      const folder = prepareStringToBeValidFilename(groupName) ?? '';
      const base = join(options.outDir ?? '', folder, title);

      const navGroup = findNavGroup(nav, groupName);
      const decoratedNavGroup = findNavGroup(decoratedNav, groupName);

      const filenameWithoutExtension = generateUniqueFilenameWithoutExtension(navGroup, base);

      const openapiMetaTag = `${
        openApiFilePathFromRoot ? `${openApiFilePathFromRoot} ` : ''
      }webhook ${webhook}`;

      const description = operation?.description;

      navGroup.push(filenameWithoutExtension);
      const page: DecoratedNavigationPage = {
        openapi: openapiMetaTag,
        href: resolve('/', filenameWithoutExtension),
        title: slugToTitle(filenameWithoutExtension),
        description,
        version: options.version,
      };
      decoratedNavGroup.push(page);
      pagesAcc[filenameWithoutExtension] = page;
      const targetPath = options.outDirBasePath
        ? join(options.outDirBasePath, `${filenameWithoutExtension}.mdx`)
        : `${filenameWithoutExtension}.mdx`;
      if (options.writeFiles && (!fse.pathExistsSync(targetPath) || options.overwrite)) {
        writePromises.push(createOpenApiFrontmatter(targetPath, openapiMetaTag, options.version));
      }
    }
  });
}