@mintlify/scraping/src/sdk/converters/sphinx.ts

Scrape documentation frameworks to Mintlify docs

import fse from 'fs-extra';
import type { Element, ElementContent, Root as HastRoot } from 'hast';
import path from 'node:path';
import { visit, SKIP } from 'unist-util-visit';

import { htmlToHast } from '../../pipeline/root.js';
import { turnChildrenIntoMdx } from '../../utils/children.js';
import { removeHastComments } from '../../utils/hastComments.js';
import { hastToMdx } from '../convert.js';
import type { SdkNavGroup, SdkPage, SdkReference } from '../types.js';

const SKIP_PAGES = new Set(['genindex', 'search', 'py-modindex']);
const LANG_MARKER = '@@mintlang:';
const FENCE_MARKER = /(`{3,})[^\S\n]*\n[> ]*@@mintlang:([\w+-]+)@@[^\S\n]*\n/g;
const ADMONITION_TAGS: Record<string, string> = {
  note: 'Note',
  hint: 'Tip',
  tip: 'Tip',
  important: 'Info',
  seealso: 'Info',
  warning: 'Warning',
  caution: 'Warning',
  danger: 'Warning',
  error: 'Warning',
  attention: 'Warning',
};

type SphinxDoc = { slug: string; title: string; body: string; tag?: string };

export async function convertSphinx(sourcePath: string): Promise<SdkReference> {
  const docs = await loadDocs(sourcePath);
  const slugs = new Set(docs.map((doc) => doc.slug));

  const guides = docs
    .filter((doc) => !doc.tag)
    .sort((left, right) =>
      left.slug === 'index' ? -1 : right.slug === 'index' ? 1 : left.slug.localeCompare(right.slug)
    );
  const modules = docs
    .filter((doc) => doc.tag)
    .sort((left, right) => left.slug.localeCompare(right.slug));

  const pages = [...guides, ...modules].map((doc) => renderPage(doc, slugs));
  const groups: SdkNavGroup[] = [];
  if (guides.length) groups.push({ group: 'Getting Started', pages: guides.map((d) => d.slug) });
  if (modules.length) groups.push({ group: 'API Reference', pages: modules.map((d) => d.slug) });

  return { pages, groups };
}

async function loadDocs(sourcePath: string): Promise<SphinxDoc[]> {
  const docs: SphinxDoc[] = [];
  for (const file of await collectFjsonFiles(sourcePath)) {
    const raw = (await fse.readJson(file)) as {
      body?: string;
      title?: string;
      current_page_name?: string;
    };
    const slug =
      raw.current_page_name ??
      path
        .relative(sourcePath, file)
        .split(path.sep)
        .join('/')
        .replace(/\.fjson$/, '');
    if (SKIP_PAGES.has(slug) || slug.startsWith('_modules')) continue;
    if (!raw.body?.trim()) continue;
    docs.push({
      slug,
      title: cleanTitle(raw.title ?? slug),
      body: raw.body,
      tag: /<dl class="py[ "]/.test(raw.body) ? 'MODULE' : undefined,
    });
  }
  return docs;
}

async function collectFjsonFiles(dir: string): Promise<string[]> {
  const files: string[] = [];
  for (const entry of await fse.readdir(dir, { withFileTypes: true })) {
    if (entry.name.startsWith('_')) continue;
    const fullPath = path.join(dir, entry.name);
    if (entry.isDirectory()) files.push(...(await collectFjsonFiles(fullPath)));
    else if (entry.name.endsWith('.fjson')) files.push(fullPath);
  }
  return files.sort();
}

function renderPage(doc: SphinxDoc, slugs: Set<string>): SdkPage {
  const hast = htmlToHast(doc.body);
  removeHastComments(hast);
  removeSphinxChrome(hast);
  const description = extractDescription(hast);
  rewriteLinks(hast, doc.slug, slugs);
  transformAdmonitions(hast);
  markHighlightLanguages(hast);
  transformPyDls(hast, 0);
  const content = hastToMdx(hast)
    .replace(FENCE_MARKER, '$1$2\n')
    .replace(/^[> ]*@@mintlang:[\w+-]+@@\n?/gm, '');
  return {
    slug: doc.slug,
    title: doc.title,
    description,
    tag: doc.tag,
    content,
  };
}

function removeSphinxChrome(hast: HastRoot): void {
  let removedH1 = false;
  visit(hast, 'element', (node: Element, index, parent) => {
    if (!parent || typeof index !== 'number') return;
    const isHeaderlink = node.tagName === 'a' && classList(node).includes('headerlink');
    const isViewcode =
      node.tagName === 'a' &&
      node.children.some(
        (child) => child.type === 'element' && classList(child).includes('viewcode-link')
      );
    const isFirstH1 = node.tagName === 'h1' && !removedH1;
    if (isFirstH1) removedH1 = true;
    if (isHeaderlink || isViewcode || isFirstH1) {
      parent.children.splice(index, 1);
      return index;
    }
  });
}

function rewriteLinks(hast: HastRoot, fromSlug: string, slugs: Set<string>): void {
  visit(hast, 'element', (node: Element) => {
    if (node.tagName !== 'a' || typeof node.properties.href !== 'string') return;
    const rewritten = rewriteHref(node.properties.href, fromSlug, slugs);
    if (rewritten === undefined) delete node.properties.href;
    else node.properties.href = rewritten;
  });
}

function rewriteHref(href: string, fromSlug: string, slugs: Set<string>): string | undefined {
  if (/^(https?:|mailto:|#)/.test(href)) return href;
  const [target = ''] = href.split('#');
  let resolved = path.posix
    .normalize(path.posix.join(path.posix.dirname(fromSlug), target))
    .replace(/\/$/, '')
    .replace(/\.html?$/, '');
  if (resolved === '.' || resolved === '') resolved = 'index';
  return slugs.has(resolved) ? `/${resolved}` : undefined;
}

function transformAdmonitions(hast: HastRoot): void {
  visit(hast, 'element', (node: Element) => {
    if (node.tagName !== 'div' || !classList(node).includes('admonition')) return;
    const type = classList(node).find((name) => name in ADMONITION_TAGS);
    const children = node.children.filter(
      (child) => !(child.type === 'element' && classList(child).includes('admonition-title'))
    );
    node.tagName = ADMONITION_TAGS[type ?? ''] ?? 'Note';
    node.properties = {};
    node.children = turnChildrenIntoMdx(children) as ElementContent[];
    return SKIP;
  });
}

function markHighlightLanguages(hast: HastRoot): void {
  visit(hast, 'element', (node: Element) => {
    const lang = classList(node)
      .find((name) => name.startsWith('highlight-'))
      ?.slice('highlight-'.length);
    if (!lang || lang === 'default' || lang === 'none') return;
    visit(node, 'element', (inner: Element) => {
      if (inner.tagName !== 'pre') return;
      inner.children.unshift({ type: 'text', value: `${LANG_MARKER}${lang}@@\n` });
      return SKIP;
    });
    return SKIP;
  });
}

function transformPyDls(node: HastRoot | Element, depth: number): void {
  const children: ElementContent[] = [];
  for (const child of node.children as ElementContent[]) {
    if (child.type === 'element' && child.tagName === 'dl' && classList(child).includes('py')) {
      children.push(...explodePyDl(child, depth));
    } else {
      if (child.type === 'element') transformPyDls(child, depth);
      children.push(child);
    }
  }
  node.children = children;
}

function explodePyDl(dl: Element, depth: number): ElementContent[] {
  const callable = classList(dl).some((name) => ['method', 'function'].includes(name));
  const out: ElementContent[] = [];
  for (const child of dl.children) {
    if (child.type !== 'element') continue;
    if (child.tagName === 'dt' && classList(child).includes('sig')) {
      const name = findDescName(child);
      if (name) out.push(heading(Math.min(3 + depth, 6), callable ? `${name}()` : name));
      out.push(pythonCodeBlock(textOf(child).replace(/\s+/g, ' ').trim()));
    } else if (child.tagName === 'dd') {
      transformPyDls(child, depth + 1);
      out.push(...child.children);
    }
  }
  return out;
}

function findDescName(node: Element): string | undefined {
  let name: string | undefined;
  visit(node, 'element', (inner: Element) => {
    if (!classList(inner).includes('descname')) return;
    name = textOf(inner).trim();
    return SKIP;
  });
  return name;
}

function heading(level: number, text: string): Element {
  return {
    type: 'element',
    tagName: `h${level}`,
    properties: {},
    children: [{ type: 'text', value: text }],
  };
}

function pythonCodeBlock(code: string): Element {
  return {
    type: 'element',
    tagName: 'pre',
    properties: {},
    children: [{ type: 'text', value: `${LANG_MARKER}python@@\n${code}` }],
  };
}

function extractDescription(hast: HastRoot): string | undefined {
  let description: string | undefined;
  visit(hast, 'element', (node: Element, _index, parent) => {
    if (description || node.tagName !== 'p') return;
    if (parent && parent.type === 'element' && parent.tagName === 'li') return SKIP;
    const text = textOf(node).replace(/\s+/g, ' ').trim();
    if (!text || text.startsWith('Bases:')) return;
    const sentence = text.split(/(?<=\.)\s/)[0] ?? text;
    description = sentence.length > 160 ? `${sentence.slice(0, 159)}…` : sentence;
  });
  return description;
}

function textOf(node: ElementContent): string {
  if (node.type === 'text') return node.value;
  if ('children' in node) return node.children.map(textOf).join('');
  return '';
}

function classList(node: Element): string[] {
  const value = node.properties.className;
  return Array.isArray(value)
    ? value.map(String)
    : typeof value === 'string'
      ? value.split(' ')
      : [];
}

function cleanTitle(title: string): string {
  return title
    .replace(/<[^>]+>/g, '')
    .replace(/&amp;/g, '&')
    .replace(/&lt;/g, '<')
    .replace(/&gt;/g, '>')
    .replace(/&quot;/g, '"')
    .replace(/&#(\d+);/g, (_, code: string) => {
      const codePoint = Number(code);
      return codePoint <= 0x10ffff ? String.fromCodePoint(codePoint) : `&#${code};`;
    })
    .replace(/\s+/g, ' ')
    .trim();
}