@mintlify/scraping/src/tabs/retrieve.ts

Scrape documentation frameworks to Mintlify docs

import { Tab } from '@mintlify/models';
import type { Root as HastRoot, Element } from 'hast';
import { visit, EXIT, CONTINUE } from 'unist-util-visit';

import { framework } from '../utils/detectFramework.js';
import { findTitle, getTitleFromLink } from '../utils/title.js';

export function retrieveTabLinks(rootNode: HastRoot): Array<Tab> | undefined {
  if (framework.vendor !== 'readme' && framework.vendor !== 'docusaurus') return undefined;

  let element: Element | undefined = undefined as Element | undefined;
  visit(rootNode, 'element', function (node) {
    if (framework.vendor === 'readme') {
      if (
        node.tagName === 'header' &&
        node.properties.className &&
        Array.isArray(node.properties.className) &&
        node.properties.className.includes('rm-Header')
      ) {
        element = node;
        return EXIT;
      }
    }

    if (framework.vendor === 'docusaurus') {
      if (
        node.tagName === 'nav' &&
        node.properties.className &&
        Array.isArray(node.properties.className) &&
        node.properties.className.includes('navbar')
      ) {
        element = node;
        return EXIT;
      }
    }
  });

  if (!element) return undefined;

  const links: Array<Tab> = [];
  visit(element as Element, 'element', function (node) {
    if (framework.vendor === 'readme') {
      if (
        node.tagName !== 'nav' &&
        !(
          node.tagName === 'div' &&
          node.properties.className &&
          Array.isArray(node.properties.className) &&
          node.properties.className.includes('rm-Header-right')
        )
      )
        return CONTINUE;

      visit(node, 'element', function (subNode) {
        if (
          subNode.tagName !== 'a' ||
          !subNode.properties.href ||
          typeof subNode.properties.href !== 'string' ||
          subNode.properties.href.startsWith('http')
        )
          return CONTINUE;
        const title = findTitle(subNode);
        links.push({
          name: title || getTitleFromLink(subNode.properties.href),
          url: subNode.properties.href,
        });
      });
    }

    if (framework.vendor === 'docusaurus') {
      if (node.tagName !== 'nav') return CONTINUE;

      visit(node, 'element', function (subNode, _, parent) {
        if (
          subNode.tagName !== 'a' ||
          !subNode.properties.href ||
          typeof subNode.properties.href !== 'string' ||
          subNode.properties.href.startsWith('http') ||
          !parent ||
          parent.type !== 'element' ||
          !Array.isArray(parent.properties.className) ||
          parent.properties.className.length !== 1 ||
          parent.properties.className[0] !== 'navbar__items' ||
          parent.properties.className.includes('navbar__items--right')
        )
          return CONTINUE;

        const title = findTitle(subNode);
        links.push({
          name: title || getTitleFromLink(subNode.properties.href),
          url: subNode.properties.href,
        });
      });
    }
  });

  return links;
}