webforai/dist/md-splitter.js

A library that provides a web interface for AI

// src/md-splitter.ts
import { mdastToMarkdown } from "./mdast-to-markdown.js";
import { chunk } from "./utils/common.js";
import { internalType, unwarpRoot, warpRoot } from "./utils/mdast-utils.js";
var PRIORITY_SPLITTERS = ["h1", "h2", "h3", "h4", "h5", "h6", "list", "table", "code"];
var getSplitterGenerator = function* () {
  for (const splitter of PRIORITY_SPLITTERS) {
    yield splitter;
  }
};
var _mdastSplitter = async (contents, checker, splitterGenerator) => {
  const splitter = splitterGenerator.next().value;
  const markdown = mdastToMarkdown(warpRoot(contents));
  if (await checker(markdown) || contents.length === 1) {
    return [contents];
  }
  const chunked = splitter ? contents.reduce((acc, content) => {
    if (internalType(content) === splitter || acc.length === 0) {
      acc.push([content]);
      return acc;
    }
    acc[acc.length - 1].push(content);
    return acc;
  }, []) : chunk(contents, Math.ceil(contents.length / 2));
  const splitting = chunked.map((chunk2) => _mdastSplitter(chunk2, checker, splitterGenerator));
  return Promise.all(splitting).then((chunks) => chunks.flat());
};
var mdastSplitter = (mdast, checker, _options) => {
  return _mdastSplitter(unwarpRoot(mdast), checker, getSplitterGenerator());
};
export {
  mdastSplitter
};