shelving/extract/DirectoryExtractor.js

Toolkit for using data in JavaScript.

import { readdir } from "node:fs/promises";
import { splitFileExtension } from "../util/file.js";
import { anyMatch, requirePath, splitPath } from "../util/index.js";
import { Extractor } from "./Extractor.js";
import { FileExtractor } from "./FileExtractor.js";
import { MarkupExtractor } from "./MarkupExtractor.js";
import { TypescriptExtractor } from "./TypescriptExtractor.js";
/** Default file extractor dispatch by extension. */
const DEFAULT_EXTRACTORS = {
    md: new MarkupExtractor(),
    ts: new TypescriptExtractor(),
    tsx: new TypescriptExtractor(),
    txt: new FileExtractor(),
};
/**
 * Default ignore patterns.
 * - Skip test and spec files.
 * - Skip `node_modules` directories.
 * - Skip hidden `.` prefixed and underscore-prefixed files and directories.
 */
const DEFAULT_IGNORE = [/\.test\.tsx?$/i, /\.spec\.tsx?$/i, /^node_modules$/i, /^[_.]/i];
/**
 * Extractor that walks a directory on disk and produces a `DirectoryElement` tree.
 * - Recursively descends into subdirectories.
 * - Dispatches non-ignored files to a matching `FileExtractor` based on extension; files with no matching extractor are silently skipped.
 * - Keys on the produced elements are the verbatim filenames (e.g. `"string.ts"`, `"README.md"`) and directory names (e.g. `"util"`).
 * - This is a pure walker: same-key merging and README absorption are intentionally *not* applied here — wrap with `MergingExtractor`
 *   and/or `IndexFileExtractor` to opt in to those behaviours.
 */
export class DirectoryExtractor extends Extractor {
    _extractors;
    _base;
    _ignore;
    constructor({ extractors = DEFAULT_EXTRACTORS, base, ignore = DEFAULT_IGNORE } = {}) {
        super();
        this._extractors = extractors;
        this._base = base;
        this._ignore = ignore;
    }
    extract(source) {
        return this._extractDirectory(requirePath(source, this._base, this.extract));
    }
    async _extractDirectory(source) {
        const name = splitPath(source).at(-1) ?? "";
        const entries = await readdir(source, { withFileTypes: true });
        const children = [];
        for (const entry of entries) {
            if (anyMatch(entry.name, ...this._ignore))
                continue;
            const child = await this._extractChild(source, entry);
            if (child)
                children.push(child);
        }
        return {
            type: "tree-directory",
            key: name,
            props: {
                source,
                name,
                children,
            },
        };
    }
    async _extractChild(base, entry) {
        const name = entry.name;
        const path = requirePath(name, base);
        if (entry.isDirectory())
            return this._extractDirectory(path);
        if (entry.isFile()) {
            const [stem, extension] = splitFileExtension(name);
            if (!stem || !extension)
                return;
            const extractor = this._extractors[extension];
            if (!extractor)
                return;
            return extractor.extract(Bun.file(path));
        }
    }
}