@apideck/docs-mcp/dist/src/store.js

MCP server that lets agents search and retrieve your documentation at query time instead of relying on stale training data.

// SPDX-License-Identifier: MIT
import { watch } from 'node:fs';
import { readdir, readFile, stat } from 'node:fs/promises';
import path from 'node:path';
import MiniSearch from 'minisearch';
import { countWords, deriveTitle, extractLinks, frontmatterDate, makeSnippet, parseFrontmatter, splitSections, toPlainText, } from './markdown.js';
/**
 * Tokenizer that keeps identifiers such as `search_docs`, `rate-limit` or
 * `foo.bar` as one token and also emits their parts, so an exact identifier
 * query outranks pages that merely contain the words.
 */
export const tokenize = (text) => {
    const out = [];
    for (const raw of text.split(/[^\p{L}\p{N}_.-]+/u)) {
        const token = raw.replace(/^[._-]+|[._-]+$/g, '');
        if (token === '')
            continue;
        out.push(token);
        if (/[._-]/.test(token)) {
            for (const part of token.split(/[._-]+/))
                if (part !== '')
                    out.push(part);
        }
    }
    return out;
};
const DEFAULT_EXTENSIONS = ['.md', '.mdx', '.markdown'];
const DEFAULT_IGNORE = ['node_modules', '.git', '.next', 'dist', '.vercel'];
const INDEX_NAMES = ['index', 'README', 'readme'];
/** Turn any user-supplied reference into the canonical page path. */
export const normalizePath = (ref) => ref
    .trim()
    .replace(/\\/g, '/')
    .replace(/^docs:\/\//, '')
    .replace(/[#?].*$/, '')
    .replace(/^(\.\/|\/)+/, '')
    .replace(/\.(mdx?|markdown)$/i, '')
    .replace(/\/+$/, '');
/** URL for a section: page URL plus anchor, unless the page URL already carries a fragment (e.g. a Redoc deep link). */
export const sectionUrl = (page, anchor) => {
    if (page.url === undefined)
        return undefined;
    if (anchor === '' || page.url.includes('#'))
        return page.url;
    return `${page.url}#${anchor}`;
};
const stripSuffix = (s, suffixes) => {
    for (const suffix of suffixes) {
        if (s.toLowerCase().endsWith(suffix.toLowerCase()))
            return s.slice(0, -suffix.length);
    }
    return s;
};
export class DocStore {
    root;
    baseUrl;
    metadata;
    extensions;
    ignore;
    pagesByPath = new Map();
    pagesByUrl = new Map();
    index = null;
    indexedAt = null;
    loading = null;
    constructor(opts) {
        this.root = path.resolve(opts.root);
        this.baseUrl = opts.baseUrl?.replace(/\/+$/, '');
        this.metadata = opts.metadata;
        this.extensions = (opts.extensions ?? DEFAULT_EXTENSIONS).map((e) => e.startsWith('.') ? e.toLowerCase() : `.${e.toLowerCase()}`);
        this.ignore = new Set(opts.ignore ?? DEFAULT_IGNORE);
    }
    /** Read every page under `root` and rebuild the search index. Safe to call repeatedly. */
    async load() {
        if (this.loading)
            return this.loading;
        this.loading = this.rebuild().finally(() => {
            this.loading = null;
        });
        return this.loading;
    }
    async rebuild() {
        const info = await stat(this.root).catch(() => null);
        if (!info?.isDirectory()) {
            throw new Error(`Docs directory not found: ${this.root}`);
        }
        const files = await this.walk(this.root);
        const pages = await Promise.all(files.map((file) => this.readPage(file)));
        pages.sort((a, b) => a.path.localeCompare(b.path));
        const index = new MiniSearch({
            idField: 'id',
            fields: ['title', 'heading', 'content', 'path'],
            storeFields: ['path', 'title', 'heading', 'anchor', 'content'],
            tokenize,
            searchOptions: {
                boost: { title: 3, heading: 2, path: 1.5 },
                prefix: true,
                fuzzy: 0.2,
            },
        });
        index.addAll(pages.flatMap((p) => p.sections));
        this.pagesByPath = new Map(pages.map((p) => [p.path, p]));
        this.pagesByUrl = new Map(pages.filter((p) => p.url !== undefined).map((p) => [p.url, p]));
        this.index = index;
        this.indexedAt = new Date();
    }
    async walk(dir) {
        const entries = await readdir(dir, { withFileTypes: true });
        const out = [];
        for (const entry of entries) {
            if (entry.name.startsWith('.') || this.ignore.has(entry.name))
                continue;
            const full = path.join(dir, entry.name);
            if (entry.isDirectory()) {
                out.push(...(await this.walk(full)));
            }
            else if (entry.isFile() && this.extensions.includes(path.extname(entry.name).toLowerCase())) {
                out.push(full);
            }
        }
        return out;
    }
    async readPage(file) {
        const [raw, info] = await Promise.all([readFile(file, 'utf8'), stat(file)]);
        const rel = path.relative(this.root, file).split(path.sep).join('/');
        const docPath = normalizePath(rel);
        const { data, body } = parseFrontmatter(raw);
        const rawSections = splitSections(body);
        const meta = this.metadata?.(docPath) ?? {};
        const title = meta.title ?? deriveTitle(data, rawSections, docPath);
        const description = meta.description ?? (typeof data['description'] === 'string' ? data['description'] : undefined);
        const url = meta.url ?? this.urlFor(docPath, data);
        const sections = rawSections.map((s) => ({
            id: s.level === 0 ? docPath : `${docPath}#${s.anchor}`,
            path: docPath,
            title,
            heading: s.heading,
            headingPath: s.headingPath,
            anchor: s.anchor,
            level: s.level,
            content: s.content,
            startLine: s.startLine,
        }));
        return {
            path: docPath,
            file,
            title,
            ...(description !== undefined ? { description } : {}),
            frontmatter: data,
            body,
            lastModified: frontmatterDate(data) ?? info.mtime,
            words: countWords(body),
            sections,
            links: extractLinks(body),
            ...(url !== undefined ? { url } : {}),
        };
    }
    urlFor(docPath, frontmatter) {
        const explicit = frontmatter['url'];
        if (typeof explicit === 'string' && explicit !== '')
            return explicit;
        if (this.baseUrl === undefined)
            return undefined;
        const clean = stripSuffix(docPath, INDEX_NAMES.map((n) => `/${n}`));
        const publicPath = INDEX_NAMES.includes(clean) ? '' : clean;
        return `${this.baseUrl}/${publicPath}`.replace(/\/+$/, '') || this.baseUrl;
    }
    ensureLoaded() {
        if (!this.index)
            throw new Error('DocStore not loaded; call load() first');
    }
    stats() {
        const pages = [...this.pagesByPath.values()];
        return {
            root: this.root,
            pages: pages.length,
            sections: pages.reduce((n, p) => n + p.sections.length, 0),
            words: pages.reduce((n, p) => n + p.words, 0),
            indexedAt: this.indexedAt,
        };
    }
    listPages(pathPrefix) {
        this.ensureLoaded();
        const prefix = pathPrefix === undefined ? '' : normalizePath(pathPrefix);
        return [...this.pagesByPath.values()].filter((p) => prefix === '' || p.path === prefix || p.path.startsWith(`${prefix}/`));
    }
    /** Resolve a page by path, tolerating extensions, leading `./`, `docs://`, anchors and folder-index shorthand. */
    get(ref) {
        this.ensureLoaded();
        const p = normalizePath(ref);
        const candidates = p === '' ? INDEX_NAMES : [p, ...INDEX_NAMES.map((n) => `${p}/${n}`)];
        for (const c of candidates) {
            const hit = this.pagesByPath.get(c);
            if (hit)
                return hit;
        }
        const lower = p.toLowerCase();
        for (const [key, page] of this.pagesByPath) {
            if (key.toLowerCase() === lower)
                return page;
        }
        return undefined;
    }
    /**
     * Resolve a page by its public URL (as configured via `baseUrl` or the
     * `metadata` hook), tolerating a trailing slash and a `#fragment`. Returns
     * the page plus the fragment as a candidate section anchor, when the URL's
     * own fragment isn't already baked into the page's stored URL (e.g. a
     * Redoc deep link) — matching that case would strip a fragment the page
     * actually needs.
     */
    getByUrl(url) {
        this.ensureLoaded();
        const trimmed = url.trim().replace(/\/(?=(?:#|$))/, '');
        const direct = this.pagesByUrl.get(trimmed);
        if (direct)
            return { page: direct };
        const hashIndex = trimmed.indexOf('#');
        if (hashIndex === -1)
            return undefined;
        const withoutFragment = trimmed.slice(0, hashIndex);
        const page = this.pagesByUrl.get(withoutFragment);
        if (!page)
            return undefined;
        const anchor = trimmed.slice(hashIndex + 1);
        return anchor === '' ? { page } : { page, anchor };
    }
    /** Resolve a relative link from `fromPath` to a page in the store. */
    resolveLink(fromPath, href) {
        const target = href.replace(/[#?].*$/, '');
        if (target === '')
            return this.get(fromPath);
        const fromDir = path.posix.dirname(fromPath);
        const joined = target.startsWith('/')
            ? target
            : path.posix.normalize(path.posix.join(fromDir === '.' ? '' : fromDir, target));
        if (joined.startsWith('..'))
            return undefined;
        return this.get(joined);
    }
    search(query, opts = {}) {
        this.ensureLoaded();
        const index = this.index;
        const limit = opts.limit ?? 8;
        const perPage = opts.perPage ?? 3;
        const prefix = opts.pathPrefix === undefined ? '' : normalizePath(opts.pathPrefix);
        const filter = prefix === ''
            ? undefined
            : (r) => r['path'] === prefix || String(r['path']).startsWith(`${prefix}/`);
        const run = (combineWith) => index.search(query, { combineWith, ...(filter ? { filter } : {}) });
        let results = run('AND');
        if (results.length === 0)
            results = run('OR');
        const terms = query.split(/\s+/).filter(Boolean);
        const counts = new Map();
        const hits = [];
        for (const r of results) {
            const docPath = r['path'];
            const seen = counts.get(docPath) ?? 0;
            if (seen >= perPage)
                continue;
            counts.set(docPath, seen + 1);
            const page = this.pagesByPath.get(docPath);
            const anchor = r['anchor'];
            const url = page ? sectionUrl(page, anchor) : undefined;
            hits.push({
                path: docPath,
                title: r['title'],
                heading: r['heading'],
                anchor,
                score: Math.round(r.score * 100) / 100,
                snippet: makeSnippet(toPlainText(r['content']), terms),
                ...(url !== undefined ? { url } : {}),
            });
            if (hits.length >= limit)
                break;
        }
        return hits;
    }
    /**
     * Rebuild the index whenever a file under `root` changes. Returns a function
     * that stops watching. Recursive watching needs Node 20+ on Linux; on older
     * setups this logs a warning and returns a no-op.
     */
    watch(onReload, debounceMs = 300) {
        let watcher;
        try {
            watcher = watch(this.root, { recursive: true }, () => schedule());
        }
        catch (err) {
            console.error(`docs-mcp: file watching unavailable (${String(err)}); index is static`);
            return () => undefined;
        }
        let timer = null;
        const schedule = () => {
            if (timer)
                clearTimeout(timer);
            timer = setTimeout(() => {
                timer = null;
                this.load().then(() => onReload?.(null), (err) => onReload?.(err instanceof Error ? err : new Error(String(err))));
            }, debounceMs);
        };
        watcher.on('error', (err) => onReload?.(err));
        return () => {
            if (timer)
                clearTimeout(timer);
            watcher.close();
        };
    }
}
//# sourceMappingURL=store.js.map