jorel/dist/documents/document.js

A unified wrapper for working with LLMs from multiple providers, including streams, images, documents & automatic tool use.

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.LlmDocument = void 0;
const fs = require("fs");
const shared_1 = require("../shared");
/**
 * A document that can be used for grounding LLM generations (either directly or passed to agents)
 */
class LlmDocument {
    constructor({ id, type, title, content, source, attributes }) {
        this.id = id || (0, shared_1.generateUniqueId)();
        this.type = type || "text";
        this.title = title;
        this.content = content;
        this.source = source;
        this.attributes = attributes || {};
    }
    /**
     * Get the definition of the document (e.g. for serialization)
     */
    get definition() {
        return (0, shared_1.shallowFilterUndefined)({
            id: this.id,
            type: this.type,
            title: this.title,
            content: this.content,
            source: this.source || undefined,
            attributes: this.attributes && Object.keys(this.attributes).length > 0 ? this.attributes : undefined,
        });
    }
    /**
     * Write the document content to a local file
     * @param path
     */
    async writeContentToLocalFile(path) {
        await fs.promises.writeFile(path, this.content, "utf-8");
    }
    /**
     * @deprecated Use `fromFile` instead
     */
    static async fromLocalFile(path, meta = {}) {
        return LlmDocument.fromFile(path, meta);
    }
    /**
     * Create a new document from a local file
     */
    static async fromFile(path, meta = {}) {
        const content = await fs.promises.readFile(path, "utf-8");
        return new LlmDocument({
            id: meta.id || (0, shared_1.generateUniqueId)(),
            type: meta.type || getDocumentType(path),
            title: meta.title || path.split("/").pop() || path,
            content,
            source: path,
        });
    }
    /**
     * Create a list of documents from local files
     */
    static async fromFiles(paths, meta = {}) {
        return Promise.all(paths.map((path, index) => LlmDocument.fromFile(path, {
            ...meta,
            title: meta.title ? `${meta.title} ${index + 1}` : undefined,
        })));
    }
    /**
     * Create a new document from a URL
     */
    static async fromUrl(url, meta = {}) {
        const response = await fetch(url);
        const content = await response.text();
        return new LlmDocument({
            id: meta.id || (0, shared_1.generateUniqueId)(),
            type: meta.type || "text",
            title: meta.title || url,
            content,
            source: url,
        });
    }
    /**
     * Create a list of documents from URLs
     */
    static async fromUrls(urls, meta = {}) {
        return Promise.all(urls.map((url, index) => LlmDocument.fromUrl(url, {
            ...meta,
            title: meta.title ? `${meta.title} ${index + 1}` : undefined,
        })));
    }
    /**
     * Create a new markdown document
     */
    static md(payload) {
        return new LlmDocument({ type: "markdown", ...payload });
    }
    /**
     * Create a new text document
     */
    static text(payload) {
        return new LlmDocument({ type: "text", ...payload });
    }
}
exports.LlmDocument = LlmDocument;
const getDocumentType = (path) => {
    const ext = path.split(".").pop();
    if (ext === "md") {
        return "MarkdownFile";
    }
    if (ext === "txt") {
        return "TextFile";
    }
    return "Document";
};