ai-utils.js/composed-function/summarize/summarizeRecursivelyWithTextGenerationAndTokenSplitting.js

Build AI applications, chatbots, and agents with JavaScript and TypeScript.

import { generateText } from "../../model-function/generate-text/generateText.js";
import { splitRecursivelyAtTokenAsSplitFunction } from "../../text-chunk/split/splitRecursively.js";
import { summarizeRecursively } from "./summarizeRecursively.js";
/**
 * Recursively summarizes a text using a text generation model, e.g. for summarization or text extraction.
 * It automatically splits the text into optimal chunks that are small enough to be processed by the model,
 * while leaving enough space for the model to generate text.
 */
export async function summarizeRecursivelyWithTextGenerationAndTokenSplitting({ text, model, prompt, reservedCompletionTokens, join, }, options) {
    const emptyPromptTokens = await model.countPromptTokens(await prompt({ text: "" }));
    return summarizeRecursively({
        split: splitRecursivelyAtTokenAsSplitFunction({
            tokenizer: model.tokenizer,
            maxChunkSize: model.contextWindowSize -
                reservedCompletionTokens -
                emptyPromptTokens,
        }),
        summarize: async (input) => {
            const { text } = await generateText(model.withMaxCompletionTokens(reservedCompletionTokens), await prompt(input), options);
            return text;
        },
        join,
        text,
    }, options);
}