@paroicms/site-generator-plugin/backend/dist/generator/lib/debug-utils.js

ParoiCMS Site Generator Plugin

import { ensureDirectory } from "@paroicms/internal-server-lib";
import { isDef, isObj, messageOf } from "@paroicms/public-anywhere-lib";
import { readFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { estimateTokenCount } from "./llm-tokens.js";
const debugSep = "\n\n========================\n\n";
export async function debugLlmOutput(ctx, llmTaskName, llmModelName, stepHandle, llmInput) {
    const aggregatedInput = Object.values(llmInput).join("\n");
    const inputTokenCount = aggregatedInput ? estimateTokenCount(aggregatedInput) : 0;
    const stored = await readDebugLlmOutputs(ctx, { llmTaskName, inputTokenCount, llmModelName });
    const singleStored = stored && stored.outputs.length === 1
        ? {
            output: stored.outputs[0],
            llmReport: stored.llmReport,
        }
        : undefined;
    if (singleStored) {
        ctx.logger.info(`[${llmTaskName}][${llmModelName}] Found debug output (skip calling LLM)`);
    }
    else {
        ctx.logger.debug(`[${llmTaskName}][${llmModelName}] Calling LLM… User tokens: ~${inputTokenCount}`);
    }
    const startTs = Date.now();
    return {
        stored: singleStored,
        async getMessageContent(llmMessage, llmReport) {
            const llmMessageContent = llmMessage;
            const totalTokens = llmReport.outputTokenCount ?? 0;
            ctx.logger.debug(`… done. Duration: ${llmReport.durationMs} ms, Tokens: ~${totalTokens} - [${llmTaskName}][${llmModelName}]`);
            await writeDebugLlmInputOutputs(ctx, stepHandle, [
                {
                    llmInput,
                    llmMessageContent,
                },
            ], llmReport, startTs);
            return { output: llmMessageContent, llmReport };
        },
    };
}
export async function debugBatchLlmOutputs(ctx, llmTaskName, llmModelName, stepHandle, llmInputs) {
    const aggregatedInput = llmInputs
        .map((llmInput) => Object.values(llmInput).join("\n"))
        .join("\n\n");
    const inputTokenCount = aggregatedInput ? estimateTokenCount(aggregatedInput) : 0;
    const stored = await readDebugLlmOutputs(ctx, { llmTaskName, inputTokenCount, llmModelName });
    if (stored) {
        ctx.logger.info(`[${llmTaskName}][${llmModelName}] Found debug output (skip calling LLM)`);
    }
    else {
        ctx.logger.debug(`[${llmTaskName}][${llmModelName}] Calling LLM… User tokens: ~${inputTokenCount}`);
    }
    const startTs = Date.now();
    return {
        stored,
        async getMessageContents({ llmMessages, llmReport }) {
            const llmMessageContents = llmMessages;
            const duration = Date.now() - startTs;
            ctx.logger.debug(`… done. Duration: ${duration} ms, Tokens: ~${llmReport.outputTokenCount} - [${llmTaskName}][${llmModelName}]`);
            if (llmMessageContents.length !== llmInputs.length) {
                throw new Error(`Expected ${llmInputs.length} LLM outputs, but got ${llmMessageContents.length}`);
            }
            const list = llmInputs.map((llmInput, i) => {
                return {
                    llmInput,
                    llmMessageContent: llmMessageContents[i],
                };
            });
            await writeDebugLlmInputOutputs(ctx, stepHandle, list, llmReport, startTs);
            return { outputs: llmMessageContents, llmReport };
        },
    };
}
async function readDebugLlmOutputs(ctx, options) {
    const { logger, debugDir } = ctx;
    if (!debugDir)
        return;
    const { llmTaskName, inputTokenCount, llmModelName } = options;
    const debugFile = join(debugDir, `${llmTaskName}.txt`);
    try {
        const debugContent = await readFile(debugFile, "utf8");
        const list = debugContent.split(debugSep);
        if (list.length < 3)
            return;
        list.shift();
        const outputs = [];
        for (let i = 1; i < list.length; i += 2) {
            outputs.push(list[i]);
        }
        const llmReport = {
            llmTaskName,
            modelName: llmModelName,
            inputTokenCount,
            durationMs: 0,
            outputTokenCount: estimateTokenCount(outputs.join(" ")),
        };
        logger.debug(`… found debug output for ${llmTaskName} (skip calling LLM)`);
        return { outputs, llmReport };
    }
    catch (error) {
        if (!isObj(error) || (error.code !== "ENOENT" && error.code !== "ENOTDIR")) {
            logger.error(`Error reading debug output from "${debugFile}": ${messageOf(error)}`);
        }
    }
}
async function writeDebugLlmInputOutputs(ctx, stepHandle, list, llmReport, startTs) {
    const { debugDir, sessionId } = ctx;
    if (!debugDir)
        return;
    const dt = new Date(startTs).toISOString();
    const nameParts = [
        dt.substring(0, 19).replace(/:/g, "-"),
        stepHandle?.stepNumber,
        llmReport.llmTaskName,
        llmReport.errorMessage ? "ERROR" : undefined,
    ].filter(isDef);
    const baseName = nameParts.join("-");
    const header = [
        `Model: ${llmReport.modelName}`,
        `Task: ${llmReport.llmTaskName}`,
        `Input tokens: ~${llmReport.inputTokenCount}`,
        `Output tokens: ~${llmReport.outputTokenCount}`,
        `Duration: ${llmReport.durationMs} ms`,
        `Date: ${dt}`,
    ];
    if (llmReport.errorMessage) {
        header.push(`Error: ${llmReport.errorMessage}`);
    }
    const content = [header.join("\n")];
    for (const { llmInput, llmMessageContent } of list) {
        content.push(debugSep, llmInputToDebugMessage(llmInput), debugSep, llmMessageContent);
    }
    const dir = join(debugDir, sessionId);
    await ensureDirectory(dir);
    await writeFile(join(dir, `${baseName}.txt`), content.join(""));
}
function llmInputToDebugMessage(input) {
    return Object.entries(input)
        .map(([key, value]) => {
        return `<${key}>
${value}
</${key}>`;
    })
        .join("\n\n");
}