@paroicms/site-generator-plugin
Version:
ParoiCMS Site Generator Plugin
145 lines (144 loc) • 5.88 kB
JavaScript
import { messageOf } from "@paroi/data-formatters-lib";
import { ensureDirectory } from "@paroicms/internal-server-lib";
import { readFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { estimateTokenCount } from "./llm-tokens.js";
const debugSep = "\n\n========================\n\n";
export async function debugLlmOutput(ctx, llmTaskName, llmModelName, stepHandle, llmInput) {
const aggregatedInput = Object.values(llmInput).join("\n");
const inputTokenCount = aggregatedInput ? estimateTokenCount(aggregatedInput) : 0;
const stored = await readDebugLlmOutputs(ctx, { llmTaskName, inputTokenCount, llmModelName });
const singleStored = stored && stored.outputs.length === 1
? {
output: stored.outputs[0],
llmReport: stored.llmReport,
}
: undefined;
if (singleStored) {
ctx.logger.info(`[${llmTaskName}][${llmModelName}] Found debug output (skip calling LLM)`);
}
else {
ctx.logger.debug(`[${llmTaskName}][${llmModelName}] Calling LLM… User tokens: ~${inputTokenCount}`);
}
const startTs = Date.now();
return {
stored: singleStored,
async getMessageContent(llmMessage, llmReport) {
const llmMessageContent = llmMessage;
const totalTokens = llmReport.outputTokenCount ?? 0;
ctx.logger.debug(`… done. Duration: ${llmReport.durationMs} ms, Tokens: ~${totalTokens} - [${llmTaskName}][${llmModelName}]`);
await writeDebugLlmInputOutputs(ctx, stepHandle, [
{
llmInput,
llmMessageContent,
},
], llmReport, startTs);
return { output: llmMessageContent, llmReport };
},
};
}
export async function debugBatchLlmOutputs(ctx, llmTaskName, llmModelName, stepHandle, llmInputs) {
const aggregatedInput = llmInputs
.map((llmInput) => Object.values(llmInput).join("\n"))
.join("\n\n");
const inputTokenCount = aggregatedInput ? estimateTokenCount(aggregatedInput) : 0;
const stored = await readDebugLlmOutputs(ctx, { llmTaskName, inputTokenCount, llmModelName });
if (stored) {
ctx.logger.info(`[${llmTaskName}][${llmModelName}] Found debug output (skip calling LLM)`);
}
else {
ctx.logger.debug(`[${llmTaskName}][${llmModelName}] Calling LLM… User tokens: ~${inputTokenCount}`);
}
const startTs = Date.now();
return {
stored,
async getMessageContents({ llmMessages, llmReport }) {
const llmMessageContents = llmMessages;
const duration = Date.now() - startTs;
ctx.logger.debug(`… done. Duration: ${duration} ms, Tokens: ~${llmReport.outputTokenCount} - [${llmTaskName}][${llmModelName}]`);
if (llmMessageContents.length !== llmInputs.length) {
throw new Error(`Expected ${llmInputs.length} LLM outputs, but got ${llmMessageContents.length}`);
}
const list = llmInputs.map((llmInput, i) => {
return {
llmInput,
llmMessageContent: llmMessageContents[i],
};
});
await writeDebugLlmInputOutputs(ctx, stepHandle, list, llmReport, startTs);
return { outputs: llmMessageContents, llmReport };
},
};
}
async function readDebugLlmOutputs(ctx, options) {
const { logger, debugDir } = ctx;
if (!debugDir)
return;
const { llmTaskName, inputTokenCount, llmModelName } = options;
const debugFile = join(debugDir, `${llmTaskName}.txt`);
try {
const debugContent = await readFile(debugFile, "utf8");
const list = debugContent.split(debugSep);
if (list.length < 3)
return;
list.shift();
const outputs = [];
for (let i = 1; i < list.length; i += 2) {
outputs.push(list[i]);
}
const llmReport = {
llmTaskName,
modelName: llmModelName,
inputTokenCount,
durationMs: 0,
outputTokenCount: estimateTokenCount(outputs.join(" ")),
};
logger.debug(`… found debug output for ${llmTaskName} (skip calling LLM)`);
return { outputs, llmReport };
}
catch (error) {
if (error.code !== "ENOENT") {
logger.error(`Error reading debug output from "${debugFile}": ${messageOf(error)}`);
}
}
}
async function writeDebugLlmInputOutputs(ctx, stepHandle, list, llmReport, startTs) {
const { debugDir, sessionId } = ctx;
if (!debugDir)
return;
const dt = new Date(startTs).toISOString();
const nameParts = [
dt.substring(0, 19).replace(/:/g, "-"),
stepHandle?.stepNumber,
llmReport.llmTaskName,
llmReport.errorMessage ? "ERROR" : undefined,
].filter(Boolean);
const baseName = nameParts.join("-");
const header = [
`Model: ${llmReport.modelName}`,
`Task: ${llmReport.llmTaskName}`,
`Input tokens: ~${llmReport.inputTokenCount}`,
`Output tokens: ~${llmReport.outputTokenCount}`,
`Duration: ${llmReport.durationMs} ms`,
`Date: ${dt}`,
];
if (llmReport.errorMessage) {
header.push(`Error: ${llmReport.errorMessage}`);
}
const content = [header.join("\n")];
for (const { llmInput, llmMessageContent } of list) {
content.push(debugSep, llmInputToDebugMessage(llmInput), debugSep, llmMessageContent);
}
const dir = join(debugDir, sessionId);
await ensureDirectory(dir);
await writeFile(join(dir, `${baseName}.txt`), content.join(""));
}
function llmInputToDebugMessage(input) {
return Object.entries(input)
.map(([key, value]) => {
return `<${key}>
${value}
</${key}>`;
})
.join("\n\n");
}