@juspay/neurolink
Version:
The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002
123 lines (122 loc) • 6.02 kB
JavaScript
/**
* Stage 3: Structured LLM Summarization
*
* Uses the structured 10-section prompt to summarize older messages
* while preserving recent ones.
*/
import { randomUUID } from "crypto";
import { generateSummary } from "../../utils/conversationMemory.js";
import { estimateTokens } from "../../utils/tokenEstimation.js";
import { logger } from "../../utils/logger.js";
import { snapSplitToBatchBoundary } from "../toolPairRepair.js";
/**
* Find the split index using token counting — walk backward from the end,
* accumulating token counts until we've reserved `targetRecentTokens` worth
* of recent content. Everything before the split index gets summarized.
*/
function findSplitIndexByTokens(messages, targetRecentTokens, provider) {
let recentTokens = 0;
let splitIndex = messages.length;
for (let i = messages.length - 1; i >= 0; i--) {
// Guarded: JSON.stringify on a giant single message can throw RangeError
// (V8 max string length). A message that big cannot be "recent-window"
// material anyway — treat it as effectively infinite tokens so it (and
// everything older) lands on the summarized side instead of aborting
// compaction.
let msgTokens;
try {
const content = typeof messages[i].content === "string"
? messages[i].content
: (JSON.stringify(messages[i].content) ?? "");
msgTokens = estimateTokens(content, provider);
}
catch {
msgTokens = Number.MAX_SAFE_INTEGER;
}
if (recentTokens + msgTokens > targetRecentTokens) {
splitIndex = i + 1;
break;
}
recentTokens += msgTokens;
}
// Ensure at least one message is summarized, then snap off any tool batch
// the boundary would otherwise cut in half — a summary that starts the
// recent window on an orphaned tool_result is rejected by the provider.
return snapSplitToBatchBoundary(messages, Math.max(1, splitIndex));
}
export async function summarizeMessages(messages, config) {
const keepRecentRatio = config?.keepRecentRatio ?? 0.3;
if (messages.length <= 4) {
return { summarized: false, messages };
}
// Determine split point: prefer token-based when a target budget is available,
// fall back to message-count-based split for backward compatibility.
let splitIndex;
if (config?.targetTokens && config.targetTokens > 0) {
// Keep `keepRecentRatio` fraction of the target budget as recent context
const targetRecentTokens = Math.floor(config.targetTokens * keepRecentRatio);
// NOTE: config.provider is the summarization provider, not the generation
// provider. Ideally we'd use the generation/budget provider for accurate
// token estimation, but SummarizeConfig doesn't carry a separate
// budgetProvider field. This is a known design limitation.
splitIndex = findSplitIndexByTokens(messages, targetRecentTokens, config.provider);
}
else {
// Legacy: message-count-based split
const keepCount = Math.max(4, Math.ceil(messages.length * keepRecentRatio));
splitIndex = messages.length - keepCount;
}
// Clamp so at least the last message is always preserved (never summarize everything)
splitIndex = Math.min(splitIndex, messages.length - 1);
// Re-snap AFTER the clamp: the clamp can pull the boundary back into a tool
// batch that findSplitIndexByTokens had already cleared, and it is also the
// only guard covering the legacy message-count branch above.
splitIndex = snapSplitToBatchBoundary(messages, splitIndex);
// A boundary that still cuts a batch means no legal split exists (e.g. the
// whole array is one tool batch). Summarizing anyway would hand the provider
// an orphaned tool_result, so decline and let a later stage reclaim instead.
if (splitIndex <= 0 ||
splitIndex >= messages.length ||
splitIndex !== snapSplitToBatchBoundary(messages, splitIndex)) {
return { summarized: false, messages };
}
const messagesToSummarize = messages.slice(0, splitIndex);
const recentMessages = messages.slice(splitIndex);
// Find previous summary if exists
const previousSummary = messagesToSummarize.find((m) => m.metadata?.isSummary)?.content;
// Build effective memory config: use provided memoryConfig, or construct from provider/model
const effectiveMemoryConfig = config?.memoryConfig ? { ...config.memoryConfig } : {};
// Fill in summarization provider/model from compactor config if not already set
if (!effectiveMemoryConfig.summarizationProvider && config?.provider) {
effectiveMemoryConfig.summarizationProvider = config.provider;
}
if (!effectiveMemoryConfig.summarizationModel && config?.model) {
effectiveMemoryConfig.summarizationModel = config.model;
}
// Only skip if there's genuinely no provider available
if (!effectiveMemoryConfig.summarizationProvider &&
!effectiveMemoryConfig.summarizationModel) {
logger.debug("[ContextCompactor] Stage 3 skipped: no summarization provider or model available");
return { summarized: false, messages };
}
const summaryText = await generateSummary(messagesToSummarize, effectiveMemoryConfig, "[ContextCompactor]", previousSummary);
if (!summaryText) {
return { summarized: false, messages };
}
const summaryMessage = {
id: `summary-${randomUUID()}`,
role: "user",
content: `[Previous conversation summary]:\n\n${summaryText}`,
timestamp: new Date().toISOString(),
metadata: {
isSummary: true,
summarizesFrom: messagesToSummarize[0]?.id,
summarizesTo: messagesToSummarize[messagesToSummarize.length - 1]?.id,
},
};
return {
summarized: true,
messages: [summaryMessage, ...recentMessages],
summaryText,
};
}