openclaw/dist/chunk-DqjJztIp.js

Multi-channel AI gateway with extensible messaging integrations

import { t as avoidTrailingHighSurrogateBreak } from "./utf16-slice-D_ngcYKd.js";
import "./session-key-BnWWjqNc.js";
import { n as normalizeAccountId } from "./account-id-CETVCrTz.js";
import { t as resolveAccountEntry } from "./account-lookup-CvFhSs3G.js";
import "./message-channel-constants-2zSoJXQC.js";
import { n as isSafeFenceBreak, r as parseFenceSpans, t as findFenceSpanAt } from "./fences-DtD_oijY.js";
import { b as resolveChannelStreamingChunkMode } from "./streaming-B_uLiLJZ.js";
import { n as normalizeChunkLimit, t as chunkTextByBreakResolver } from "./text-chunking-DrL5h9H9.js";
//#region src/auto-reply/chunk.ts
const DEFAULT_CHUNK_LIMIT = 4e3;
const DEFAULT_CHUNK_MODE = "length";
function resolveChunkLimitForProvider(cfgSection, accountId) {
	if (!cfgSection) return;
	const normalizedAccountId = normalizeAccountId(accountId);
	const accounts = cfgSection.accounts;
	if (accounts && typeof accounts === "object") {
		const direct = resolveAccountEntry(accounts, normalizedAccountId);
		if (typeof direct?.textChunkLimit === "number") return direct.textChunkLimit;
	}
	return cfgSection.textChunkLimit;
}
function resolveTextChunkLimit(cfg, provider, accountId, opts) {
	const fallback = typeof opts?.fallbackLimit === "number" && opts.fallbackLimit > 0 ? opts.fallbackLimit : DEFAULT_CHUNK_LIMIT;
	const providerOverride = (() => {
		if (!provider || provider === "webchat") return;
		const providerConfig = (cfg?.channels)?.[provider];
		return resolveChunkLimitForProvider(providerConfig, accountId);
	})();
	if (typeof providerOverride === "number" && providerOverride > 0) return providerOverride;
	return fallback;
}
function resolveChunkModeForProvider(cfgSection, accountId) {
	if (!cfgSection) return;
	const normalizedAccountId = normalizeAccountId(accountId);
	const accounts = cfgSection.accounts;
	if (accounts && typeof accounts === "object") {
		const direct = resolveAccountEntry(accounts, normalizedAccountId);
		const directMode = resolveChannelStreamingChunkMode(direct);
		if (directMode) return directMode;
	}
	return resolveChannelStreamingChunkMode(cfgSection);
}
function resolveChunkMode(cfg, provider, accountId) {
	if (!provider || provider === "webchat") return DEFAULT_CHUNK_MODE;
	const providerConfig = (cfg?.channels)?.[provider];
	return resolveChunkModeForProvider(providerConfig, accountId) ?? DEFAULT_CHUNK_MODE;
}
/**
* Split text on newlines, trimming line whitespace.
* Blank lines are folded into the next non-empty line as leading "\n" prefixes.
* Leading and trailing blank lines are capped to the available UTF-16 space.
* Long lines can be split by length (default) or kept intact via splitLongLines:false.
*/
function chunkByNewline(text, maxLineLength, opts) {
	if (!text) return [];
	const lineLimit = normalizeChunkLimit(maxLineLength);
	if (lineLimit <= 0) return text.trim() ? [text] : [];
	const splitLongLines = opts?.splitLongLines !== false;
	const trimLines = opts?.trimLines !== false;
	const lines = splitByNewline(text, opts?.isSafeBreak);
	const chunks = [];
	let pendingBlankLines = 0;
	for (const line of lines) {
		const trimmed = line.trim();
		if (!trimmed) {
			pendingBlankLines += 1;
			continue;
		}
		const lineValue = trimLines ? trimmed : line;
		const firstCodePointLength = avoidTrailingHighSurrogateBreak(lineValue, 0, 1);
		const maxPrefix = Math.max(0, lineLimit - firstCodePointLength);
		const prefix = "\n".repeat(Math.min(pendingBlankLines, maxPrefix));
		pendingBlankLines = 0;
		if (!splitLongLines || lineValue.length + prefix.length <= lineLimit) {
			chunks.push(prefix + lineValue);
			continue;
		}
		const rawLimit = Math.max(1, lineLimit - prefix.length);
		const firstLimit = avoidTrailingHighSurrogateBreak(lineValue, 0, rawLimit);
		const first = lineValue.slice(0, firstLimit);
		chunks.push(prefix + first);
		const remaining = lineValue.slice(firstLimit);
		if (remaining) chunks.push(...chunkText(remaining, lineLimit));
	}
	const lastChunk = chunks.at(-1);
	if (pendingBlankLines > 0 && lastChunk !== void 0) {
		const trailingLines = Math.min(pendingBlankLines, Math.max(0, lineLimit - lastChunk.length));
		chunks[chunks.length - 1] = lastChunk + "\n".repeat(trailingLines);
	}
	return chunks;
}
/**
* Split text into chunks on paragraph boundaries (blank lines), preserving lists and
* single-newline line wraps inside paragraphs.
*
* - Only breaks at paragraph separators ("\n\n" or more, allowing whitespace on blank lines)
* - Packs multiple paragraphs into a single chunk up to `limit`
* - Falls back to length-based splitting when a single paragraph exceeds `limit`
*   (unless `splitLongParagraphs` is disabled)
*/
function chunkByParagraph(text, limit, opts) {
	if (!text) return [];
	if (limit <= 0) return [text];
	const splitLongParagraphs = opts?.splitLongParagraphs !== false;
	const normalized = text.replace(/\u2029/g, "\n\n").replace(/\r\n?|\u2028/g, "\n");
	if (!/\n[\t ]*\n+/.test(normalized)) {
		if (normalized.length <= limit) return [normalized];
		if (!splitLongParagraphs) return [normalized];
		return chunkText(normalized, limit);
	}
	const spans = parseFenceSpans(normalized);
	const parts = [];
	const separators = [];
	const re = /\n[\t ]*\n+/g;
	let lastIndex = 0;
	for (const match of normalized.matchAll(re)) {
		const idx = match.index ?? 0;
		if (!isSafeFenceBreak(spans, idx)) continue;
		parts.push(normalized.slice(lastIndex, idx));
		separators.push(match[0]);
		lastIndex = idx + match[0].length;
	}
	parts.push(normalized.slice(lastIndex));
	const chunks = [];
	let currentChunk = "";
	const pushParagraph = (paragraph, separatorBefore) => {
		if (!currentChunk) {
			if (paragraph.length <= limit) {
				currentChunk = paragraph;
				return;
			}
			if (!splitLongParagraphs) {
				chunks.push(paragraph);
				return;
			}
			chunks.push(...chunkText(paragraph, limit));
			return;
		}
		const candidate = `${currentChunk}${separatorBefore ?? "\n\n"}${paragraph}`;
		if (candidate.length <= limit) {
			currentChunk = candidate;
			return;
		}
		chunks.push(currentChunk);
		currentChunk = "";
		pushParagraph(paragraph);
	};
	for (const [index, part] of parts.entries()) {
		const paragraph = part.replace(/\s+$/g, "");
		if (!paragraph.trim()) continue;
		pushParagraph(paragraph, separators[index - 1]);
	}
	if (currentChunk) chunks.push(currentChunk);
	return chunks;
}
/**
* Unified chunking function that dispatches based on mode.
*/
function chunkTextWithMode(text, limit, mode) {
	if (mode === "newline") return chunkByParagraph(text, limit);
	return chunkText(text, limit);
}
function chunkMarkdownTextWithMode(text, limit, mode) {
	const normalizedLimit = normalizeChunkLimit(limit);
	if (mode === "newline") {
		const paragraphChunks = chunkByParagraph(text, normalizedLimit, { splitLongParagraphs: false });
		const out = [];
		for (const chunk of paragraphChunks.flatMap((paragraphChunk) => paragraphChunk.length > normalizedLimit ? splitPackedFenceParagraphChunk(paragraphChunk) : paragraphChunk)) out.push(...chunkMarkdownText(chunk, normalizedLimit));
		return out;
	}
	return chunkMarkdownText(text, normalizedLimit);
}
function splitByNewline(text, isSafeBreak = () => true) {
	const lines = [];
	let start = 0;
	for (let i = 0; i < text.length; i++) if (text[i] === "\n" && isSafeBreak(i)) {
		lines.push(text.slice(start, i));
		start = i + 1;
	}
	lines.push(text.slice(start));
	return lines;
}
function splitPackedFenceParagraphChunk(chunk) {
	const chunks = [];
	let start = 0;
	for (const span of parseFenceSpans(chunk)) {
		if (span.end <= start) continue;
		const separator = chunk.slice(span.end).match(/^\n[\t ]*\n+/)?.[0];
		if (!separator) continue;
		if (!chunk.slice(span.end + separator.length).trim()) continue;
		chunks.push(chunk.slice(start, span.end));
		start = span.end + separator.length;
	}
	if (chunks.length === 0) return [chunk];
	const tail = chunk.slice(start);
	if (tail) chunks.push(tail);
	return chunks;
}
function resolveChunkEarlyReturn(text, limit) {
	if (!text) return [];
	if (limit <= 0) return [text];
	if (text.length <= limit) return [text];
}
function chunkText(text, limit) {
	const early = resolveChunkEarlyReturn(text, limit);
	if (early) return early;
	return chunkTextByBreakResolver(text, limit, (window) => {
		const { lastNewline, lastWhitespace } = scanParenAwareBreakpoints(window, 0, window.length);
		return lastNewline > 0 ? lastNewline : lastWhitespace;
	});
}
function chunkMarkdownText(text, limit) {
	const normalizedLimit = normalizeChunkLimit(limit);
	const early = resolveChunkEarlyReturn(text, normalizedLimit);
	if (early) return early;
	const chunks = [];
	const spans = parseFenceSpans(text);
	let start = 0;
	let reopenFence;
	while (start < text.length) {
		const reopenLine = reopenFence ? resolveFenceReopenLine(reopenFence, normalizedLimit) : "";
		const reopenPrefix = reopenLine ? `${reopenLine}\n` : "";
		const contentLimit = Math.max(1, normalizedLimit - reopenPrefix.length);
		if (text.length - start <= contentLimit) {
			const finalChunk = `${reopenPrefix}${text.slice(start)}`;
			if (finalChunk.length > 0) chunks.push(finalChunk);
			break;
		}
		reopenFence = void 0;
		const windowEnd = Math.min(text.length, start + contentLimit);
		const softBreak = pickSafeBreakIndex(text, start, windowEnd, spans);
		let breakIdx = softBreak > start ? softBreak : windowEnd;
		const initialFence = findFenceSpanAt(spans, breakIdx);
		let fenceToSplit = initialFence;
		if (initialFence) {
			const closeLine = `${initialFence.indent}${initialFence.marker}`;
			if (!resolveFenceReopenLine(initialFence, normalizedLimit)) {
				breakIdx = windowEnd;
				fenceToSplit = void 0;
			} else {
				const maxIdxIfNeedNewline = start + (contentLimit - (closeLine.length + 1));
				const minProgressIdx = Math.min(text.length, reopenPrefix ? start + 1 : Math.max(start + 1, initialFence.start + initialFence.openLine.length + 2));
				const maxIdxIfAlreadyNewline = start + (contentLimit - closeLine.length);
				let pickedNewline = false;
				let lastNewline = text.lastIndexOf("\n", Math.max(start, maxIdxIfAlreadyNewline - 1));
				while (lastNewline >= start) {
					const candidateBreak = lastNewline + 1;
					if (candidateBreak < minProgressIdx) break;
					const candidateFence = findFenceSpanAt(spans, candidateBreak);
					if (candidateFence && candidateFence.start === initialFence.start) {
						breakIdx = candidateBreak;
						pickedNewline = true;
						break;
					}
					lastNewline = text.lastIndexOf("\n", lastNewline - 1);
				}
				if (!pickedNewline && minProgressIdx >= maxIdxIfAlreadyNewline) {
					breakIdx = windowEnd;
					fenceToSplit = void 0;
					reopenFence = initialFence;
				} else {
					if (!pickedNewline) breakIdx = Math.max(minProgressIdx, maxIdxIfNeedNewline);
					const fenceAtBreak = findFenceSpanAt(spans, breakIdx);
					fenceToSplit = fenceAtBreak && fenceAtBreak.start === initialFence.start ? fenceAtBreak : void 0;
				}
			}
		}
		const safeBreakIdx = avoidTrailingHighSurrogateBreak(text, start, breakIdx);
		if (safeBreakIdx !== breakIdx) {
			breakIdx = safeBreakIdx;
			if (fenceToSplit) {
				const fenceAtBreak = findFenceSpanAt(spans, breakIdx);
				fenceToSplit = fenceAtBreak && fenceAtBreak.start === fenceToSplit.start ? fenceAtBreak : void 0;
			}
		}
		const rawContent = text.slice(start, breakIdx);
		if (!rawContent) break;
		let rawChunk = `${reopenPrefix}${rawContent}`;
		let nextStart = breakIdx;
		if (fenceToSplit) {
			const closeLine = `${fenceToSplit.indent}${fenceToSplit.marker}`;
			rawChunk = rawChunk.endsWith("\n") ? `${rawChunk}${closeLine}` : `${rawChunk}\n${closeLine}`;
			reopenFence = fenceToSplit;
		} else if (!initialFence) {
			const brokeOnSeparator = breakIdx < text.length && /\s/.test(text.charAt(breakIdx));
			nextStart = Math.min(text.length, breakIdx + (brokeOnSeparator ? 1 : 0));
			nextStart = skipLeadingNewlines(text, nextStart);
		}
		chunks.push(rawChunk);
		start = nextStart;
	}
	return chunks;
}
function resolveFenceReopenLine(fence, limit) {
	const markerLine = `${fence.indent}${fence.marker}`;
	if (fence.openLine.length + markerLine.length + 3 <= limit) return fence.openLine;
	return markerLine.length * 2 + 3 <= limit ? markerLine : "";
}
function skipLeadingNewlines(value, start = 0) {
	let i = start;
	while (i < value.length && value[i] === "\n") i++;
	return i;
}
function pickSafeBreakIndex(text, start, end, spans) {
	const { lastNewline, lastWhitespace } = scanParenAwareBreakpoints(text, start, end, (index) => findFenceSpanAt(spans, index)?.end);
	if (lastNewline > start) return lastNewline;
	if (lastWhitespace > start) return lastWhitespace;
	return -1;
}
function scanParenAwareBreakpoints(text, start, end, skipTo) {
	let lastNewline = -1;
	let lastWhitespace = -1;
	let depth = 0;
	for (let i = start; i < end; i++) {
		const skippedEnd = skipTo?.(i);
		if (skippedEnd !== void 0) {
			i = skippedEnd - 1;
			continue;
		}
		const char = text.charAt(i);
		if (char === "(") {
			depth += 1;
			continue;
		}
		if (char === ")" && depth > 0) {
			depth -= 1;
			continue;
		}
		if (depth !== 0) continue;
		if (char === "\n") lastNewline = i;
		else if (/\s/.test(char)) lastWhitespace = i;
	}
	return {
		lastNewline,
		lastWhitespace
	};
}
//#endregion
export { chunkText as a, resolveTextChunkLimit as c, chunkMarkdownTextWithMode as i, chunkByParagraph as n, chunkTextWithMode as o, chunkMarkdownText as r, resolveChunkMode as s, chunkByNewline as t };