@tanstack/ai/dist/esm/utilities/sanitize-unicode.js

Type-safe TypeScript AI SDK for streaming chat, tool calling, agents, structured outputs, and multimodal generation.

//#region src/utilities/sanitize-unicode.ts
/** Remove lone UTF-16 surrogates. Keep valid pairs unchanged. */
function sanitizeUnicode(text) {
	return text.replace(/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g, "");
}
/** Sanitize decoded argument strings. Keep original JSON bytes when nothing changes. */
function sanitizeJsonArguments(argumentsJson) {
	try {
		JSON.parse(argumentsJson);
		return argumentsJson.replace(/"(?:\\.|[^"\\])*"/g, (token) => {
			const decoded = JSON.parse(token);
			if (typeof decoded !== "string") return token;
			const clean = sanitizeUnicode(decoded);
			return clean === decoded ? token : JSON.stringify(clean);
		});
	} catch {
		return argumentsJson;
	}
}
/**
* Clean string content that has a block order map. Each text block is
* cleaned on its own, so the map lengths still match the content.
*/
function sanitizeOrderedContent(content, order) {
	let offset = 0;
	let clean = "";
	const blockOrder = [];
	for (const block of order) {
		if (block.type !== "text") {
			blockOrder.push(block);
			continue;
		}
		const text = sanitizeUnicode(content.slice(offset, offset + block.length));
		offset += block.length;
		clean += text;
		if (text.length > 0) blockOrder.push({
			...block,
			length: text.length
		});
	}
	return {
		content: clean + sanitizeUnicode(content.slice(offset)),
		blockOrder
	};
}
/**
* Remove lone surrogates from the text `chat()` sends to a provider. A lone
* surrogate is invalid UTF-8 / JSON on the wire, so providers reject the
* whole request.
*/
function sanitizeProviderRequest(messages, systemPrompts) {
	return {
		messages: messages.map((message) => ({
			...message,
			...typeof message.content === "string" ? message.blockOrder ? sanitizeOrderedContent(message.content, message.blockOrder) : { content: sanitizeUnicode(message.content) } : { content: message.content?.map((part) => part.type === "text" ? {
				...part,
				content: sanitizeUnicode(part.content)
			} : part) ?? null },
			...message.toolCalls && { toolCalls: message.toolCalls.map((toolCall) => ({
				...toolCall,
				function: {
					...toolCall.function,
					arguments: sanitizeJsonArguments(toolCall.function.arguments)
				}
			})) }
		})),
		systemPrompts: systemPrompts?.map((prompt) => typeof prompt === "string" ? sanitizeUnicode(prompt) : {
			...prompt,
			content: sanitizeUnicode(prompt.content)
		})
	};
}
//#endregion
export { sanitizeProviderRequest };

//# sourceMappingURL=sanitize-unicode.js.map