@tanstack/ai/dist/esm/utilities/sanitize-unicode.js

Type-safe TypeScript AI SDK for streaming chat, tool calling, agents, structured outputs, and multimodal generation.

//#region src/utilities/sanitize-unicode.ts
/** Remove lone UTF-16 surrogates. Keep valid pairs unchanged. */
function sanitizeUnicode(text) {
	return text.replace(/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g, "");
}
/** Sanitize decoded argument strings. Keep original JSON bytes when nothing changes. */
function sanitizeJsonArguments(argumentsJson) {
	try {
		JSON.parse(argumentsJson);
		return argumentsJson.replace(/"(?:\\.|[^"\\])*"/g, (token) => {
			const decoded = JSON.parse(token);
			if (typeof decoded !== "string") return token;
			const clean = sanitizeUnicode(decoded);
			return clean === decoded ? token : JSON.stringify(clean);
		});
	} catch {
		return argumentsJson;
	}
}
/**
* Remove lone surrogates from the text `chat()` sends to a provider. A lone
* surrogate is invalid UTF-8 / JSON on the wire, so providers reject the
* whole request.
*/
function sanitizeProviderRequest(messages, systemPrompts) {
	return {
		messages: messages.map((message) => ({
			...message,
			content: typeof message.content === "string" ? sanitizeUnicode(message.content) : message.content?.map((part) => part.type === "text" ? {
				...part,
				content: sanitizeUnicode(part.content)
			} : part) ?? null,
			...message.toolCalls && { toolCalls: message.toolCalls.map((toolCall) => ({
				...toolCall,
				function: {
					...toolCall.function,
					arguments: sanitizeJsonArguments(toolCall.function.arguments)
				}
			})) }
		})),
		systemPrompts: systemPrompts?.map((prompt) => typeof prompt === "string" ? sanitizeUnicode(prompt) : {
			...prompt,
			content: sanitizeUnicode(prompt.content)
		})
	};
}
//#endregion
export { sanitizeProviderRequest };

//# sourceMappingURL=sanitize-unicode.js.map