manifest
Version:
Self-hosted Manifest LLM router with embedded server, SQLite database, and dashboard
147 lines • 5.61 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.extractUsageFromSse = extractUsageFromSse;
exports.initSseHeaders = initSseHeaders;
exports.parseSseEvents = parseSseEvents;
exports.pipeStream = pipeStream;
function extractUsageFromSse(sseText) {
for (const line of sseText.split('\n')) {
const trimmed = line.trim();
if (!trimmed.startsWith('data:'))
continue;
const json = trimmed.slice(5).trim();
if (json === '[DONE]')
continue;
try {
const obj = JSON.parse(json);
if (obj.usage && typeof obj.usage.prompt_tokens === 'number') {
return {
prompt_tokens: obj.usage.prompt_tokens,
completion_tokens: obj.usage.completion_tokens ?? 0,
cache_read_tokens: obj.usage.cache_read_tokens,
cache_creation_tokens: obj.usage.cache_creation_tokens,
};
}
}
catch {
}
}
return null;
}
function initSseHeaders(res, extraHeaders = {}) {
res.setHeader('Content-Type', 'text/event-stream');
res.setHeader('Cache-Control', 'no-cache');
res.setHeader('Connection', 'keep-alive');
res.setHeader('X-Accel-Buffering', 'no');
for (const [key, value] of Object.entries(extraHeaders)) {
res.setHeader(key, value);
}
res.flushHeaders();
}
function parseSseEvents(buffer) {
const events = [];
let remaining = buffer;
let idx;
while ((idx = remaining.indexOf('\n\n')) !== -1) {
const raw = remaining.slice(0, idx).trim();
remaining = remaining.slice(idx + 2);
if (!raw)
continue;
const payload = raw
.split('\n')
.map((line) => (line.startsWith('data: ') ? line.slice(6) : line))
.join('\n')
.trim();
if (payload && payload !== '[DONE]') {
events.push(payload);
}
}
return { events, remaining };
}
const MAX_SSE_BUFFER_SIZE = 1_048_576;
async function pipeStream(source, dest, transform) {
const reader = source.getReader();
const decoder = new TextDecoder();
let sseBuffer = '';
let passthroughBuffer = '';
let capturedUsage = null;
try {
let done = false;
while (!done) {
if (dest.writableEnded)
break;
const result = await reader.read();
done = result.done;
if (result.value) {
const text = decoder.decode(result.value, { stream: !done });
if (transform) {
sseBuffer += text;
if (sseBuffer.length > MAX_SSE_BUFFER_SIZE) {
throw new Error('SSE buffer overflow: provider sent data without event boundaries');
}
const { events, remaining } = parseSseEvents(sseBuffer);
sseBuffer = remaining;
for (const event of events) {
const transformed = transform(event);
if (transformed) {
dest.write(transformed);
const usage = extractUsageFromSse(transformed);
if (usage)
capturedUsage = usage;
}
}
}
else {
dest.write(text);
passthroughBuffer += text;
if (passthroughBuffer.length > MAX_SSE_BUFFER_SIZE) {
throw new Error('SSE buffer overflow: provider sent data without event boundaries');
}
const { events: ptEvents, remaining } = parseSseEvents(passthroughBuffer);
passthroughBuffer = remaining;
for (const ev of ptEvents) {
try {
const obj = JSON.parse(ev);
if (obj.usage && typeof obj.usage.prompt_tokens === 'number') {
capturedUsage = {
prompt_tokens: obj.usage.prompt_tokens,
completion_tokens: obj.usage.completion_tokens ?? 0,
cache_read_tokens: obj.usage.cache_read_tokens,
cache_creation_tokens: obj.usage.cache_creation_tokens,
};
}
}
catch {
}
}
}
}
}
if (transform && sseBuffer.trim()) {
const payload = sseBuffer
.split('\n')
.map((line) => (line.startsWith('data: ') ? line.slice(6) : line))
.join('\n')
.trim();
if (payload && payload !== '[DONE]') {
const transformed = transform(payload);
if (transformed) {
dest.write(transformed);
const usage = extractUsageFromSse(transformed);
if (usage)
capturedUsage = usage;
}
}
}
if (transform) {
dest.write('data: [DONE]\n\n');
}
}
finally {
reader.releaseLock();
if (!dest.writableEnded)
dest.end();
}
return capturedUsage;
}
//# sourceMappingURL=stream-writer.js.map