llmforge
Version:
One API, every AI model, instant switching. Change from GPT-4 to Gemini to local models with a single config update. LLMForge is the lightweight, TypeScript-first solution for multi-provider AI applications with zero vendor lock-in.
335 lines • 14 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.OpenAIStreamProcessor = void 0;
const logger_1 = require("../../utils/logger");
class OpenAIStreamProcessor {
constructor() {
this.decoder = new TextDecoder();
}
async processStream(response, options = {}) {
if (!response.body) {
throw new Error('Response body is empty');
}
const reader = response.body.getReader();
let buffer = '';
let aggregatedResponse = {
candidates: [],
};
try {
while (true) {
const { done, value } = await reader.read();
if (done)
break;
buffer += this.decoder.decode(value, { stream: true });
const lines = buffer.split('\n');
buffer = lines.pop() || ''; // Keep incomplete line in buffer
for (const line of lines) {
if (line.trim()) {
logger_1.logger.info('Raw API stream line:', line);
try {
const openAIChunk = this.parseOpenAIStreamChunk(line);
if (openAIChunk) {
// Handle reasoning content
if (this.hasReasoningContent(openAIChunk)) {
const reasoning = this.extractReasoningContent(openAIChunk);
if (reasoning && options.onReasoning) {
options.onReasoning(reasoning);
}
}
// Handle thinking content (from JSON schema responses)
if (this.hasThinkingContent(openAIChunk)) {
const thinking = this.extractThinkingContent(openAIChunk);
if (thinking && options.onThinking) {
options.onThinking(thinking);
}
}
const geminiChunk = this.convertToGeminiChunk(openAIChunk);
this.aggregateChunk(aggregatedResponse, geminiChunk);
// Call chunk handler with converted chunk
if (options.onChunk) {
options.onChunk(geminiChunk);
}
}
}
catch (error) {
logger_1.logger.warn('Failed to parse stream chunk:', line, error);
}
}
}
}
// Process any remaining buffer content
if (buffer.trim()) {
try {
const openAIChunk = this.parseOpenAIStreamChunk(buffer);
if (openAIChunk) {
const geminiChunk = this.convertToGeminiChunk(openAIChunk);
this.aggregateChunk(aggregatedResponse, geminiChunk);
if (options.onChunk) {
options.onChunk(geminiChunk);
}
}
}
catch (error) {
logger_1.logger.warn('Failed to parse final chunk:', buffer, error);
}
}
if (options.onComplete) {
options.onComplete(aggregatedResponse);
}
return aggregatedResponse;
}
catch (error) {
if (options.onError) {
options.onError(error);
}
throw error;
}
finally {
reader.releaseLock();
}
}
parseOpenAIStreamChunk(line) {
// Remove "data: " prefix if present
const cleanLine = line.replace(/^data:\s*/, '').trim();
if (!cleanLine || cleanLine === '[DONE]') {
return null;
}
try {
return JSON.parse(cleanLine);
}
catch (error) {
// Some lines might not be valid JSON
return null;
}
}
convertToGeminiChunk(openAIChunk) {
const geminiChunk = {
candidates: [],
};
if (openAIChunk.choices) {
geminiChunk.candidates = openAIChunk.choices.map(choice => {
var _a, _b;
return ({
content: {
role: 'model',
parts: [
{
text: ((_a = choice.delta) === null || _a === void 0 ? void 0 : _a.content) || ((_b = choice.message) === null || _b === void 0 ? void 0 : _b.content) || '',
},
],
},
finishReason: choice.finish_reason,
index: choice.index,
});
});
}
if (openAIChunk.usage) {
geminiChunk.usageMetadata = {
promptTokenCount: openAIChunk.usage.prompt_tokens,
candidatesTokenCount: openAIChunk.usage.completion_tokens,
totalTokenCount: openAIChunk.usage.total_tokens,
};
}
return geminiChunk;
}
hasReasoningContent(chunk) {
return chunk.type === 'reasoning' || !!chunk.reasoning;
}
extractReasoningContent(chunk) {
return chunk.reasoning;
}
hasThinkingContent(chunk) {
var _a;
// Check if the content contains JSON with thinking field
return (((_a = chunk.choices) === null || _a === void 0 ? void 0 : _a.some(choice => {
var _a, _b;
const content = ((_a = choice.delta) === null || _a === void 0 ? void 0 : _a.content) || ((_b = choice.message) === null || _b === void 0 ? void 0 : _b.content);
if (!content)
return false;
try {
const parsed = JSON.parse(content);
return parsed.thinking !== undefined;
}
catch (_c) {
return content.includes('"thinking"') || content.includes('thinking:');
}
})) || false);
}
extractThinkingContent(chunk) {
var _a, _b;
for (const choice of chunk.choices || []) {
const content = ((_a = choice.delta) === null || _a === void 0 ? void 0 : _a.content) || ((_b = choice.message) === null || _b === void 0 ? void 0 : _b.content);
if (!content)
continue;
try {
const parsed = JSON.parse(content);
if (parsed.thinking) {
return parsed.thinking;
}
}
catch (_c) {
// Try to extract thinking from partial JSON or text
const thinkingMatch = content.match(/"thinking":\s*"([^"]*)"/) || content.match(/thinking:\s*(.+?)(?:,|\}|$)/);
if (thinkingMatch) {
return thinkingMatch[1].replace(/\\n/g, '\n').replace(/\\"/g, '"');
}
}
}
return null;
}
aggregateChunk(aggregated, chunk) {
var _a;
if (!chunk.candidates)
return;
// Ensure aggregated has candidates property
const agg = aggregated;
for (let i = 0; i < chunk.candidates.length; i++) {
const candidate = chunk.candidates[i];
if (!agg.candidates) {
agg.candidates = [];
}
if (!agg.candidates[i]) {
agg.candidates[i] = {
content: { parts: [] },
index: i,
};
}
const aggregatedCandidate = agg.candidates[i];
// Merge content parts
if ((_a = candidate.content) === null || _a === void 0 ? void 0 : _a.parts) {
for (let j = 0; j < candidate.content.parts.length; j++) {
const part = candidate.content.parts[j];
if (!aggregatedCandidate.content.parts[j]) {
aggregatedCandidate.content.parts[j] = { text: '' };
}
if ('text' in part && part.text) {
const aggregatedPart = aggregatedCandidate.content.parts[j];
if ('text' in aggregatedPart) {
aggregatedPart.text += part.text;
}
}
}
}
if (candidate.finishReason) {
aggregatedCandidate.finishReason = candidate.finishReason;
}
if (candidate.safetyRatings) {
aggregatedCandidate.safetyRatings = candidate.safetyRatings;
}
}
if (chunk.usageMetadata) {
aggregated.usageMetadata = chunk.usageMetadata;
}
}
async *createAsyncGenerator(response) {
var _a, _b;
if (!response.body) {
throw new Error('Response body is empty');
}
const reader = response.body.getReader();
let buffer = '';
try {
while (true) {
const { done, value } = await reader.read();
if (done)
break;
buffer += this.decoder.decode(value, { stream: true });
const lines = buffer.split('\n');
buffer = lines.pop() || '';
for (const line of lines) {
if (line.trim()) {
// logger.info('Raw API stream line:', line);
// Remove event: ... lines, only process data: ...
if (line.startsWith('data:')) {
const dataStr = line.replace(/^data:\s*/, '');
let dataObj;
try {
dataObj = JSON.parse(dataStr);
}
catch (_c) {
continue;
}
// Handle delta tokens
if (dataObj.type === 'response.output_text.delta' && dataObj.delta) {
yield {
type: 'delta',
token: dataObj.delta,
};
}
// Handle final completion
if (dataObj.type === 'response.output_text.done' && dataObj.text) {
yield {
type: 'done',
token: '',
completeOutput: dataObj.text,
thinkingOutput: '',
model: dataObj.model || '',
usage: undefined,
status: dataObj.status || '',
usageMetadata: undefined,
};
}
if (dataObj.type === 'response.completed') {
yield {
type: 'completed',
token: '',
completeOutput: (_b = (_a = dataObj === null || dataObj === void 0 ? void 0 : dataObj.response) === null || _a === void 0 ? void 0 : _a.output[0]) === null || _b === void 0 ? void 0 : _b.content[0].text,
thinkingOutput: '',
model: dataObj.response.model || '',
usage: {
input_tokens: dataObj.response.usage.input_tokens,
output_tokens: dataObj.response.usage.output_tokens,
total_tokens: dataObj.response.usage.total_tokens,
},
status: dataObj.status || '',
};
}
}
}
}
}
if (buffer.trim()) {
// Optionally handle any remaining buffer
}
}
finally {
reader.releaseLock();
}
}
async *createOpenAIAsyncGenerator(response) {
if (!response.body) {
throw new Error('Response body is empty');
}
const reader = response.body.getReader();
let buffer = '';
try {
while (true) {
const { done, value } = await reader.read();
if (done)
break;
buffer += this.decoder.decode(value, { stream: true });
const lines = buffer.split('\n');
buffer = lines.pop() || '';
for (const line of lines) {
if (line.trim()) {
const chunk = this.parseOpenAIStreamChunk(line);
if (chunk) {
yield chunk;
}
}
}
}
if (buffer.trim()) {
const chunk = this.parseOpenAIStreamChunk(buffer);
if (chunk) {
yield chunk;
}
}
}
finally {
reader.releaseLock();
}
}
}
exports.OpenAIStreamProcessor = OpenAIStreamProcessor;
//# sourceMappingURL=openai.stream.js.map