llmforge
Version:
One API, every AI model, instant switching. Change from GPT-4 to Gemini to local models with a single config update. LLMForge is the lightweight, TypeScript-first solution for multi-provider AI applications with zero vendor lock-in.
144 lines • 5.01 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.GroqStreamProcessor = void 0;
class GroqStreamProcessor {
constructor() {
this.completeOutput = '';
this.model = '';
this.chatId = '';
}
/**
* Process a raw SSE data chunk and convert to StreamResponse format
*/
processChunk(rawData) {
var _a, _b, _c;
if (!rawData.trim() || rawData.trim() === '[DONE]') {
return null;
}
const jsonStr = rawData.startsWith('data: ') ? rawData.slice(6) : rawData;
try {
if (jsonStr === '[DONE]') {
return null; // End of stream marker
}
const chunk = JSON.parse(jsonStr);
// Store metadata from first chunk
if (!this.model && chunk.model) {
this.model = chunk.model;
this.chatId = chunk.id;
}
const choice = (_a = chunk.choices) === null || _a === void 0 ? void 0 : _a[0];
if (!choice)
return null;
// Handle content delta
if ((_b = choice.delta) === null || _b === void 0 ? void 0 : _b.content) {
const token = choice.delta.content;
this.completeOutput += token;
return {
type: 'delta',
token,
};
}
if (choice.finish_reason === 'stop') {
const completeResponse = {
type: 'completed',
token: '',
completeOutput: this.completeOutput,
model: this.model,
status: 'completed',
};
// Add usage information if available
if ((_c = chunk.x_groq) === null || _c === void 0 ? void 0 : _c.usage) {
completeResponse.usage = {
input_tokens: chunk.x_groq.usage.prompt_tokens,
output_tokens: chunk.x_groq.usage.completion_tokens,
total_tokens: chunk.x_groq.usage.total_tokens,
};
}
return completeResponse;
}
return null;
}
catch (error) {
console.error('Error parsing chunk:', error, 'Raw data:', rawData);
return null;
}
}
/**
* Create an async generator that yields StreamResponse objects
*/
async *createAsyncGenerator(response) {
if (!response.body) {
throw new Error('Response body is empty');
}
// Reset state for new stream
this.completeOutput = '';
this.model = '';
this.chatId = '';
const reader = response.body.getReader();
let buffer = '';
try {
while (true) {
const { done, value } = await reader.read();
if (done)
break;
const chunk = new TextDecoder().decode(value);
buffer += chunk;
// Process complete lines
const lines = buffer.split('\n');
buffer = lines.pop() || ''; // Keep incomplete line in buffer
for (const line of lines) {
const trimmedLine = line.trim();
if (trimmedLine) {
const processedChunk = this.processChunk(trimmedLine);
if (processedChunk) {
yield processedChunk;
}
}
}
}
if (buffer.trim()) {
const processedChunk = this.processChunk(buffer.trim());
if (processedChunk) {
yield processedChunk;
}
}
}
catch (error) {
console.error('Error in GroqStreamProcessor:', error);
throw new Error(`Failed to process stream: ${error.message}`);
}
finally {
reader.releaseLock();
}
}
async *processRawData(rawData) {
// Reset state
this.completeOutput = '';
this.model = '';
this.chatId = '';
const lines = rawData.split('\n');
for (const line of lines) {
const trimmedLine = line.trim();
if (trimmedLine && !trimmedLine.startsWith('data: [DONE]')) {
const processedChunk = this.processChunk(trimmedLine);
if (processedChunk) {
yield processedChunk;
}
}
}
yield {
type: 'done',
token: '',
};
}
getCompleteOutput() {
return this.completeOutput;
}
reset() {
this.completeOutput = '';
this.model = '';
this.chatId = '';
}
}
exports.GroqStreamProcessor = GroqStreamProcessor;
//# sourceMappingURL=groq.stream.js.map