gpt-research
Version:
Autonomous AI research agent that conducts comprehensive research on any topic and generates detailed reports with citations
308 lines • 8.26 kB
JavaScript
;
/**
* Streaming utilities for handling real-time data
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.ProgressTracker = exports.StreamBuffer = exports.TransformStream = exports.StreamProcessor = void 0;
exports.parseSSEStream = parseSSEStream;
exports.createReadableStream = createReadableStream;
exports.iteratorToArray = iteratorToArray;
exports.mergeAsyncIterators = mergeAsyncIterators;
exports.batchStream = batchStream;
exports.rateLimitStream = rateLimitStream;
exports.filterStream = filterStream;
exports.mapStream = mapStream;
exports.reduceStream = reduceStream;
exports.timeoutStream = timeoutStream;
/**
* Parse Server-Sent Events stream
*/
function parseSSEStream(data) {
const lines = data.split('\n');
const messages = [];
for (const line of lines) {
if (line.startsWith('data: ')) {
const content = line.slice(6);
if (content && content !== '[DONE]') {
messages.push(content);
}
}
}
return messages;
}
/**
* Stream processor for handling chunked data
*/
class StreamProcessor {
buffer = '';
decoder = new TextDecoder();
/**
* Process a chunk of data
*/
processChunk(chunk) {
const text = this.decoder.decode(chunk, { stream: true });
this.buffer += text;
const messages = [];
const lines = this.buffer.split('\n');
// Keep the last incomplete line in buffer
this.buffer = lines[lines.length - 1];
// Process complete lines
for (let i = 0; i < lines.length - 1; i++) {
const line = lines[i].trim();
if (line.startsWith('data: ')) {
const content = line.slice(6);
if (content && content !== '[DONE]') {
messages.push(content);
}
}
}
return messages;
}
/**
* Get any remaining buffered data
*/
flush() {
const messages = [];
if (this.buffer.trim().startsWith('data: ')) {
const content = this.buffer.trim().slice(6);
if (content && content !== '[DONE]') {
messages.push(content);
}
}
this.buffer = '';
return messages;
}
/**
* Reset the processor
*/
reset() {
this.buffer = '';
}
}
exports.StreamProcessor = StreamProcessor;
/**
* Create a readable stream from async generator
*/
function createReadableStream(generator) {
return new ReadableStream({
async start(controller) {
try {
for await (const chunk of generator) {
controller.enqueue(chunk);
}
controller.close();
}
catch (error) {
controller.error(error);
}
}
});
}
/**
* Convert async iterator to array
*/
async function iteratorToArray(iterator) {
const result = [];
let next = await iterator.next();
while (!next.done) {
result.push(next.value);
next = await iterator.next();
}
return result;
}
/**
* Merge multiple async iterators
*/
async function* mergeAsyncIterators(...iterators) {
const promises = iterators.map((it, index) => it.next().then(result => ({ index, result })));
const pending = new Set(iterators.map((_, index) => index));
while (pending.size > 0) {
const { index, result } = await Promise.race(Array.from(pending).map(i => promises[i]));
if (result.done) {
pending.delete(index);
}
else {
yield result.value;
promises[index] = iterators[index]
.next()
.then(result => ({ index, result }));
}
}
}
/**
* Transform stream with backpressure support
*/
class TransformStream {
// private queue: R[] = []; // Reserved for future use
// private processing = false; // Reserved for future use
transformer;
constructor(transformer) {
this.transformer = transformer;
}
async *transform(source) {
for await (const value of source) {
const result = await this.transformer(value);
yield result;
}
}
}
exports.TransformStream = TransformStream;
/**
* Batch stream items
*/
async function* batchStream(source, batchSize) {
let batch = [];
for await (const item of source) {
batch.push(item);
if (batch.length >= batchSize) {
yield batch;
batch = [];
}
}
if (batch.length > 0) {
yield batch;
}
}
/**
* Rate limit stream
*/
async function* rateLimitStream(source, itemsPerSecond) {
const delay = 1000 / itemsPerSecond;
let lastEmit = 0;
for await (const item of source) {
const now = Date.now();
const timeSinceLastEmit = now - lastEmit;
if (timeSinceLastEmit < delay) {
await new Promise(resolve => setTimeout(resolve, delay - timeSinceLastEmit));
}
lastEmit = Date.now();
yield item;
}
}
/**
* Filter stream
*/
async function* filterStream(source, predicate) {
for await (const item of source) {
if (await predicate(item)) {
yield item;
}
}
}
/**
* Map stream
*/
async function* mapStream(source, mapper) {
for await (const item of source) {
yield await mapper(item);
}
}
/**
* Reduce stream
*/
async function reduceStream(source, reducer, initialValue) {
let accumulator = initialValue;
for await (const item of source) {
accumulator = await reducer(accumulator, item);
}
return accumulator;
}
/**
* Timeout for async iterators
*/
async function* timeoutStream(source, timeout) {
for await (const item of source) {
const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('Stream timeout')), timeout));
yield await Promise.race([
Promise.resolve(item),
timeoutPromise
]);
}
}
/**
* Buffer stream items
*/
class StreamBuffer {
buffer = [];
maxSize;
constructor(maxSize = 100) {
this.maxSize = maxSize;
}
push(item) {
this.buffer.push(item);
if (this.buffer.length > this.maxSize) {
this.buffer.shift();
}
}
getAll() {
return [...this.buffer];
}
getLast(n) {
return this.buffer.slice(-n);
}
clear() {
this.buffer = [];
}
get size() {
return this.buffer.length;
}
}
exports.StreamBuffer = StreamBuffer;
class ProgressTracker {
current = 0;
total;
startTime;
lastUpdate = 0;
onProgress;
constructor(total, onProgress) {
this.total = total;
this.startTime = Date.now();
this.lastUpdate = this.startTime;
this.onProgress = onProgress;
}
update(current, message) {
this.current = current;
const now = Date.now();
const progress = {
current: this.current,
total: this.total,
percentage: (this.current / this.total) * 100,
message
};
// Calculate ETA
if (this.current > 0) {
const elapsed = now - this.startTime;
const rate = this.current / elapsed;
const remaining = this.total - this.current;
progress.eta = remaining / rate;
}
this.lastUpdate = now;
if (this.onProgress) {
this.onProgress(progress);
}
}
updateProgress(percentage, message) {
const current = Math.round((percentage / 100) * this.total);
this.update(current, message);
}
increment(message) {
this.update(this.current + 1, message);
}
reset() {
this.current = 0;
this.startTime = Date.now();
this.lastUpdate = this.startTime;
}
getProgress() {
return {
current: this.current,
total: this.total,
percentage: (this.current / this.total) * 100
};
}
getLastUpdate() {
return this.lastUpdate;
}
}
exports.ProgressTracker = ProgressTracker;
//# sourceMappingURL=stream.js.map