gpt-research
Version:
Autonomous AI research agent that conducts comprehensive research on any topic and generates detailed reports with citations
339 lines • 9.32 kB
JavaScript
"use strict";
/**
* Worker pool and batch processing utilities
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.AsyncQueue = exports.RateLimiter = exports.Throttle = exports.BatchProcessor = exports.WorkerPool = void 0;
exports.retryWithBackoff = retryWithBackoff;
exports.sleep = sleep;
exports.parallelMap = parallelMap;
exports.series = series;
exports.parallel = parallel;
/**
* Simple promise-based concurrency limiter (replacement for p-limit)
*/
class ConcurrencyLimiter {
concurrency;
running = 0;
queue = [];
constructor(concurrency) {
this.concurrency = concurrency;
}
async run(fn) {
while (this.running >= this.concurrency) {
await new Promise(resolve => this.queue.push(resolve));
}
this.running++;
try {
return await fn();
}
finally {
this.running--;
const next = this.queue.shift();
if (next)
next();
}
}
}
/**
* Worker pool for managing concurrent operations
*/
class WorkerPool {
limiter;
activeJobs = new Set();
completed = 0;
failed = 0;
constructor(concurrency = 5) {
this.limiter = new ConcurrencyLimiter(concurrency);
}
/**
* Execute a function with concurrency control
*/
async execute(fn) {
return this.limiter.run(async () => {
const job = fn();
this.activeJobs.add(job);
try {
const result = await job;
this.completed++;
return result;
}
catch (error) {
this.failed++;
throw error;
}
finally {
this.activeJobs.delete(job);
}
});
}
/**
* Execute multiple functions with concurrency control
*/
async executeAll(functions) {
return Promise.all(functions.map(fn => this.execute(fn)));
}
/**
* Execute functions and return results as they complete
*/
async *executeStream(functions) {
const promises = functions.map((fn, index) => this.execute(fn)
.then(result => ({ index, result }))
.catch(error => ({ index, error })));
for (const promise of promises) {
const result = await promise;
if ('error' in result) {
yield { index: result.index, error: result.error };
}
else {
yield { index: result.index, result: result.result };
}
}
}
/**
* Wait for all active jobs to complete
*/
async waitForAll() {
await Promise.all(Array.from(this.activeJobs));
}
/**
* Get statistics
*/
getStats() {
return {
active: this.activeJobs.size,
completed: this.completed,
failed: this.failed,
total: this.completed + this.failed + this.activeJobs.size
};
}
/**
* Reset statistics
*/
reset() {
this.completed = 0;
this.failed = 0;
}
}
exports.WorkerPool = WorkerPool;
/**
* Batch processor for handling items in batches
*/
class BatchProcessor {
concurrency;
limiter;
constructor(concurrency = 3) {
this.concurrency = concurrency;
this.limiter = new ConcurrencyLimiter(concurrency);
}
/**
* Process items in batches
*/
async processBatch(items, processor, batchSize) {
const actualBatchSize = batchSize || this.concurrency;
const results = [];
// Process in batches
for (let i = 0; i < items.length; i += actualBatchSize) {
const batch = items.slice(i, i + actualBatchSize);
const batchResults = await Promise.all(batch.map(item => this.limiter.run(() => processor(item))));
results.push(...batchResults);
}
return results;
}
/**
* Process items and yield results as they complete
*/
async *processStream(items, processor) {
const promises = items.map((item, index) => this.limiter.run(() => processor(item))
.then(result => ({ index, item, result }))
.catch(error => ({ index, item, error })));
const results = await Promise.allSettled(promises);
for (const result of results) {
if (result.status === 'fulfilled') {
const value = result.value;
if (value.error) {
yield { item: value.item, error: value.error };
}
else {
yield { item: value.item, result: value.result };
}
}
}
}
/**
* Get worker pool statistics
*/
getStats() {
return {
concurrency: this.concurrency
};
}
}
exports.BatchProcessor = BatchProcessor;
/**
* Retry a function with exponential backoff
*/
async function retryWithBackoff(fn, options = {}) {
const { retries = 3, initialDelay = 1000, maxDelay = 30000, factor = 2, onRetry } = options;
let lastError;
let delay = initialDelay;
for (let attempt = 0; attempt <= retries; attempt++) {
try {
return await fn();
}
catch (error) {
lastError = error;
if (attempt === retries) {
throw error;
}
if (onRetry) {
onRetry(error, attempt + 1);
}
await sleep(delay);
delay = Math.min(delay * factor, maxDelay);
}
}
throw lastError;
}
/**
* Sleep for a specified duration
*/
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
/**
* Throttle function calls
*/
class Throttle {
minInterval;
lastCall = 0;
constructor(minInterval) {
this.minInterval = minInterval;
}
async execute(fn) {
const now = Date.now();
const timeSinceLastCall = now - this.lastCall;
if (timeSinceLastCall < this.minInterval) {
await sleep(this.minInterval - timeSinceLastCall);
}
this.lastCall = Date.now();
return fn();
}
}
exports.Throttle = Throttle;
/**
* Rate limiter with token bucket algorithm
*/
class RateLimiter {
maxTokens;
refillRate;
refillInterval;
tokens;
lastRefill = Date.now();
constructor(maxTokens, refillRate, // tokens per second
refillInterval = 1000 // ms
) {
this.maxTokens = maxTokens;
this.refillRate = refillRate;
this.refillInterval = refillInterval;
this.tokens = maxTokens;
}
refill() {
const now = Date.now();
const timePassed = now - this.lastRefill;
const tokensToAdd = (timePassed / 1000) * this.refillRate;
this.tokens = Math.min(this.maxTokens, this.tokens + tokensToAdd);
this.lastRefill = now;
}
async acquire(tokens = 1) {
while (true) {
this.refill();
if (this.tokens >= tokens) {
this.tokens -= tokens;
return;
}
// Wait for refill
const tokensNeeded = tokens - this.tokens;
const waitTime = (tokensNeeded / this.refillRate) * 1000;
await sleep(Math.min(waitTime, this.refillInterval));
}
}
tryAcquire(tokens = 1) {
this.refill();
if (this.tokens >= tokens) {
this.tokens -= tokens;
return true;
}
return false;
}
getAvailableTokens() {
this.refill();
return this.tokens;
}
}
exports.RateLimiter = RateLimiter;
/**
* Simple queue for managing async operations
*/
class AsyncQueue {
queue = [];
processing = false;
processor;
constructor(processor) {
this.processor = processor;
}
enqueue(item) {
this.queue.push(item);
this.process();
}
async process() {
if (this.processing || !this.processor) {
return;
}
this.processing = true;
while (this.queue.length > 0) {
const item = this.queue.shift();
try {
await this.processor(item);
}
catch (error) {
// Log error but continue processing
console.error('Queue processing error:', error);
}
}
this.processing = false;
}
size() {
return this.queue.length;
}
clear() {
this.queue = [];
}
}
exports.AsyncQueue = AsyncQueue;
/**
* Parallel map with concurrency control
*/
async function parallelMap(items, mapper, concurrency = 5) {
const pool = new WorkerPool(concurrency);
const functions = items.map((item, index) => () => mapper(item, index));
return pool.executeAll(functions);
}
/**
* Execute functions in series
*/
async function series(functions) {
const results = [];
for (const fn of functions) {
results.push(await fn());
}
return results;
}
/**
* Execute functions in parallel with a limit
*/
async function parallel(functions, limit = 5) {
const pool = new WorkerPool(limit);
return pool.executeAll(functions);
}
//# sourceMappingURL=workers.js.map