@stackmemoryai/stackmemory/dist/src/core/execution/parallel-executor.js

Project-scoped memory for AI coding tools. Durable context across sessions with MCP integration, frames, smart retrieval, Claude Code skills, and automatic hooks.

import { fileURLToPath as __fileURLToPath } from 'url';
import { dirname as __pathDirname } from 'path';
const __filename = __fileURLToPath(import.meta.url);
const __dirname = __pathDirname(__filename);
import { EventEmitter } from "events";
import { logger } from "../monitoring/logger.js";
class ParallelExecutor extends EventEmitter {
  maxConcurrency;
  queueSize;
  defaultTimeout;
  defaultRetries;
  rateLimitPerMinute;
  activeCount = 0;
  queue = [];
  rateLimitTokens;
  lastRateLimitReset;
  // Metrics
  totalExecuted = 0;
  totalSucceeded = 0;
  totalFailed = 0;
  totalDuration = 0;
  constructor(maxConcurrency = 5, options = {}) {
    super();
    this.maxConcurrency = maxConcurrency;
    this.queueSize = options.queueSize || 100;
    this.defaultTimeout = options.defaultTimeout || 3e5;
    this.defaultRetries = options.defaultRetries || 3;
    this.rateLimitPerMinute = options.rateLimitPerMinute || 60;
    this.rateLimitTokens = this.rateLimitPerMinute;
    this.lastRateLimitReset = Date.now();
    logger.info("Parallel Executor initialized", {
      maxConcurrency,
      queueSize: this.queueSize,
      rateLimitPerMinute: this.rateLimitPerMinute
    });
  }
  /**
   * Execute multiple tasks in parallel
   */
  async executeParallel(items, executor, options) {
    const results = [];
    const batchSize = options?.batchSize || this.maxConcurrency;
    const delayBetweenBatches = options?.delayBetweenBatches || 0;
    for (let i = 0; i < items.length; i += batchSize) {
      const batch = items.slice(i, i + batchSize);
      const batchPromises = batch.map(
        (item, index) => this.executeWithTracking(
          `parallel-${i + index}`,
          () => executor(item, i + index)
        )
      );
      const batchResults = await Promise.allSettled(batchPromises);
      batchResults.forEach((result, index) => {
        if (result.status === "fulfilled") {
          results.push(result.value);
        } else {
          results.push({
            taskId: `parallel-${i + index}`,
            success: false,
            error: result.reason,
            duration: 0,
            attempts: 1
          });
        }
      });
      if (delayBetweenBatches > 0 && i + batchSize < items.length) {
        await this.delay(delayBetweenBatches);
      }
      logger.debug("Batch completed", {
        batchNumber: Math.floor(i / batchSize) + 1,
        totalBatches: Math.ceil(items.length / batchSize),
        successRate: results.filter((r) => r.success).length / results.length
      });
    }
    return results;
  }
  /**
   * Execute a single task with tracking and retries
   */
  async executeWithTracking(taskId, executor, options) {
    const timeout = options?.timeout || this.defaultTimeout;
    const maxRetries = options?.retries || this.defaultRetries;
    let attempts = 0;
    let lastError;
    const startTime = Date.now();
    while (attempts < maxRetries) {
      attempts++;
      try {
        await this.checkRateLimit();
        await this.waitForSlot();
        this.activeCount++;
        this.emit("task-start", { taskId, attempt: attempts });
        const result = await this.executeWithTimeout(executor, timeout);
        this.totalSucceeded++;
        this.emit("task-success", { taskId, attempts });
        return {
          taskId,
          success: true,
          result,
          duration: Date.now() - startTime,
          attempts
        };
      } catch (error) {
        lastError = error;
        logger.warn(`Task failed (attempt ${attempts}/${maxRetries})`, {
          taskId,
          error: lastError.message
        });
        this.emit("task-retry", { taskId, attempt: attempts, error: lastError });
        if (attempts < maxRetries) {
          await this.delay(Math.pow(2, attempts) * 1e3);
        }
      } finally {
        this.activeCount--;
        this.totalExecuted++;
        this.totalDuration += Date.now() - startTime;
      }
    }
    this.totalFailed++;
    this.emit("task-failed", { taskId, attempts, error: lastError });
    return {
      taskId,
      success: false,
      error: lastError,
      duration: Date.now() - startTime,
      attempts
    };
  }
  /**
   * Execute task with timeout
   */
  async executeWithTimeout(executor, timeout) {
    return Promise.race([
      executor(),
      new Promise(
        (_, reject) => setTimeout(() => reject(new Error("Task timeout")), timeout)
      )
    ]);
  }
  /**
   * Check and enforce rate limiting
   */
  async checkRateLimit() {
    const now = Date.now();
    const timeSinceReset = now - this.lastRateLimitReset;
    if (timeSinceReset >= 6e4) {
      this.rateLimitTokens = this.rateLimitPerMinute;
      this.lastRateLimitReset = now;
    }
    if (this.rateLimitTokens <= 0) {
      const waitTime = 6e4 - timeSinceReset;
      logger.debug(`Rate limit reached, waiting ${waitTime}ms`);
      await this.delay(waitTime);
      this.rateLimitTokens = this.rateLimitPerMinute;
      this.lastRateLimitReset = Date.now();
    }
    this.rateLimitTokens--;
  }
  /**
   * Wait for an available execution slot
   */
  async waitForSlot() {
    while (this.activeCount >= this.maxConcurrency) {
      await this.delay(100);
    }
  }
  /**
   * Queue a task for execution
   */
  async queueTask(task) {
    if (this.queue.length >= this.queueSize) {
      throw new Error("Execution queue is full");
    }
    return new Promise((resolve) => {
      this.queue.push({
        ...task,
        execute: async () => {
          const result = await task.execute();
          resolve({
            taskId: task.id,
            success: true,
            result,
            duration: 0,
            attempts: 1
          });
          return result;
        }
      });
      this.processQueue();
    });
  }
  /**
   * Process queued tasks
   */
  async processQueue() {
    if (this.activeCount >= this.maxConcurrency || this.queue.length === 0) {
      return;
    }
    this.queue.sort((a, b) => (b.priority || 0) - (a.priority || 0));
    const task = this.queue.shift();
    if (task) {
      this.executeWithTracking(task.id, task.execute, {
        timeout: task.timeout,
        retries: task.retries
      }).then(() => {
        this.processQueue();
      });
    }
  }
  /**
   * Utility delay function
   */
  delay(ms) {
    return new Promise((resolve) => setTimeout(resolve, ms));
  }
  /**
   * Get execution metrics
   */
  getMetrics() {
    return {
      activeCount: this.activeCount,
      queueLength: this.queue.length,
      totalExecuted: this.totalExecuted,
      totalSucceeded: this.totalSucceeded,
      totalFailed: this.totalFailed,
      successRate: this.totalExecuted > 0 ? this.totalSucceeded / this.totalExecuted : 0,
      averageDuration: this.totalExecuted > 0 ? this.totalDuration / this.totalExecuted : 0,
      rateLimitTokens: this.rateLimitTokens
    };
  }
  /**
   * Reset all metrics
   */
  resetMetrics() {
    this.totalExecuted = 0;
    this.totalSucceeded = 0;
    this.totalFailed = 0;
    this.totalDuration = 0;
  }
  /**
   * Gracefully shutdown executor
   */
  async shutdown() {
    logger.info("Shutting down Parallel Executor", {
      activeCount: this.activeCount,
      queueLength: this.queue.length
    });
    this.queue = [];
    while (this.activeCount > 0) {
      await this.delay(100);
    }
    logger.info("Parallel Executor shutdown complete");
  }
}
export {
  ParallelExecutor
};
//# sourceMappingURL=parallel-executor.js.map