@stackmemoryai/stackmemory/dist/src/core/digest/hybrid-digest-generator.js

Project-scoped memory for AI coding tools. Durable context across sessions with MCP integration, frames, smart retrieval, Claude Code skills, and automatic hooks.

import { fileURLToPath as __fileURLToPath } from 'url';
import { dirname as __pathDirname } from 'path';
const __filename = __fileURLToPath(import.meta.url);
const __dirname = __pathDirname(__filename);
import {
  DEFAULT_DIGEST_CONFIG
} from "./types.js";
import { logger } from "../monitoring/logger.js";
class HybridDigestGenerator {
  db;
  config;
  llmProvider;
  queue = [];
  processing = false;
  idleTimer;
  stats = {
    pending: 0,
    processing: 0,
    completed: 0,
    failed: 0,
    avgProcessingTimeMs: 0
  };
  constructor(db, config = {}, llmProvider) {
    this.db = db;
    this.config = { ...DEFAULT_DIGEST_CONFIG, ...config };
    this.llmProvider = llmProvider;
    this.initializeSchema();
  }
  initializeSchema() {
    this.db.exec(`
      CREATE TABLE IF NOT EXISTS digest_queue (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        frame_id TEXT NOT NULL UNIQUE,
        frame_name TEXT NOT NULL,
        frame_type TEXT NOT NULL,
        priority TEXT DEFAULT 'normal',
        status TEXT DEFAULT 'pending',
        retry_count INTEGER DEFAULT 0,
        created_at INTEGER DEFAULT (unixepoch()),
        updated_at INTEGER DEFAULT (unixepoch()),
        error_message TEXT
      );

      CREATE INDEX IF NOT EXISTS idx_digest_queue_status ON digest_queue(status);
      CREATE INDEX IF NOT EXISTS idx_digest_queue_priority ON digest_queue(priority, created_at);
    `);
  }
  /**
   * Generate digest for a frame (immediate deterministic, queued AI)
   */
  generateDigest(input) {
    const startTime = Date.now();
    const deterministic = this.extractDeterministicFields(input);
    const text = this.generateDeterministicText(input.frame, deterministic);
    const digest = {
      frameId: input.frame.frame_id,
      frameName: input.frame.name,
      frameType: input.frame.type,
      deterministic,
      status: "deterministic_only",
      text,
      version: 1,
      createdAt: Date.now(),
      updatedAt: Date.now()
    };
    if (this.config.enableAIGeneration && this.llmProvider) {
      this.queueForAIGeneration({
        frameId: input.frame.frame_id,
        frameName: input.frame.name,
        frameType: input.frame.type,
        priority: this.determinePriority(input),
        createdAt: Date.now(),
        retryCount: 0,
        maxRetries: this.config.maxRetries
      });
      digest.status = "ai_pending";
    }
    logger.debug("Generated deterministic digest", {
      frameId: input.frame.frame_id,
      durationMs: Date.now() - startTime,
      aiQueued: digest.status === "ai_pending"
    });
    return digest;
  }
  /**
   * Extract deterministic fields from frame data (60%)
   */
  extractDeterministicFields(input) {
    const { frame, anchors, events } = input;
    const filesModified = this.extractFilesModified(events);
    const testsRun = this.extractTestResults(events);
    const errorsEncountered = this.extractErrors(events);
    const toolCalls = events.filter((e) => e.event_type === "tool_call");
    const toolCallsByType = {};
    for (const tc of toolCalls) {
      const toolName = tc.payload?.tool_name || "unknown";
      toolCallsByType[toolName] = (toolCallsByType[toolName] || 0) + 1;
    }
    const anchorCounts = {};
    for (const anchor of anchors) {
      anchorCounts[anchor.type] = (anchorCounts[anchor.type] || 0) + 1;
    }
    const decisions = anchors.filter((a) => a.type === "DECISION").map((a) => a.text);
    const constraints = anchors.filter((a) => a.type === "CONSTRAINT").map((a) => a.text);
    const risks = anchors.filter((a) => a.type === "RISK").map((a) => a.text);
    const durationSeconds = frame.closed_at ? frame.closed_at - frame.created_at : Math.floor(Date.now() / 1e3 - frame.created_at);
    const exitStatus = this.determineExitStatus(frame, errorsEncountered);
    return {
      filesModified,
      testsRun,
      errorsEncountered,
      toolCallCount: toolCalls.length,
      toolCallsByType,
      durationSeconds,
      exitStatus,
      anchorCounts,
      decisions,
      constraints,
      risks
    };
  }
  extractFilesModified(events) {
    const fileMap = /* @__PURE__ */ new Map();
    for (const event of events) {
      if (event.event_type === "tool_call" || event.event_type === "tool_result") {
        const payload = event.payload || {};
        const filePath = payload.file_path || payload.path || payload.file;
        if (filePath && typeof filePath === "string") {
          const toolName = payload.tool_name || "";
          let operation = "read";
          if (toolName.includes("write") || toolName.includes("edit") || toolName.includes("create")) {
            operation = "modify";
          } else if (toolName.includes("delete") || toolName.includes("remove")) {
            operation = "delete";
          } else if (toolName.includes("read") || toolName.includes("cat") || toolName.includes("view")) {
            operation = "read";
          }
          const existing = fileMap.get(filePath);
          if (!existing || this.operationPriority(operation) > this.operationPriority(existing.operation)) {
            fileMap.set(filePath, {
              path: filePath,
              operation,
              linesChanged: payload.lines_changed
            });
          }
        }
        const filesAffected = payload.filesAffected || payload.files_affected;
        if (Array.isArray(filesAffected)) {
          for (const f of filesAffected) {
            if (typeof f === "string" && !fileMap.has(f)) {
              fileMap.set(f, { path: f, operation: "modify" });
            }
          }
        }
      }
    }
    return Array.from(fileMap.values());
  }
  operationPriority(op) {
    const priorities = { delete: 4, create: 3, modify: 2, read: 1 };
    return priorities[op] || 0;
  }
  extractTestResults(events) {
    const tests = [];
    for (const event of events) {
      const payload = event.payload || {};
      if (payload.tool_name?.includes("test") || payload.command?.includes("test") || payload.test_name) {
        const testName = payload.test_name || payload.command || "unknown test";
        const success = payload.success !== false && !payload.error;
        tests.push({
          name: testName,
          status: success ? "passed" : "failed",
          duration: payload.duration
        });
      }
      const output = payload.output || payload.result;
      if (typeof output === "string") {
        const passMatch = output.match(/(\d+)\s*(?:tests?\s*)?passed/i);
        const failMatch = output.match(/(\d+)\s*(?:tests?\s*)?failed/i);
        if (passMatch || failMatch) {
          const passed = passMatch ? parseInt(passMatch[1], 10) : 0;
          const failed = failMatch ? parseInt(failMatch[1], 10) : 0;
          if (passed > 0) {
            tests.push({ name: `${passed} tests`, status: "passed" });
          }
          if (failed > 0) {
            tests.push({ name: `${failed} tests`, status: "failed" });
          }
        }
      }
    }
    return tests;
  }
  extractErrors(events) {
    const errorMap = /* @__PURE__ */ new Map();
    for (const event of events) {
      const payload = event.payload || {};
      if (payload.error || payload.success === false) {
        const errorType = payload.error_type || "UnknownError";
        const message = payload.error?.message || payload.error || "Unknown error";
        const key = `${errorType}:${message.substring(0, 50)}`;
        const existing = errorMap.get(key);
        if (existing) {
          existing.count++;
        } else {
          errorMap.set(key, {
            type: errorType,
            message: String(message).substring(0, 200),
            resolved: false,
            count: 1
          });
        }
      }
    }
    return Array.from(errorMap.values());
  }
  determineExitStatus(frame, errors) {
    const outputs = frame.outputs || {};
    if (outputs.cancelled || outputs.status === "cancelled") return "cancelled";
    if (errors.length === 0) return "success";
    if (errors.some((e) => !e.resolved)) return "failure";
    return "partial";
  }
  /**
   * Generate text summary from deterministic data
   */
  generateDeterministicText(frame, det) {
    const parts = [];
    parts.push(`## ${frame.name} (${frame.type})`);
    parts.push(`Status: ${det.exitStatus}`);
    if (det.durationSeconds > 0) {
      const mins = Math.floor(det.durationSeconds / 60);
      const secs = det.durationSeconds % 60;
      parts.push(`Duration: ${mins}m ${secs}s`);
    }
    if (det.filesModified.length > 0) {
      parts.push(`
### Files Modified (${det.filesModified.length})`);
      for (const f of det.filesModified.slice(0, 10)) {
        parts.push(`- ${f.operation}: ${f.path}`);
      }
      if (det.filesModified.length > 10) {
        parts.push(`  ...and ${det.filesModified.length - 10} more`);
      }
    }
    if (det.toolCallCount > 0) {
      parts.push(`
### Tool Calls (${det.toolCallCount})`);
      const sorted = Object.entries(det.toolCallsByType).sort((a, b) => b[1] - a[1]).slice(0, 5);
      for (const [tool, count] of sorted) {
        parts.push(`- ${tool}: ${count}`);
      }
    }
    if (det.decisions.length > 0) {
      parts.push(`
### Decisions (${det.decisions.length})`);
      for (const d of det.decisions.slice(0, 5)) {
        parts.push(`- ${d}`);
      }
    }
    if (det.constraints.length > 0) {
      parts.push(`
### Constraints (${det.constraints.length})`);
      for (const c of det.constraints.slice(0, 3)) {
        parts.push(`- ${c}`);
      }
    }
    if (det.errorsEncountered.length > 0) {
      parts.push(`
### Errors (${det.errorsEncountered.length})`);
      for (const e of det.errorsEncountered.slice(0, 3)) {
        parts.push(`- ${e.type}: ${e.message.substring(0, 80)}`);
      }
    }
    if (det.testsRun.length > 0) {
      const passed = det.testsRun.filter((t) => t.status === "passed").length;
      const failed = det.testsRun.filter((t) => t.status === "failed").length;
      parts.push(`
### Tests: ${passed} passed, ${failed} failed`);
    }
    return parts.join("\n");
  }
  /**
   * Queue frame for AI generation
   */
  queueForAIGeneration(request) {
    try {
      this.db.prepare(
        `
          INSERT OR REPLACE INTO digest_queue 
          (frame_id, frame_name, frame_type, priority, status, retry_count, created_at)
          VALUES (?, ?, ?, ?, 'pending', ?, ?)
        `
      ).run(
        request.frameId,
        request.frameName,
        request.frameType,
        request.priority,
        request.retryCount,
        Math.floor(request.createdAt / 1e3)
      );
      this.stats.pending++;
      this.scheduleIdleProcessing();
      logger.debug("Queued frame for AI digest generation", {
        frameId: request.frameId,
        priority: request.priority
      });
    } catch (error) {
      logger.error("Failed to queue digest generation", error);
    }
  }
  /**
   * Determine priority based on frame characteristics
   */
  determinePriority(input) {
    const { frame, anchors, events } = input;
    const decisionCount = anchors.filter((a) => a.type === "DECISION").length;
    const errorCount = events.filter(
      (e) => e.payload?.error || e.payload?.success === false
    ).length;
    if (decisionCount >= 3 || errorCount >= 2) return "high";
    if (decisionCount >= 1 || events.length >= 20) return "normal";
    return "low";
  }
  /**
   * Schedule idle-time processing
   */
  scheduleIdleProcessing() {
    if (this.idleTimer) {
      clearTimeout(this.idleTimer);
    }
    this.idleTimer = setTimeout(() => {
      this.processQueue();
    }, this.config.idleThresholdMs);
  }
  /**
   * Process queued AI generation requests
   */
  async processQueue() {
    if (this.processing || !this.llmProvider) return;
    this.processing = true;
    try {
      const pending = this.db.prepare(
        `
          SELECT * FROM digest_queue 
          WHERE status = 'pending' 
          ORDER BY 
            CASE priority 
              WHEN 'high' THEN 1 
              WHEN 'normal' THEN 2 
              WHEN 'low' THEN 3 
            END,
            created_at ASC
          LIMIT ?
        `
      ).all(this.config.batchSize);
      for (const item of pending) {
        await this.processQueueItem(item);
      }
    } finally {
      this.processing = false;
    }
  }
  async processQueueItem(item) {
    const startTime = Date.now();
    try {
      this.db.prepare(
        `UPDATE digest_queue SET status = 'processing', updated_at = unixepoch() WHERE frame_id = ?`
      ).run(item.frame_id);
      this.stats.processing++;
      this.stats.pending--;
      const frame = this.db.prepare(`SELECT * FROM frames WHERE frame_id = ?`).get(item.frame_id);
      if (!frame) {
        throw new Error(`Frame not found: ${item.frame_id}`);
      }
      const anchors = this.db.prepare(`SELECT * FROM anchors WHERE frame_id = ?`).all(item.frame_id);
      const events = this.db.prepare(`SELECT * FROM events WHERE frame_id = ? ORDER BY ts ASC`).all(item.frame_id);
      const parsedFrame = {
        ...frame,
        inputs: JSON.parse(frame.inputs || "{}"),
        outputs: JSON.parse(frame.outputs || "{}"),
        digest_json: JSON.parse(frame.digest_json || "{}")
      };
      const input = {
        frame: parsedFrame,
        anchors: anchors.map((a) => ({
          ...a,
          metadata: JSON.parse(a.metadata || "{}")
        })),
        events: events.map((e) => ({
          ...e,
          payload: JSON.parse(e.payload || "{}")
        }))
      };
      const deterministic = this.extractDeterministicFields(input);
      const aiGenerated = await this.llmProvider.generateSummary(
        input,
        deterministic,
        this.config.maxTokens
      );
      const existingDigest = parsedFrame.digest_json || {};
      const updatedDigest = {
        ...existingDigest,
        aiGenerated,
        status: "complete",
        updatedAt: Date.now()
      };
      const enhancedText = this.generateEnhancedText(
        parsedFrame,
        deterministic,
        aiGenerated
      );
      this.db.prepare(
        `
          UPDATE frames 
          SET digest_json = ?, digest_text = ?
          WHERE frame_id = ?
        `
      ).run(JSON.stringify(updatedDigest), enhancedText, item.frame_id);
      this.db.prepare(
        `UPDATE digest_queue SET status = 'completed', updated_at = unixepoch() WHERE frame_id = ?`
      ).run(item.frame_id);
      this.stats.processing--;
      this.stats.completed++;
      const processingTime = Date.now() - startTime;
      this.stats.avgProcessingTimeMs = (this.stats.avgProcessingTimeMs * (this.stats.completed - 1) + processingTime) / this.stats.completed;
      logger.info("Generated AI digest", {
        frameId: item.frame_id,
        processingTimeMs: processingTime
      });
    } catch (error) {
      const newRetryCount = item.retry_count + 1;
      if (newRetryCount < this.config.maxRetries) {
        this.db.prepare(
          `
            UPDATE digest_queue 
            SET status = 'pending', retry_count = ?, error_message = ?, updated_at = unixepoch()
            WHERE frame_id = ?
          `
        ).run(newRetryCount, error.message, item.frame_id);
        this.stats.processing--;
        this.stats.pending++;
        logger.warn("AI digest generation failed, will retry", {
          frameId: item.frame_id,
          retryCount: newRetryCount,
          error: error.message
        });
      } else {
        this.db.prepare(
          `
            UPDATE digest_queue 
            SET status = 'failed', error_message = ?, updated_at = unixepoch()
            WHERE frame_id = ?
          `
        ).run(error.message, item.frame_id);
        this.stats.processing--;
        this.stats.failed++;
        logger.error("AI digest generation failed permanently", error, {
          frameId: item.frame_id
        });
      }
    }
  }
  /**
   * Generate enhanced text with AI review (20%)
   */
  generateEnhancedText(frame, det, ai) {
    const parts = [];
    parts.push(this.generateDeterministicText(frame, det));
    parts.push(`
---`);
    parts.push(`**AI Review**: ${ai.summary}`);
    if (ai.insight) {
      parts.push(`**Insight**: ${ai.insight}`);
    }
    if (ai.flaggedIssue) {
      parts.push(`**Flag**: ${ai.flaggedIssue}`);
    }
    return parts.join("\n");
  }
  /**
   * Get queue statistics
   */
  getStats() {
    return { ...this.stats };
  }
  /**
   * Set LLM provider
   */
  setLLMProvider(provider) {
    this.llmProvider = provider;
  }
  /**
   * Force process queue (for testing or manual trigger)
   */
  async forceProcessQueue() {
    if (this.idleTimer) {
      clearTimeout(this.idleTimer);
    }
    await this.processQueue();
  }
  /**
   * Get digest for a frame
   */
  getDigest(frameId) {
    const frame = this.db.prepare(`SELECT * FROM frames WHERE frame_id = ?`).get(frameId);
    if (!frame) return null;
    const digestJson = JSON.parse(frame.digest_json || "{}");
    const anchors = this.db.prepare(`SELECT * FROM anchors WHERE frame_id = ?`).all(frameId);
    const events = this.db.prepare(`SELECT * FROM events WHERE frame_id = ?`).all(frameId);
    const parsedFrame = {
      ...frame,
      inputs: JSON.parse(frame.inputs || "{}"),
      outputs: JSON.parse(frame.outputs || "{}"),
      digest_json: digestJson
    };
    const input = {
      frame: parsedFrame,
      anchors: anchors.map((a) => ({
        ...a,
        metadata: JSON.parse(a.metadata || "{}")
      })),
      events: events.map((e) => ({
        ...e,
        payload: JSON.parse(e.payload || "{}")
      }))
    };
    const deterministic = this.extractDeterministicFields(input);
    const queueItem = this.db.prepare(`SELECT status FROM digest_queue WHERE frame_id = ?`).get(frameId);
    let status = "deterministic_only";
    if (digestJson.aiGenerated) {
      status = "complete";
    } else if (queueItem) {
      status = queueItem.status === "processing" ? "ai_processing" : queueItem.status === "failed" ? "ai_failed" : "ai_pending";
    }
    return {
      frameId: frame.frame_id,
      frameName: frame.name,
      frameType: frame.type,
      deterministic,
      aiGenerated: digestJson.aiGenerated,
      status,
      text: frame.digest_text || this.generateDeterministicText(parsedFrame, deterministic),
      version: 1,
      createdAt: frame.created_at * 1e3,
      updatedAt: digestJson.updatedAt || frame.created_at * 1e3
    };
  }
}
export {
  HybridDigestGenerator
};
//# sourceMappingURL=hybrid-digest-generator.js.map