@dooor-ai/toolkit/src/observability/logger.ts

Guards, Evals & Observability for AI applications - works seamlessly with LangChain/LangGraph

import { TraceData } from "../core/types";
import { getCortexDBClient } from "./cortexdb-client";

/**
 * Interface for observability backends
 */
export interface ObservabilityBackend {
  logTrace(trace: TraceData): void | Promise<void>;
  updateTrace(traceId: string, updates: Partial<TraceData>): void | Promise<void>;
  logError(error: Error, context?: Record<string, any>): void | Promise<void>;
  logMetric(name: string, value: number, tags?: Record<string, string>): void | Promise<void>;
}

/**
 * Console-based observability backend for development
 */
export class ConsoleBackend implements ObservabilityBackend {
  private verbose: boolean;

  constructor(verbose: boolean = true) {
    this.verbose = verbose;
  }

  logTrace(trace: TraceData): void {
    if (!this.verbose) return;

    console.log("\n" + "=".repeat(80));
    console.log(`🔍 DOOOR Trace [${trace.traceId}]`);
    console.log("=".repeat(80));
    
    console.log(`\n📝 Input:\n${trace.input.substring(0, 200)}${trace.input.length > 200 ? "..." : ""}`);
    
    if (trace.output) {
      console.log(`\n✅ Output:\n${trace.output.substring(0, 200)}${trace.output.length > 200 ? "..." : ""}`);
    }

    console.log(`\n⚙️  Model: ${trace.model}`);
    console.log(`⏱️  Latency: ${trace.latency}ms`);
    
    if (trace.tokens) {
      console.log(`🔢 Tokens: ${trace.tokens.total} (prompt: ${trace.tokens.prompt}, completion: ${trace.tokens.completion})`);
    }
    
    if (trace.cost) {
      console.log(`💰 Cost: $${trace.cost.toFixed(4)}`);
    }

    if (trace.guards && trace.guards.length > 0) {
      console.log("\n🛡️  Guards:");
      trace.guards.forEach((guard) => {
        const icon = guard.result.passed ? "✅" : "🚫";
        console.log(`  ${icon} ${guard.name} (${guard.latency}ms)`);
        if (!guard.result.passed) {
          console.log(`     Reason: ${guard.result.reason}`);
          console.log(`     Severity: ${guard.result.severity}`);
        }
      });
    }

    if (trace.evals && trace.evals.length > 0) {
      console.log("\n📊 Evals:");
      trace.evals.forEach((evalResult) => {
        const icon = evalResult.result.passed ? "✅" : "⚠️";
        console.log(`  ${icon} ${evalResult.name}: ${evalResult.result.score.toFixed(2)} (${evalResult.latency}ms)`);
        if (evalResult.result.details) {
          console.log(`     ${evalResult.result.details}`);
        }
      });
    }

    console.log("\n" + "=".repeat(80) + "\n");
  }

  logError(error: Error, context?: Record<string, any>): void {
    console.error("\n🚨 DOOOR Error:");
    console.error(`  ${error.name}: ${error.message}`);
    if (context) {
      console.error("  Context:", JSON.stringify(context, null, 2));
    }
    console.error(error.stack);
  }

  async updateTrace(traceId: string, updates: Partial<TraceData>): Promise<void> {
    // ConsoleBackend doesn't need to update traces (already logged)
    if (this.verbose && updates.evals) {
      console.log(`\n📊 Updated Evals for Trace [${traceId}]:`);
      updates.evals.forEach((evalResult) => {
        const icon = evalResult.result.passed ? "✅" : "⚠️";
        console.log(`  ${icon} ${evalResult.name}: ${evalResult.result.score.toFixed(2)} (${evalResult.latency}ms)`);
      });
    }
  }

  logMetric(name: string, value: number, tags?: Record<string, string>): void {
    if (!this.verbose) return;

    const tagsStr = tags ? ` [${Object.entries(tags).map(([k, v]) => `${k}:${v}`).join(", ")}]` : "";
    console.log(`📈 ${name}: ${value}${tagsStr}`);
  }
}

/**
 * CortexDB-based observability backend for production
 * Saves traces, evals, and guard blocks to CortexDB
 */
export class CortexDBBackend implements ObservabilityBackend {
  private project?: string;

  constructor(project?: string) {
    this.project = project;
  }

  async logTrace(trace: TraceData): Promise<void> {
    try {
      console.log("[CortexDBBackend] logTrace called with:", {
        traceId: trace.traceId,
        project: this.project,
        model: trace.model,
        hasInput: !!trace.input,
        hasOutput: !!trace.output,
      });

      const client = getCortexDBClient();
      console.log("[CortexDBBackend] Got CortexDB client, config:", {
        baseUrl: client.getConfig().baseUrl,
        database: client.getConfig().database,
        hasApiKey: !!client.getConfig().apiKey,
      });

      // Save trace
    const promptTokens = trace.tokens?.prompt || 0;
    const completionTokens = trace.tokens?.completion || 0;
    const totalTokens = trace.tokens?.total || promptTokens + completionTokens;

    const metadataPayload: Record<string, any> = {
      ...(trace.metadata || {}),
      timestamp: trace.timestamp?.toISOString() || new Date().toISOString(),
    };

    if (trace.guards && trace.guards.length > 0) {
      metadataPayload.guards = trace.guards.map((guard) => ({
        name: guard.name,
        passed: guard.result.passed,
        reason: guard.result.reason,
        severity: guard.result.severity,
        latency_ms: guard.latency,
        metadata: guard.result.metadata || {},
      }));
    }

    if (trace.toolCalls && trace.toolCalls.length > 0) {
      metadataPayload.toolCalls = trace.toolCalls;
    }

    const traceData: any = {
      trace_id: trace.traceId,
      session_id: trace.sessionId || null,
      project: this.project || "default",
      model: trace.model,
      input: trace.input,
      output: trace.output,
      latency_ms: trace.latency,
      input_tokens: promptTokens,
      output_tokens: completionTokens,
      total_tokens: totalTokens,
      tokens_input: promptTokens,
      tokens_output: completionTokens,
      tokens_total: totalTokens,
      timestamp: Date.now(),
      metadata: {
        ...metadataPayload,
        trace_sequence: trace.traceSequence,
        trace_type: trace.traceType,
      },
    };

    // Only send cost_usd if explicitly provided (non-zero)
    // Otherwise, let the gateway calculate it based on tokens and pricing table
    console.log("[CortexDBBackend] Cost decision:", {
      traceCost: trace.cost,
      hasCost: trace.cost !== undefined,
      isPositive: trace.cost && trace.cost > 0,
      willSendCostUsd: !!(trace.cost && trace.cost > 0),
    });

    if (trace.cost && trace.cost > 0) {
      traceData.cost_usd = trace.cost;
      console.log("[CortexDBBackend] Including cost_usd in payload:", trace.cost);
    } else {
      console.log("[CortexDBBackend] NOT sending cost_usd - letting gateway calculate from tokens");
    }

      console.log("[CortexDBBackend] Final trace payload:", traceData);
      await client.saveTrace(traceData);
      console.log("[CortexDBBackend] ✅ Trace saved successfully!");

      // Save guards results as guard_blocks
      if (trace.guards) {
        const config = client.getConfig();
        for (const guard of trace.guards) {
          if (!guard.result.passed) {
            const guardPayload = {
              trace_id: trace.traceId,
              project: this.project || "default",
              session_id: (trace as any).sessionId,
              guard_name: guard.name,
              guard_type: guard.result.metadata?.type,
              timestamp: Date.now(),
              input: trace.input,
              reason: guard.result.reason || "Unknown",
              score: guard.result.confidence,
              threshold: (guard.result as any)?.threshold,
              metadata: {
                ...(guard.result.metadata || {}),
                severity: guard.result.severity || "medium",
                blocked: true,
                trace_id: trace.traceId,
              },
            };

            await fetch(
              `${config.baseUrl}/databases/${config.database}/guard_blocks`,
              {
                method: "POST",
                headers: {
                  "Content-Type": "application/json",
                  "Authorization": `Bearer ${config.apiKey}`,
                },
                body: JSON.stringify(guardPayload),
              }
            );
          }
        }
      }

    } catch (error) {
      console.error("Failed to save trace to CortexDB:", error);
      // Don't throw - observability should never break the main flow
    }
  }

  async logError(error: Error, context?: Record<string, any>): Promise<void> {
    // For now, just log to console
    // TODO: Add error tracking endpoint to CortexDB
    console.error("🚨 DOOOR Error:", error.message, context);
  }

  async updateTrace(traceId: string, updates: Partial<TraceData>): Promise<void> {
    try {
      console.log("[CortexDBBackend] updateTrace called with:", { traceId, hasEvals: !!updates.evals });

      // Save evals if provided
      if (updates.evals) {
        const client = getCortexDBClient();
        for (const evalResult of updates.evals) {
          await client.saveEval({
            trace_id: traceId,
            eval_name: evalResult.name,
            score: evalResult.result.score,
            passed: evalResult.result.passed,
            latency_ms: evalResult.latency,
            details: evalResult.result.details || null,
            metadata: evalResult.result.metadata || {},
          });
        }
        console.log("[CortexDBBackend] ✅ Evals saved successfully for trace:", traceId);
      }

      if (updates.toolCalls) {
        const client = getCortexDBClient();
        await client.updateToolCalls(traceId, updates.toolCalls);
        console.log("[CortexDBBackend] ✅ Tool calls updated for trace:", traceId);
      }
    } catch (error) {
      console.error("Failed to update trace in CortexDB:", error);
    }
  }

  async logMetric(name: string, value: number, tags?: Record<string, string>): Promise<void> {
    // For now, just log to console
    // TODO: Add metrics endpoint to CortexDB
    console.log(`📈 ${name}: ${value}`, tags);
  }
}

/**
 * Observability collector that manages trace data
 */
export class ObservabilityCollector {
  private backend: ObservabilityBackend;
  private enabled: boolean;

  constructor(backend?: ObservabilityBackend, enabled: boolean = true) {
    this.backend = backend ?? new ConsoleBackend();
    this.enabled = enabled;
  }

  async logTrace(trace: TraceData): Promise<void> {
    if (!this.enabled) return;
    await this.backend.logTrace(trace);
  }

  async updateTrace(traceId: string, updates: Partial<TraceData>): Promise<void> {
    if (!this.enabled) return;
    await this.backend.updateTrace(traceId, updates);
  }

  logError(error: Error, context?: Record<string, any>): void {
    if (!this.enabled) return;
    this.backend.logError(error, context);
  }

  logMetric(name: string, value: number, tags?: Record<string, string>): void {
    if (!this.enabled) return;
    this.backend.logMetric(name, value, tags);
  }

  setEnabled(enabled: boolean): void {
    this.enabled = enabled;
  }

  isEnabled(): boolean {
    return this.enabled;
  }
}

export interface LogTraceOptions {
  /**
   * Reuse an existing collector. If provided, backend/enabled/project options are ignored.
   */
  collector?: ObservabilityCollector;
  /**
   * Custom backend implementation (Console, CortexDB, etc).
   */
  backend?: ObservabilityBackend;
  /**
   * Convenience project name when instantiating a CortexDB backend automatically.
   */
  project?: string;
  /**
   * Allows disabling logging without removing the helper.
   */
  enabled?: boolean;
}

/**
 * Convenience helper for manual integrations (e.g., HTTP fetch calls).
 *
 * Example:
 * ```ts
 * configureCortexDBFromConnectionString(process.env.CORTEXDB!);
 * await logTrace(
 *   {
 *     traceId: crypto.randomUUID(),
 *     input: prompt,
 *     output: response,
 *     model: "claude-3-sonnet",
 *     latency,
 *     tokens: { prompt: 100, completion: 200, total: 300 },
 *     timestamp: new Date(),
 *   },
 *   { project: "vaultly-api" }
 * );
 * ```
 */
export async function logTrace(
  trace: TraceData,
  options?: LogTraceOptions
): Promise<void> {
  // Generate traceId and sessionId if not provided
  const uuid = await import("uuid");
  const traceWithId: TraceData = {
    ...trace,
    traceId: trace.traceId || uuid.v4(),
    sessionId: trace.sessionId || uuid.v4(),
  };

  const collector =
    options?.collector ||
    new ObservabilityCollector(
      resolveBackend(options),
      options?.enabled ?? true
    );

  try {
    await collector.logTrace(traceWithId);
  } catch (error) {
    console.warn(
      "[logTrace] Failed to send trace to configured backend. Falling back to console.",
      error
    );
    if (!options?.collector) {
      const fallback = new ObservabilityCollector(new ConsoleBackend(true));
      await fallback.logTrace(traceWithId);
    }
  }
}

function resolveBackend(options?: LogTraceOptions): ObservabilityBackend {
  if (options?.backend) {
    return options.backend;
  }

  // Prefer CortexDB backend if a project is provided
  if (options?.project) {
    return new CortexDBBackend(options.project);
  }

  return new ConsoleBackend(true);
}