UNPKG

@traceprompt/node

Version:

Client-side encrypted, audit-ready logging for LLM applications

894 lines (884 loc) 27.9 kB
'use strict'; var perf_hooks = require('perf_hooks'); var fs = require('fs'); var path2 = require('path'); var yaml = require('yaml'); var clientNode = require('@aws-crypto/client-node'); var promClient = require('prom-client'); var winston = require('winston'); var blakeHash = require('@napi-rs/blake-hash'); var fs2 = require('fs/promises'); var crypto = require('crypto'); var readline = require('readline'); var undici = require('undici'); function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; } function _interopNamespace(e) { if (e && e.__esModule) return e; var n = Object.create(null); if (e) { Object.keys(e).forEach(function (k) { if (k !== 'default') { var d = Object.getOwnPropertyDescriptor(e, k); Object.defineProperty(n, k, d.get ? d : { enumerable: true, get: function () { return e[k]; } }); } }); } n.default = e; return Object.freeze(n); } var fs__namespace = /*#__PURE__*/_interopNamespace(fs); var path2__namespace = /*#__PURE__*/_interopNamespace(path2); var yaml__namespace = /*#__PURE__*/_interopNamespace(yaml); var winston__default = /*#__PURE__*/_interopDefault(winston); var fs2__default = /*#__PURE__*/_interopDefault(fs2); var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, { get: (a, b) => (typeof require !== "undefined" ? require : a)[b] }) : x)(function(x) { if (typeof require !== "undefined") return require.apply(this, arguments); throw Error('Dynamic require of "' + x + '" is not supported'); }); process.env.AWS_PROFILE = "traceprompt-ingest-role"; async function resolveOrgFromApiKey(apiKey, ingestUrl) { try { const whoamiUrl = `${ingestUrl.replace("/v1/ingest", "")}/v1/whoami`; const response = await fetch(whoamiUrl, { method: "GET", headers: { "x-api-key": apiKey, "Content-Type": "application/json" } }); if (!response.ok) { throw new Error( `Failed to resolve organization: ${response.status} ${response.statusText}` ); } const result = await response.json(); if (!result.success) { throw new Error("Failed to resolve organization from API key"); } const orgId = result.data.orgId; if (!orgId) { throw new Error("No organization ID found in API key response"); } const cmkArn = result.data.kmsKeyArn; console.log(`\u2713 Traceprompt auto-resolved organization: ${orgId}`); return { orgId, cmkArn }; } catch (error) { throw new Error( `Failed to auto-resolve organization from API key: ${error instanceof Error ? error.message : String(error)}` ); } } function readYaml(filePath) { try { const abs = path2__namespace.resolve(process.cwd(), filePath); if (!fs__namespace.existsSync(abs)) return {}; const raw = fs__namespace.readFileSync(abs, "utf8"); return yaml__namespace.parse(raw) ?? {}; } catch { return {}; } } var ConfigManagerClass = class { async load(userCfg = {}) { if (this._cfg) return; if (this._loadPromise) { await this._loadPromise; return; } this._loadPromise = this._doLoad(userCfg); await this._loadPromise; } async _doLoad(userCfg = {}) { const fileCfg = process.env["TRACEPROMPT_RC"] ? readYaml(process.env["TRACEPROMPT_RC"]) : {}; const envCfg = { ...process.env["TRACEPROMPT_API_KEY"] && { apiKey: process.env["TRACEPROMPT_API_KEY"] }, ...process.env["TRACEPROMPT_BATCH_SIZE"] && { batchSize: Number(process.env["TRACEPROMPT_BATCH_SIZE"]) }, ...process.env["TRACEPROMPT_FLUSH_INTERVAL_MS"] && { flushIntervalMs: Number(process.env["TRACEPROMPT_FLUSH_INTERVAL_MS"]) }, ...process.env["TRACEPROMPT_LOG_LEVEL"] && { logLevel: process.env["TRACEPROMPT_LOG_LEVEL"] } }; const merged = { apiKey: "", cmkArn: "", ingestUrl: "http://localhost:8080/v1/ingest", // Default for local development batchSize: 25, flushIntervalMs: 2e3, staticMeta: {}, logLevel: "verbose", ...fileCfg, ...envCfg, ...userCfg }; if (!merged.apiKey) throw new Error("Traceprompt: apiKey is required"); let orgId; let cmkArn; try { const resolved = await resolveOrgFromApiKey( merged.apiKey, merged.ingestUrl ); orgId = resolved.orgId; cmkArn = resolved.cmkArn; } catch (error) { throw new Error( `Failed to auto-resolve organization: ${error instanceof Error ? error.message : String(error)}` ); } if (merged.batchSize <= 0) merged.batchSize = 25; if (merged.flushIntervalMs <= 0) merged.flushIntervalMs = 2e3; this._cfg = { ...merged, orgId, cmkArn, apiKey: merged.apiKey, ingestUrl: merged.ingestUrl }; } get cfg() { if (!this._cfg) { throw new Error("Traceprompt: initTracePrompt() must be called first"); } return this._cfg; } }; async function initTracePrompt(cfg) { await ConfigManager.load(cfg); } var ConfigManager = new ConfigManagerClass(); function buildKeyring() { const { cmkArn } = ConfigManager.cfg; return new clientNode.KmsKeyringNode({ generatorKeyId: cmkArn }); } var registry = new promClient.Registry(); var encryptHist = new promClient.Histogram({ name: "traceprompt_encrypt_ms", help: "Latency of client-side AES-GCM envelope encryption (ms)", buckets: [0.05, 0.1, 0.25, 0.5, 1, 2, 5], registers: [registry] }); new promClient.Histogram({ name: "traceprompt_token_count", help: "Tokens counted per prompt/response", buckets: [1, 5, 10, 20, 50, 100, 200, 500, 1e3], registers: [registry] }); var flushFailures = new promClient.Counter({ name: "traceprompt_flush_failures_total", help: "Number of failed POSTs to the Traceprompt ingest API", registers: [registry] }); var queueGauge = new promClient.Gauge({ name: "traceprompt_queue_depth", help: "Number of events currently buffered in memory", registers: [registry] }); var logger = null; function createLogger() { const cfg = ConfigManager.cfg; const logLevel = cfg.logLevel || "verbose"; return winston__default.default.createLogger({ level: logLevel, format: winston__default.default.format.combine( winston__default.default.format.timestamp({ format: "YYYY-MM-DD HH:mm:ss" }), winston__default.default.format.errors({ stack: true }), winston__default.default.format.printf(({ level, message, timestamp, stack }) => { const prefix = `[${timestamp}] [Traceprompt] [${level.toUpperCase()}]`; if (stack) { return `${prefix} ${message} ${stack}`; } return `${prefix} ${message}`; }) ), transports: [ new winston__default.default.transports.Console({ handleExceptions: true, handleRejections: true }) ], exitOnError: false }); } function getLogger() { if (!logger) { logger = createLogger(); } return logger; } var log = { error: (message, meta) => getLogger().error(message, meta), warn: (message, meta) => getLogger().warn(message, meta), info: (message, meta) => getLogger().info(message, meta), verbose: (message, meta) => getLogger().verbose(message, meta), debug: (message, meta) => getLogger().debug(message, meta), silly: (message, meta) => getLogger().silly(message, meta) }; // src/crypto/encryptor.ts var { encrypt, decrypt } = clientNode.buildClient( clientNode.CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT ); async function encryptBuffer(plain) { const keyring = buildKeyring(); const endTimer = encryptHist.startTimer(); try { log.info("Encrypting buffer", { orgId: ConfigManager.cfg.orgId }); const { result, messageHeader } = await encrypt(keyring, plain, { encryptionContext: { org_id: ConfigManager.cfg.orgId } }); const bundle = { ciphertext: Buffer.from(result).toString("base64"), encryptedDataKey: Buffer.from( messageHeader.encryptedDataKeys[0].encryptedDataKey ).toString("base64"), suiteId: messageHeader.suiteId }; return bundle; } finally { endTimer(); } } async function decryptBundle(bundle) { const keyring = buildKeyring(); const { plaintext } = await decrypt( keyring, Buffer.from(bundle.ciphertext, "base64") ); return plaintext; } function computeLeaf(data) { if (data === void 0) { data = "null"; } return blakeHash.blake3(data).toString("hex"); } var encodeFn = null; var tokenCountHist2 = new promClient.Histogram({ name: "traceprompt_tokens_per_string", help: "Number of tokens counted per string passed to countTokens()", buckets: [1, 5, 10, 20, 50, 100, 200, 500, 1e3], registers: [registry] }); function countTokens(text) { if (encodeFn) { const t = encodeFn(text); tokenCountHist2.observe(t); return t; } if (maybeInitTiktoken()) { const t = encodeFn(text); tokenCountHist2.observe(t); return t; } const words = text.trim().split(/\s+/g).length; const tokens = Math.ceil(words * 1.33); tokenCountHist2.observe(tokens); return tokens; } var triedTiktoken = false; function maybeInitTiktoken() { if (encodeFn || triedTiktoken) return !!encodeFn; triedTiktoken = true; try { const { encoding_for_model } = __require("@dqbd/tiktoken"); const enc = encoding_for_model("cl100k_base"); encodeFn = (s) => enc.encode(s).length; return true; } catch { return false; } } // src/utils/retry.ts async function retry(fn, attempts = 5, baseDelay = 250, onError) { let attempt = 0; while (true) { try { attempt++; return await fn(); } catch (err) { onError?.(err, attempt); if (attempt >= attempts) throw err; const exp = baseDelay * 2 ** (attempt - 1); const jitter = Math.random() * exp; await new Promise((res) => setTimeout(res, jitter)); } } } // src/network/transport.ts var Transport = { async post(path3, body, retries = 5, headers) { await sendJson({ path: path3, body, retries, method: "POST", headers }); } }; async function sendJson(opts) { const { ingestUrl, apiKey } = ConfigManager.cfg; const url = new URL(opts.path, ingestUrl).toString(); const extra = opts.headers ?? {}; log.verbose(`Sending request to ${opts.path}`, { url, method: opts.method ?? "POST", retries: opts.retries ?? 5, hasBody: !!opts.body }); await retry( async () => { const res = await undici.fetch(url, { method: opts.method ?? "POST", headers: { "content-type": "application/json", "user-agent": "traceprompt-sdk/0.1.0", "x-api-key": apiKey, ...extra }, body: JSON.stringify(opts.body) }); if (res.status >= 400) { const msg = await res.text(); const errorMessage = `HTTP ${res.status} - ${msg}`; if (res.status >= 500) { log.warn(`Server error (will retry): ${errorMessage}`, { status: res.status, url, response: msg }); } else if (res.status === 429) { log.warn(`Rate limited (will retry): ${errorMessage}`, { status: res.status, url, response: msg }); } else if (res.status === 401 || res.status === 403) { log.error(`Authentication/authorization error: ${errorMessage}`, { status: res.status, url, response: msg, hint: "Check your API key and organization permissions" }); } else { log.error(`Client error: ${errorMessage}`, { status: res.status, url, response: msg }); } throw new Error(`Traceprompt: ${errorMessage}`); } log.debug(`Request successful`, { status: res.status, url }); }, opts.retries ?? 5, 250, (error, attempt) => { log.verbose(`Request attempt ${attempt} failed, retrying...`, { error: error instanceof Error ? error.message : String(error), attempt, maxRetries: opts.retries ?? 5, url }); } ); log.verbose(`Request completed successfully`, { url }); } // src/queue/persistentBatcher.ts function getConfig() { return ConfigManager.cfg; } function getDir() { const cfg = getConfig(); return path2__namespace.default.resolve(cfg.dataDir ?? ".traceprompt", "queue"); } function getLogPath() { return path2__namespace.default.join(getDir(), "outbox.log"); } function getMaxRamRecords() { const cfg = getConfig(); return (cfg.batchSize || 10) * 2; } var MAX_FILE_BYTES = 5 * 1024 * 1024; var bootstrapDone = false; var pLimitPromise = null; var closing = false; async function getPLimit() { if (!pLimitPromise) { pLimitPromise = import('p-limit').then((module) => module.default); } return pLimitPromise; } async function bootstrap() { if (bootstrapDone) return; await fs2__default.default.mkdir(getDir(), { recursive: true }); bootstrapDone = true; } var ring = []; var head = 0; var len = 0; var ringInitialized = false; function initializeRing() { if (ringInitialized) return; const maxRecords = getMaxRamRecords(); ring = new Array(maxRecords); ringInitialized = true; } function ringPush(item) { initializeRing(); const maxRecords = getMaxRamRecords(); ring[(head + len) % maxRecords] = item; if (len < maxRecords) { len++; return; } head = (head + 1) % maxRecords; } function ringDrip(n) { initializeRing(); const maxRecords = getMaxRamRecords(); const out = []; while (out.length < n && len > 0) { out.push(ring[head]); head = (head + 1) % maxRecords; len--; } return out; } async function append(item) { if (closing) { throw new Error("Traceprompt SDK is shutting down, rejecting new events"); } await bootstrap(); initializeTimer(); const rec = JSON.stringify({ id: crypto.randomUUID(), ...item }) + "\n"; try { await fs2__default.default.appendFile(getLogPath(), rec, "utf8"); log.debug("Record appended to outbox", { outboxPath: getLogPath(), recordSize: rec.length }); } catch (error) { log.error("Failed to append record to outbox", { error: error instanceof Error ? error.message : String(error), outboxPath: getLogPath() }); throw error; } ringPush(item); queueGauge.set(len); log.verbose("Record added to ring buffer", { ringSize: len, maxRingSize: getMaxRamRecords() }); try { const { size } = await fs2__default.default.stat(getLogPath()); if (size > MAX_FILE_BYTES) { log.error("Outbox file size exceeded limit - applying backpressure", { currentSize: size, maxSize: MAX_FILE_BYTES, outboxPath: getLogPath() }); throw new Error( "Traceprompt SDK backpressure: local outbox full, ingest unreachable." ); } if (size > MAX_FILE_BYTES * 0.8) { log.warn("Outbox file size approaching limit", { currentSize: size, maxSize: MAX_FILE_BYTES, percentFull: Math.round(size / MAX_FILE_BYTES * 100), outboxPath: getLogPath() }); } } catch (e) { if (e.code !== "ENOENT") { log.warn("Failed to check outbox file size", { error: e instanceof Error ? e.message : String(e), outboxPath: getLogPath() }); throw e; } } } var limit = null; async function flushOnce() { await bootstrap(); initializeTimer(); if (!limit) { const pLimit = await getPLimit(); limit = pLimit(1); } return limit(async () => { const cfg = getConfig(); const batchSize = cfg.batchSize || 10; let batch = []; const ringRecords = ringDrip(batchSize); if (ringRecords.length > 0) { log.verbose("Using ring buffer records for flush", { ringRecords: ringRecords.length, batchSize }); batch = ringRecords.map((record) => ({ id: crypto.randomUUID(), ...record })); } let diskLines = []; let totalDiskRecords = 0; if (batch.length < batchSize) { const needed = batchSize - batch.length; try { const rl = readline.createInterface({ input: fs.createReadStream(getLogPath()) }); const diskBatch = []; for await (const line of rl) { if (!line.trim()) continue; if (diskBatch.length < needed) { diskBatch.push(JSON.parse(line)); } diskLines.push(line); totalDiskRecords++; if (diskBatch.length >= needed && totalDiskRecords >= needed * 2) { break; } } rl.close(); if (diskBatch.length > 0) { log.verbose("Supplementing with disk records", { ringRecords: batch.length, diskRecords: diskBatch.length, totalDiskRecordsRead: totalDiskRecords }); batch.push(...diskBatch); } } catch (error) { if (error.code === "ENOENT") { if (batch.length === 0) { log.debug("No records in ring buffer or disk, nothing to flush"); return; } } else { log.warn("Error reading outbox file", { error: error.message, outboxPath: getLogPath() }); } } } else { try { const rl = readline.createInterface({ input: fs.createReadStream(getLogPath()) }); for await (const line of rl) { if (line.trim()) { diskLines.push(line); totalDiskRecords++; } } rl.close(); } catch (error) { if (error.code !== "ENOENT") { log.warn("Error counting disk records", { error: error.message, outboxPath: getLogPath() }); } } } if (batch.length === 0) { log.debug("No records available for flush"); return; } const totalPending = totalDiskRecords + (ringRecords.length > batch.length ? 0 : len); queueGauge.set(totalPending); log.info("Starting batch flush", { batchSize: batch.length, fromRingBuffer: Math.min(ringRecords.length, batch.length), fromDisk: Math.max(0, batch.length - ringRecords.length), totalPendingAfterFlush: totalPending - batch.length, outboxPath: getLogPath() }); const body = { orgId: cfg.orgId, records: batch.map(({ payload, leafHash }) => ({ payload, leafHash })) }; try { await Transport.post("/v1/ingest", body, { "Idempotency-Key": batch[0].leafHash }); if (totalDiskRecords > 0) { const diskRecordsUsed = Math.max(0, batch.length - ringRecords.length); if (diskRecordsUsed > 0) { let allDiskLines; if (diskLines.length === totalDiskRecords) { allDiskLines = diskLines; } else { try { const text = await fs2__default.default.readFile(getLogPath(), "utf8"); allDiskLines = text.trim().split("\n").filter(Boolean); } catch (error) { log.error("Failed to read outbox file for cleanup", { error: error instanceof Error ? error.message : String(error), outboxPath: getLogPath() }); return; } } const remaining = allDiskLines.slice(diskRecordsUsed); if (remaining.length > 0) { await fs2__default.default.writeFile(getLogPath(), remaining.join("\n") + "\n"); log.info("Batch flush successful, updated outbox", { flushedRecords: batch.length, fromRingBuffer: ringRecords.length, fromDisk: diskRecordsUsed, remainingOnDisk: remaining.length }); queueGauge.set(totalPending - batch.length); } else { await fs2__default.default.writeFile(getLogPath(), ""); log.info("Batch flush successful, outbox cleared", { flushedRecords: batch.length, fromRingBuffer: ringRecords.length, fromDisk: diskRecordsUsed }); queueGauge.set(totalPending - batch.length); } } else { log.info("Batch flush successful, used only ring buffer", { flushedRecords: batch.length, diskRecordsRemaining: totalDiskRecords }); queueGauge.set(totalPending - batch.length); } } else { log.info("Batch flush successful, used only ring buffer", { flushedRecords: batch.length }); queueGauge.set(totalPending - batch.length); } } catch (e) { const errorMessage = e instanceof Error ? e.message : String(e); if (ringRecords.length > 0) { log.warn("Flush failed, restoring ring buffer records to disk", { ringRecordsToRestore: ringRecords.length }); const ringRecordsAsLines = ringRecords.map( (record) => JSON.stringify({ id: crypto.randomUUID(), ...record }) ); try { let existingContent = ""; try { existingContent = await fs2__default.default.readFile(getLogPath(), "utf8"); } catch { } const allLines = [...ringRecordsAsLines]; if (existingContent.trim()) { allLines.push( ...existingContent.trim().split("\n").filter(Boolean) ); } await fs2__default.default.writeFile(getLogPath(), allLines.join("\n") + "\n"); } catch (restoreError) { log.error("Failed to restore ring buffer records to disk", { error: restoreError instanceof Error ? restoreError.message : String(restoreError), lostRecords: ringRecords.length }); } } if (errorMessage.includes("HTTP 5")) { log.warn("Server error during batch flush, will retry", { error: errorMessage, batchSize: batch.length, totalPending }); } else if (errorMessage.includes("HTTP 429")) { log.warn("Rate limited during batch flush, will retry", { error: errorMessage, batchSize: batch.length, totalPending }); } else if (errorMessage.includes("HTTP 4")) { log.error("Client error during batch flush", { error: errorMessage, batchSize: batch.length, totalPending, hint: "Check API configuration and request format" }); } else { log.error("Network error during batch flush", { error: errorMessage, batchSize: batch.length, totalPending }); } flushFailures.inc(); throw e; } }); } var timerInitialized = false; var flushTimer = null; function initializeTimer() { if (timerInitialized) return; timerInitialized = true; const cfg = getConfig(); log.info("Initializing periodic flush timer", { flushIntervalMs: cfg.flushIntervalMs }); flushTimer = setInterval( () => flushOnce().catch((error) => { log.verbose("Periodic flush failed, will retry on next interval", { error: error instanceof Error ? error.message : String(error), nextRetryIn: cfg.flushIntervalMs }); }), cfg.flushIntervalMs ); flushTimer.unref(); } async function flushWithRetry(opts) { for (let attempt = 1; attempt <= opts.maxRetries; attempt++) { try { await flushOnce(); return; } catch (error) { if (attempt === opts.maxRetries) throw error; const delayMs = Math.min(500 * Math.pow(2, attempt - 1), 4e3); log.debug("Flush attempt failed, retrying", { attempt, maxRetries: opts.maxRetries, delayMs, error: error instanceof Error ? error.message : String(error) }); await new Promise((resolve2) => setTimeout(resolve2, delayMs)); } } } async function drainOutboxWithRetry(opts) { const startTime = Date.now(); let attempt = 0; while (Date.now() - startTime < opts.maxTimeoutMs) { attempt++; try { const outboxContent = await fs2__default.default.readFile(getLogPath(), "utf8").catch(() => ""); if (!outboxContent.trim()) { log.info("Outbox is empty, drain complete"); return; } await flushWithRetry({ maxRetries: opts.maxRetries }); } catch (error) { log.warn("Outbox drain attempt failed", { attempt, error: error instanceof Error ? error.message : String(error) }); const delayMs = Math.min(500 * Math.pow(2, attempt - 1), 4e3); await new Promise((resolve2) => setTimeout(resolve2, delayMs)); } } throw new Error(`Outbox drain timed out after ${opts.maxTimeoutMs}ms`); } async function gracefulShutdown() { log.info("Starting graceful shutdown"); closing = true; if (flushTimer) { clearInterval(flushTimer); log.debug("Cleared periodic flush timer"); } log.info("Flushing in-memory ring buffer"); await flushWithRetry({ maxRetries: 3 }); log.info("Draining persistent outbox"); await drainOutboxWithRetry({ maxRetries: 5, maxTimeoutMs: 3e4 }); log.info("Graceful shutdown completed successfully"); } process.on("SIGTERM", async () => { try { await gracefulShutdown(); process.exit(0); } catch (error) { log.error("Graceful shutdown failed", { error: error instanceof Error ? error.message : String(error) }); flushFailures.inc(); process.exit(1); } }); process.on("SIGINT", async () => { try { await gracefulShutdown(); process.exit(0); } catch (error) { log.error("Graceful shutdown failed", { error: error instanceof Error ? error.message : String(error) }); flushFailures.inc(); process.exit(1); } }); var PersistentBatcher = { enqueue: append, flush: flushOnce, gracefulShutdown }; var stringify = __require("json-stable-stringify"); var wrapperLatencyHist = new promClient.Histogram({ name: "traceprompt_llm_wrapper_latency_ms", help: "End\u2011to\u2011end latency from prompt send to response receive in the SDK wrapper (ms)", buckets: [50, 100, 250, 500, 1e3, 2e3, 5e3], registers: [registry] }); async function initTracePrompt2(cfg) { await initTracePrompt(cfg); } function wrapLLM(originalFn, meta) { const staticMeta = ConfigManager.cfg.staticMeta; return async function wrapped(prompt, params) { const t0 = perf_hooks.performance.now(); const result = await originalFn(prompt, params); const t1 = perf_hooks.performance.now(); wrapperLatencyHist.observe(t1 - t0); const plaintextJson = JSON.stringify({ prompt, response: result }); const enc = await encryptBuffer(Buffer.from(plaintextJson, "utf8")); const payload = { ...staticMeta, orgId: ConfigManager.cfg.orgId, modelVendor: meta.modelVendor, modelName: meta.modelName, userId: meta.userId, ts_client: (/* @__PURE__ */ new Date()).toISOString(), latency_ms: +(t1 - t0).toFixed(2), prompt_tokens: countTokens(prompt), response_tokens: countTokens( typeof result === "string" ? result : JSON.stringify(result) ), enc }; const leafHash = computeLeaf(stringify(payload)); PersistentBatcher.enqueue({ payload, leafHash }); return result; }; } exports.PersistentBatcher = PersistentBatcher; exports.decryptBundle = decryptBundle; exports.initTracePrompt = initTracePrompt2; exports.registry = registry; exports.wrapLLM = wrapLLM; //# sourceMappingURL=index.js.map //# sourceMappingURL=index.js.map