UNPKG

openclaw

Version:

Multi-channel AI gateway with extensible messaging integrations

561 lines (560 loc) 22.9 kB
import "./src-vebZIeLe.js"; import { t as expectDefined } from "./expect-CyE8FADM.js"; import { r as truncateUtf16Safe } from "./utf16-slice-D_ngcYKd.js"; import { r as isMissingPathError } from "./errno-CkbDOfLk.js"; import { p as redactSensitiveText } from "./redact-BtvPPfTi.js"; import { w as resolveStateDir } from "./paths-D2sRr1a_.js"; import "./errors-Db3Ymjlb.js"; import { t as formatDiagnosticFilenameTimestamp } from "./timestamps-CD_Oo5x3.js"; import { r as replaceFileAtomicSync } from "./replace-file-BAJ-TWzD.js"; import { t as registerFatalErrorHook } from "./fatal-error-hooks-D8bswZzH.js"; import { n as getDiagnosticStabilitySnapshot, t as MAX_DIAGNOSTIC_STABILITY_LIMIT } from "./diagnostic-stability-CLaOJzMC.js"; import fs from "node:fs"; import process from "node:process"; import path from "node:path"; //#region src/logging/diagnostic-stability-bundle.ts const DIAGNOSTIC_STABILITY_BUNDLE_VERSION = 1; const DEFAULT_DIAGNOSTIC_STABILITY_BUNDLE_LIMIT = MAX_DIAGNOSTIC_STABILITY_LIMIT; const DEFAULT_DIAGNOSTIC_STABILITY_BUNDLE_RETENTION = 20; const MAX_DIAGNOSTIC_STABILITY_BUNDLE_BYTES = 5242880; const SAFE_REASON_CODE = /^[A-Za-z0-9_.:-]{1,120}$/u; const BUNDLE_PREFIX = "openclaw-stability-"; const BUNDLE_SUFFIX = ".json"; const REDACTED_HOSTNAME = "<redacted-hostname>"; const MAX_SAFE_ERROR_MESSAGE_LENGTH = 500; let fatalHookUnsubscribe = null; function normalizeReason(reason) { return SAFE_REASON_CODE.test(reason) ? reason : "unknown"; } function readErrorCode(error) { if (!error || typeof error !== "object" || !("code" in error)) return; const code = error.code; if (typeof code === "string" && SAFE_REASON_CODE.test(code)) return code; if (typeof code === "number" && Number.isFinite(code)) return String(code); } function readErrorName(error) { if (!error || typeof error !== "object" || !("name" in error)) return; const name = error.name; return typeof name === "string" && SAFE_REASON_CODE.test(name) ? name : void 0; } function readErrorMessage(error) { if (!error || typeof error !== "object" || !("message" in error)) return; const message = error.message; if (typeof message !== "string") return; const sanitized = redactSensitiveText(message, { mode: "tools" }).replace(/\s+/gu, " ").trim(); if (!sanitized) return; return sanitized.length > MAX_SAFE_ERROR_MESSAGE_LENGTH ? `${truncateUtf16Safe(sanitized, MAX_SAFE_ERROR_MESSAGE_LENGTH)}...` : sanitized; } function readSafeErrorMetadata(error) { const name = readErrorName(error); const code = readErrorCode(error); const message = readErrorMessage(error); if (!name && !code && !message) return; return { ...name ? { name } : {}, ...code ? { code } : {}, ...message ? { message } : {} }; } function resolveDiagnosticStabilityBundleDir(options = {}) { return path.join(options.stateDir ?? resolveStateDir(options.env ?? process.env), "logs", "stability"); } function buildBundlePath(dir, now, reason) { return path.join(dir, `${BUNDLE_PREFIX}${formatDiagnosticFilenameTimestamp(now)}-${process.pid}-${normalizeReason(reason)}${BUNDLE_SUFFIX}`); } function isBundleFile(name) { return name.startsWith(BUNDLE_PREFIX) && name.endsWith(BUNDLE_SUFFIX); } function readObject(value, label) { if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error(`Invalid stability bundle: ${label} must be an object`); return value; } function readRequiredNumber(value, label) { if (typeof value !== "number" || !Number.isFinite(value)) throw new Error(`Invalid stability bundle: ${label} must be a finite number`); return value; } function readOptionalPositiveInteger(value, label) { if (value === void 0) return; const parsed = readRequiredNumber(value, label); return parsed >= 0 ? Math.floor(parsed) : void 0; } function readTimestampMs(value, label) { const timestamp = readRequiredNumber(value, label); if (Number.isNaN(new Date(timestamp).getTime())) throw new Error(`Invalid stability bundle: ${label} must be a valid timestamp`); return timestamp; } function readOptionalNumber(value, label) { if (value === void 0) return; return readRequiredNumber(value, label); } function readRequiredString(value, label) { if (typeof value !== "string") throw new Error(`Invalid stability bundle: ${label} must be a string`); return value; } function readTimestampString(value, label) { const timestamp = readRequiredString(value, label); if (Number.isNaN(new Date(timestamp).getTime())) throw new Error(`Invalid stability bundle: ${label} must be a valid timestamp`); return timestamp; } function readCodeString(value, label) { const code = readRequiredString(value, label); if (!SAFE_REASON_CODE.test(code)) throw new Error(`Invalid stability bundle: ${label} must be a safe diagnostic code`); return code; } function readOptionalCodeString(value, label) { if (value === void 0) return; const code = readRequiredString(value, label); return SAFE_REASON_CODE.test(code) ? code : void 0; } function assignOptionalFields(target, source, label, fields, read) { for (const key of fields) { const parsed = read(source[key], `${label}.${key}`); if (parsed !== void 0) target[key] = parsed; } } function readMemoryUsage(value, label) { const memory = readObject(value, label); return { rssBytes: readRequiredNumber(memory.rssBytes, `${label}.rssBytes`), heapTotalBytes: readRequiredNumber(memory.heapTotalBytes, `${label}.heapTotalBytes`), heapUsedBytes: readRequiredNumber(memory.heapUsedBytes, `${label}.heapUsedBytes`), externalBytes: readRequiredNumber(memory.externalBytes, `${label}.externalBytes`), arrayBuffersBytes: readRequiredNumber(memory.arrayBuffersBytes, `${label}.arrayBuffersBytes`) }; } function readHeapStatistics(value) { if (value === void 0) return; const source = readObject(value, "evidence.memoryPressure.heapStatistics"); const result = {}; assignOptionalFields(result, source, "evidence.memoryPressure.heapStatistics", [ "totalHeapSizeBytes", "totalHeapSizeExecutableBytes", "totalPhysicalSizeBytes", "totalAvailableSizeBytes", "usedHeapSizeBytes", "heapSizeLimitBytes", "mallocedMemoryBytes", "externalMemoryBytes" ], readOptionalPositiveInteger); return Object.keys(result).length > 0 ? result : void 0; } function readHeapSpaces(value) { if (value === void 0) return; if (!Array.isArray(value)) throw new Error("Invalid stability bundle: evidence.memoryPressure.heapSpaces must be an array"); const spaces = []; for (const [index, entry] of value.entries()) { const source = readObject(entry, `evidence.memoryPressure.heapSpaces[${index}]`); const spaceName = readOptionalCodeString(source.spaceName, `evidence.memoryPressure.heapSpaces[${index}].spaceName`); if (!spaceName) continue; spaces.push({ spaceName, spaceSizeBytes: readOptionalPositiveInteger(source.spaceSizeBytes, `evidence.memoryPressure.heapSpaces[${index}].spaceSizeBytes`) ?? 0, spaceUsedBytes: readOptionalPositiveInteger(source.spaceUsedBytes, `evidence.memoryPressure.heapSpaces[${index}].spaceUsedBytes`) ?? 0, spaceAvailableBytes: readOptionalPositiveInteger(source.spaceAvailableBytes, `evidence.memoryPressure.heapSpaces[${index}].spaceAvailableBytes`) ?? 0, physicalSpaceSizeBytes: readOptionalPositiveInteger(source.physicalSpaceSizeBytes, `evidence.memoryPressure.heapSpaces[${index}].physicalSpaceSizeBytes`) ?? 0 }); } return spaces.length > 0 ? spaces : void 0; } function readCgroupMemorySummary(value) { if (value === void 0) return; const source = readObject(value, "evidence.memoryPressure.cgroup"); const version = readCodeString(source.version, "evidence.memoryPressure.cgroup.version"); if (version !== "v2") return; const valuesSource = readObject(source.values, "evidence.memoryPressure.cgroup.values"); const values = {}; for (const [key, raw] of Object.entries(valuesSource)) { if (!SAFE_REASON_CODE.test(key)) continue; if (raw === "max") values[key] = "max"; else values[key] = readOptionalPositiveInteger(raw, `evidence.memoryPressure.cgroup.values.${key}`) ?? 0; } return { version, values, events: readNumberMap(source.events, "evidence.memoryPressure.cgroup.events") }; } function readActiveResources(value) { if (value === void 0) return; const source = readObject(value, "evidence.memoryPressure.activeResources"); return { total: readOptionalPositiveInteger(source.total, "evidence.memoryPressure.activeResources.total") ?? 0, byType: readNumberMap(source.byType, "evidence.memoryPressure.activeResources.byType") }; } function readSessionFiles(value) { if (value === void 0) return; if (!Array.isArray(value)) throw new Error("Invalid stability bundle: evidence.memoryPressure.topSessionFiles must be an array"); const files = []; for (const [index, entry] of value.entries()) { const source = readObject(entry, `evidence.memoryPressure.topSessionFiles[${index}]`); const relativePath = readRequiredString(source.relativePath, `evidence.memoryPressure.topSessionFiles[${index}].relativePath`); if (path.isAbsolute(relativePath) || relativePath.includes("..") || relativePath.length > 300 || /[\r\n]/u.test(relativePath)) continue; files.push({ relativePath: sanitizeSessionEvidencePath(relativePath), sizeBytes: readOptionalPositiveInteger(source.sizeBytes, `evidence.memoryPressure.topSessionFiles[${index}].sizeBytes`) ?? 0, mtimeMs: readOptionalPositiveInteger(source.mtimeMs, `evidence.memoryPressure.topSessionFiles[${index}].mtimeMs`) ?? 0 }); } return files.length > 0 ? files : void 0; } function readMemoryPressureEvidence(value) { if (value === void 0) return; const pressure = readObject(value, "evidence.memoryPressure"); const level = readCodeString(pressure.level, "evidence.memoryPressure.level"); const reason = readCodeString(pressure.reason, "evidence.memoryPressure.reason"); if (level !== "warning" && level !== "critical" || !isMemoryPressureReason(reason)) return; const heapStatistics = readHeapStatistics(pressure.heapStatistics); const heapSpaces = readHeapSpaces(pressure.heapSpaces); const cgroup = readCgroupMemorySummary(pressure.cgroup); const activeResources = readActiveResources(pressure.activeResources); const topSessionFiles = readSessionFiles(pressure.topSessionFiles); const result = { level, reason, memory: readMemoryUsage(pressure.memory, "evidence.memoryPressure.memory") }; assignOptionalFields(result, pressure, "evidence.memoryPressure", [ "thresholdBytes", "rssGrowthBytes", "windowMs" ], readOptionalNumber); return { ...result, ...heapStatistics ? { heapStatistics } : {}, ...heapSpaces ? { heapSpaces } : {}, ...cgroup ? { cgroup } : {}, ...activeResources ? { activeResources } : {}, ...topSessionFiles ? { topSessionFiles } : {} }; } function readBundleEvidence(value) { if (value === void 0) return; const memoryPressure = readMemoryPressureEvidence(readObject(value, "evidence").memoryPressure); return memoryPressure ? { memoryPressure } : void 0; } function readNumberMap(value, label) { const source = readObject(value, label); const result = {}; for (const [key, entry] of Object.entries(source)) { if (!SAFE_REASON_CODE.test(key)) continue; result[key] = readRequiredNumber(entry, `${label}.${key}`); } return result; } function readOptionalMemorySummary(value) { if (value === void 0) return; const memory = readObject(value, "snapshot.summary.memory"); const latest = memory.latest === void 0 ? void 0 : readMemoryUsage(memory.latest, "snapshot.summary.memory.latest"); return { ...latest ? { latest } : {}, ...memory.maxRssBytes !== void 0 ? { maxRssBytes: readRequiredNumber(memory.maxRssBytes, "snapshot.summary.memory.maxRssBytes") } : {}, ...memory.maxHeapUsedBytes !== void 0 ? { maxHeapUsedBytes: readRequiredNumber(memory.maxHeapUsedBytes, "snapshot.summary.memory.maxHeapUsedBytes") } : {}, pressureCount: readRequiredNumber(memory.pressureCount, "snapshot.summary.memory.pressureCount") }; } function readOptionalPayloadLargeSummary(value) { if (value === void 0) return; const payloadLarge = readObject(value, "snapshot.summary.payloadLarge"); return { count: readRequiredNumber(payloadLarge.count, "snapshot.summary.payloadLarge.count"), rejected: readRequiredNumber(payloadLarge.rejected, "snapshot.summary.payloadLarge.rejected"), truncated: readRequiredNumber(payloadLarge.truncated, "snapshot.summary.payloadLarge.truncated"), chunked: readRequiredNumber(payloadLarge.chunked, "snapshot.summary.payloadLarge.chunked"), bySurface: readNumberMap(payloadLarge.bySurface, "snapshot.summary.payloadLarge.bySurface") }; } function readStabilityEventRecord(value, label) { const record = readObject(value, label); const sanitized = { seq: readRequiredNumber(record.seq, `${label}.seq`), ts: readTimestampMs(record.ts, `${label}.ts`), type: readCodeString(record.type, `${label}.type`) }; assignOptionalFields(sanitized, record, label, [ "channel", "pluginId", "source", "surface", "action", "reason", "outcome", "level", "phase", "approvalId", "detector", "toolName", "activeWorkKind", "pairedToolName", "provider", "model" ], readOptionalCodeString); assignOptionalFields(sanitized, record, label, [ "durationMs", "requestBytes", "responseBytes", "timeToFirstByteMs", "costUsd", "count", "bytes", "limitBytes", "thresholdBytes", "rssGrowthBytes", "windowMs", "ageMs", "queueDepth", "queueSize", "queueLength", "waitMs", "active", "waiting", "queued", "droppedEvents", "droppedTrustedEvents", "droppedUntrustedEvents", "droppedPriorityEvents", "maxQueueLength", "drainBatchSize" ], readOptionalNumber); if (record.webhooks !== void 0) { const webhooks = readObject(record.webhooks, `${label}.webhooks`); sanitized.webhooks = { received: readRequiredNumber(webhooks.received, `${label}.webhooks.received`), processed: readRequiredNumber(webhooks.processed, `${label}.webhooks.processed`), errors: readRequiredNumber(webhooks.errors, `${label}.webhooks.errors`) }; } if (record.memory !== void 0) sanitized.memory = readMemoryUsage(record.memory, `${label}.memory`); if (record.usage !== void 0) { const usage = readObject(record.usage, `${label}.usage`); sanitized.usage = {}; assignOptionalFields(sanitized.usage, usage, `${label}.usage`, [ "input", "output", "cacheRead", "cacheWrite", "promptTokens", "total" ], readOptionalNumber); } if (record.context !== void 0) { const context = readObject(record.context, `${label}.context`); sanitized.context = {}; assignOptionalFields(sanitized.context, context, `${label}.context`, ["limit", "used"], readOptionalNumber); } return sanitized; } function readStabilitySnapshot(value) { const snapshot = readObject(value, "snapshot"); const generatedAt = readTimestampString(snapshot.generatedAt, "snapshot.generatedAt"); const capacity = readRequiredNumber(snapshot.capacity, "snapshot.capacity"); const count = readRequiredNumber(snapshot.count, "snapshot.count"); const dropped = readRequiredNumber(snapshot.dropped, "snapshot.dropped"); const firstSeq = readOptionalNumber(snapshot.firstSeq, "snapshot.firstSeq"); const lastSeq = readOptionalNumber(snapshot.lastSeq, "snapshot.lastSeq"); if (!Array.isArray(snapshot.events)) throw new Error("Invalid stability bundle: snapshot.events must be an array"); const events = snapshot.events.map((event, index) => readStabilityEventRecord(event, `snapshot.events[${index}]`)); const summary = readObject(snapshot.summary, "snapshot.summary"); return { generatedAt, capacity, count, dropped, ...firstSeq !== void 0 ? { firstSeq } : {}, ...lastSeq !== void 0 ? { lastSeq } : {}, events, summary: { byType: readNumberMap(summary.byType, "snapshot.summary.byType"), ...summary.memory !== void 0 ? { memory: readOptionalMemorySummary(summary.memory) } : {}, ...summary.payloadLarge !== void 0 ? { payloadLarge: readOptionalPayloadLargeSummary(summary.payloadLarge) } : {} } }; } function parseDiagnosticStabilityBundle(value) { const bundle = readObject(value, "bundle"); if (bundle.version !== 1) throw new Error(`Unsupported stability bundle version: ${String(bundle.version)}`); const processInfo = readObject(bundle.process, "process"); readObject(bundle.host, "host"); const error = bundle.error === void 0 ? void 0 : readSafeErrorMetadata(bundle.error); const evidence = readBundleEvidence(bundle.evidence); return { version: 1, generatedAt: readTimestampString(bundle.generatedAt, "generatedAt"), reason: normalizeReason(readRequiredString(bundle.reason, "reason")), process: { pid: readRequiredNumber(processInfo.pid, "process.pid"), platform: readCodeString(processInfo.platform, "process.platform"), arch: readCodeString(processInfo.arch, "process.arch"), node: readCodeString(processInfo.node, "process.node"), uptimeMs: readRequiredNumber(processInfo.uptimeMs, "process.uptimeMs") }, host: { hostname: REDACTED_HOSTNAME }, ...error ? { error } : {}, ...evidence ? { evidence } : {}, snapshot: readStabilitySnapshot(bundle.snapshot) }; } function sanitizeSessionEvidencePath(relativePath) { const parts = relativePath.split("/"); if (parts.length === 4 && parts[0] === "agents" && parts[2] === "sessions") return `agents/<agent>/sessions/${sanitizeSessionEvidenceFileName(expectDefined(parts[3], "parts entry at 3"))}`; if (parts.length === 2 && parts[0] === "sessions") return `sessions/${sanitizeSessionEvidenceFileName(expectDefined(parts[1], "parts entry at 1"))}`; return redactSensitiveText(relativePath, { mode: "tools" }); } function sanitizeSessionEvidenceFileName(fileName) { if (fileName === "sessions.json") return "sessions.json"; if (fileName.endsWith(".jsonl")) return "<session>.jsonl"; if (fileName.endsWith(".json")) return "<session>.json"; return "<session>"; } function isMemoryPressureReason(reason) { return reason === "rss_threshold" || reason === "heap_threshold" || reason === "rss_growth"; } function listDiagnosticStabilityBundleFilesSync(options = {}) { const dir = resolveDiagnosticStabilityBundleDir(options); try { return fs.readdirSync(dir, { withFileTypes: true }).filter((entry) => entry.isFile() && isBundleFile(entry.name)).map((entry) => { const file = path.join(dir, entry.name); return { path: file, mtimeMs: fs.statSync(file).mtimeMs }; }).toSorted((a, b) => b.mtimeMs - a.mtimeMs || b.path.localeCompare(a.path)); } catch (error) { if (isMissingPathError(error)) return []; throw error; } } function readDiagnosticStabilityBundleFileSync(file) { try { const stat = fs.statSync(file); if (stat.size > 5242880) throw new Error(`Stability bundle is too large: ${stat.size} bytes exceeds ${MAX_DIAGNOSTIC_STABILITY_BUNDLE_BYTES}`); const raw = fs.readFileSync(file, "utf8"); const bundle = parseDiagnosticStabilityBundle(JSON.parse(raw)); return { status: "found", path: file, mtimeMs: stat.mtimeMs, bundle }; } catch (error) { return { status: "failed", path: file, error }; } } function readLatestDiagnosticStabilityBundleSync(options = {}) { try { const latest = listDiagnosticStabilityBundleFilesSync(options)[0]; if (!latest) return { status: "missing", dir: resolveDiagnosticStabilityBundleDir(options) }; return readDiagnosticStabilityBundleFileSync(latest.path); } catch (error) { return { status: "failed", error }; } } function pruneOldBundles(dir, retention, retainedFile) { if (!Number.isFinite(retention) || retention < 1) return; try { const entries = fs.readdirSync(dir, { withFileTypes: true }).filter((entry) => entry.isFile() && isBundleFile(entry.name)).map((entry) => { const file = path.join(dir, entry.name); let mtimeMs = 0; try { mtimeMs = fs.statSync(file).mtimeMs; } catch {} return { file, mtimeMs }; }).filter((entry) => entry.file !== retainedFile).toSorted((a, b) => b.mtimeMs - a.mtimeMs || b.file.localeCompare(a.file)); for (const entry of entries.slice(retention - 1)) try { fs.unlinkSync(entry.file); } catch {} } catch {} } function writeDiagnosticStabilityBundleSync(options) { try { const now = options.now ?? /* @__PURE__ */ new Date(); const snapshot = getDiagnosticStabilitySnapshot({ limit: options.limit ?? DEFAULT_DIAGNOSTIC_STABILITY_BUNDLE_LIMIT }); if (!options.includeEmpty && snapshot.count === 0) return { status: "skipped", reason: "empty" }; const reason = normalizeReason(options.reason); const error = options.error ? readSafeErrorMetadata(options.error) : void 0; const bundle = { version: 1, generatedAt: now.toISOString(), reason, process: { pid: process.pid, platform: process.platform, arch: process.arch, node: process.versions.node, uptimeMs: Math.round(process.uptime() * 1e3) }, host: { hostname: REDACTED_HOSTNAME }, ...error ? { error } : {}, ...options.evidence ? { evidence: options.evidence } : {}, snapshot }; const dir = resolveDiagnosticStabilityBundleDir(options); const file = buildBundlePath(dir, now, reason); replaceFileAtomicSync({ filePath: file, content: `${JSON.stringify(bundle, null, 2)}\n`, dirMode: 448, mode: 384, tempPrefix: ".openclaw-stability" }); pruneOldBundles(dir, options.retention ?? DEFAULT_DIAGNOSTIC_STABILITY_BUNDLE_RETENTION, file); return { status: "written", path: file, bundle }; } catch (error) { return { status: "failed", error }; } } function writeDiagnosticStabilityBundleForFailureSync(reason, error, options = {}) { const result = writeDiagnosticStabilityBundleSync({ ...options, reason, error, includeEmpty: true }); if (result.status === "written") return { status: "written", path: result.path, message: `wrote stability bundle: ${result.path}` }; if (result.status === "failed") return { status: "failed", error: result.error, message: `failed to write stability bundle: ${String(result.error)}` }; return result; } function installDiagnosticStabilityFatalHook(options = {}) { if (fatalHookUnsubscribe) return; fatalHookUnsubscribe = registerFatalErrorHook(({ reason, error }) => { const result = writeDiagnosticStabilityBundleForFailureSync(reason, error, options); return "message" in result ? result.message : void 0; }); } function uninstallDiagnosticStabilityFatalHook() { fatalHookUnsubscribe?.(); fatalHookUnsubscribe = null; } //#endregion export { readLatestDiagnosticStabilityBundleSync as a, writeDiagnosticStabilityBundleSync as c, readDiagnosticStabilityBundleFileSync as i, MAX_DIAGNOSTIC_STABILITY_BUNDLE_BYTES as n, uninstallDiagnosticStabilityFatalHook as o, installDiagnosticStabilityFatalHook as r, writeDiagnosticStabilityBundleForFailureSync as s, DIAGNOSTIC_STABILITY_BUNDLE_VERSION as t };