openclaw/dist/state-migrations.media-persistence-BbW15Isv.js

Multi-channel AI gateway with extensible messaging integrations

import { c as isRecord } from "./record-coerce-DItp3I4t.js";
import { t as formatErrorMessage } from "./errors-Db3Ymjlb.js";
import { r as replaceFileAtomicSync } from "./replace-file-BAJ-TWzD.js";
import { h as clearNodeSqliteKyselyCacheForDatabase, l as runSqliteDeferredTransactionSync, t as openNodeSqliteDatabase, u as runSqliteImmediateTransactionSync } from "./node-sqlite-BpQX3W0e.js";
import { a as getNodeSqliteKysely, r as executeSqliteQuerySync } from "./kysely-sync-COmh4HWh.js";
import { o as OPENCLAW_SQLITE_BUSY_TIMEOUT_MS } from "./openclaw-state-db-contract-DYCYxE4w.js";
import { t as VERSION } from "./version-v1kuAkGj.js";
import { a as readSqliteUserVersion } from "./sqlite-user-version-DFJCxX41.js";
import { Wt as repairCanonicalSqliteIndexes } from "./openclaw-state-db-BRTnL-D8.js";
import { _ as readSessionArchiveContentSync, h as encodeSessionArchiveContent, m as decodeSessionArchiveBytes, p as SESSION_ARCHIVE_ZSTD_SUFFIX } from "./artifacts-DMDO9PHy.js";
import { Y as OPENCLAW_AGENT_SCHEMA_SQL, d as registerOpenClawAgentDatabase, j as withLegacySessionParticipantsSchema, n as assertOpenClawAgentDatabaseOwner, o as ensureOpenClawAgentDatabaseSchema, s as migrateOpenClawAgentDatabaseToMediaPrerequisiteSchema, w as assertOpenClawAgentSchemaContains } from "./openclaw-agent-db-maintenance-wTIy-jt-.js";
import "./openclaw-agent-db-contract-CGTyjij4.js";
import { _ as withAgentDatabaseMaintenanceLease } from "./openclaw-agent-db-CWtDoRbC.js";
import { i as hasMeaningfulRetiredMediaCarrier, r as canonicalizePersistedUserMessageMedia } from "./media-facts-8Sl_nPCc.js";
import { u as rewriteSqliteTranscriptEventRowsInTransaction } from "./session-accessor.sqlite-transcript-store-B0zw3fAU.js";
import { u as resolveSqliteTranscriptArchiveDirectory } from "./session-accessor.sqlite-scope-2KfMzb44.js";
import { c as repairGatewayAgentMediaMigrationStartupFailures } from "./gateway-boot-lifecycle-JblQ4jbl.js";
import { n as listTranscriptArchives, r as resolveAgentDatabaseMigrationTargets } from "./state-migrations.media-persistence-targets-D45MAuwp.js";
import fs from "node:fs";
import path from "node:path";
import { createHash } from "node:crypto";
//#region src/infra/state-migrations.media-persistence.ts
const PREVIOUS_MEDIA_SCHEMA_VERSION = 16;
const ARCHIVE_TEMP_MARKER = ".media-retirement";
const MEDIA_MIGRATION_ROW_BATCH_SIZE = 64;
function transformTranscriptEvent(event) {
	if (!isRecord(event) || event.type !== "message" || !isRecord(event.message)) return {
		changed: false,
		event
	};
	const canonical = canonicalizePersistedUserMessageMedia(event.message);
	return canonical.changed ? {
		changed: true,
		event: {
			...event,
			message: canonical.message
		}
	} : {
		changed: false,
		event
	};
}
function parseTranscriptEvent(raw, owner) {
	try {
		return JSON.parse(raw);
	} catch (error) {
		throw new Error(`${owner} contains invalid transcript JSON: ${String(error)}`, { cause: error });
	}
}
function eventIdentity(event) {
	if (!isRecord(event)) return JSON.stringify({
		id: null,
		parentId: null,
		type: null
	});
	return JSON.stringify({
		id: typeof event.id === "string" ? event.id : null,
		parentId: typeof event.parentId === "string" ? event.parentId : null,
		type: typeof event.type === "string" ? event.type : null
	});
}
function assertEventIdentitiesUnchanged(before, after, owner) {
	if (before.length !== after.length) throw new Error(`${owner} event count changed during media migration`);
	for (let index = 0; index < before.length; index += 1) if (eventIdentity(before[index]) !== eventIdentity(after[index])) throw new Error(`${owner} event identity changed at index ${index}`);
}
function forEachMediaEventBatch(params) {
	const db = getNodeSqliteKysely(params.database);
	let cursor;
	while (true) {
		let query = db.selectFrom(params.table).select([
			"session_id",
			"seq",
			"event_json"
		]).orderBy("session_id", "asc").orderBy("seq", "asc").limit(MEDIA_MIGRATION_ROW_BATCH_SIZE);
		const after = cursor;
		if (after) query = query.where((expression) => expression(expression.refTuple("session_id", "seq"), ">", expression.tuple(after.sessionId, after.seq)));
		const rows = executeSqliteQuerySync(params.database, query).rows;
		const last = rows.at(-1);
		if (!last) return;
		params.visit(rows);
		cursor = {
			seq: last.seq,
			sessionId: last.session_id
		};
	}
}
function scanTranscriptRows(params) {
	const { database, pathname, writer } = params;
	const db = getNodeSqliteKysely(database);
	let lastChangedSessionId;
	let changedSessions = 0;
	forEachMediaEventBatch({
		database,
		table: "transcript_events",
		visit: (rows) => {
			const sessionIds = [...new Set(rows.map((row) => row.session_id))];
			const sessionKeys = new Map(executeSqliteQuerySync(database, db.selectFrom("session_windows").select(["session_id", "session_key"]).where("session_id", "in", sessionIds)).rows.map((row) => [row.session_id, row.session_key]));
			for (const sessionId of sessionIds) if (!sessionKeys.has(sessionId)) throw new Error(`${pathname}:${sessionId} has transcript rows without a session window`);
			const rewritesBySession = /* @__PURE__ */ new Map();
			for (const row of rows) {
				const owner = `${pathname}:${row.session_id}:${row.seq}`;
				const event = parseTranscriptEvent(row.event_json, owner);
				const transformed = transformTranscriptEvent(event);
				if (!transformed.changed) continue;
				if (eventIdentity(event) !== eventIdentity(transformed.event)) throw new Error(`${owner} event identity changed during media migration`);
				if (lastChangedSessionId !== row.session_id) {
					lastChangedSessionId = row.session_id;
					changedSessions += 1;
				}
				const rewrites = rewritesBySession.get(row.session_id) ?? [];
				rewrites.push({
					event: transformed.event,
					expectedEventJson: row.event_json,
					seq: row.seq
				});
				rewritesBySession.set(row.session_id, rewrites);
			}
			if (writer) for (const [sessionId, rewrites] of rewritesBySession) {
				const sessionKey = sessionKeys.get(sessionId);
				if (!sessionKey) throw new Error(`${pathname}:${sessionId} has transcript rows without a session window`);
				rewriteSqliteTranscriptEventRowsInTransaction(writer, {
					agentId: writer.agentId,
					path: pathname,
					sessionId,
					sessionKey
				}, rewrites);
			}
		}
	});
	return changedSessions;
}
function rewriteTrajectoryEventJson(eventJson, owner) {
	let event;
	try {
		event = JSON.parse(eventJson);
	} catch (error) {
		throw new Error(`${owner} contains invalid trajectory JSON: ${String(error)}`, { cause: error });
	}
	if (!isRecord(event) || !isRecord(event.data) || !Array.isArray(event.data.messagesSnapshot)) return eventJson;
	let changed = false;
	const messagesSnapshot = event.data.messagesSnapshot.map((message) => {
		if (!isRecord(message) || !hasMeaningfulRetiredMediaCarrier(message)) return message;
		const canonical = canonicalizePersistedUserMessageMedia(message);
		changed ||= canonical.changed;
		return canonical.message;
	});
	return changed ? JSON.stringify({
		...event,
		data: {
			...event.data,
			messagesSnapshot
		}
	}) : eventJson;
}
function scanTrajectoryRows(params) {
	const { database, pathname, rewrite } = params;
	const db = getNodeSqliteKysely(database);
	let changedRows = 0;
	forEachMediaEventBatch({
		database,
		table: "trajectory_runtime_events",
		visit: (rows) => {
			for (const row of rows) {
				const rewrittenEventJson = rewriteTrajectoryEventJson(row.event_json, `${pathname}:${row.session_id}:${row.seq}`);
				if (rewrittenEventJson === row.event_json) continue;
				changedRows += 1;
				if (rewrite) executeSqliteQuerySync(database, db.updateTable("trajectory_runtime_events").set({ event_json: rewrittenEventJson }).where("session_id", "=", row.session_id).where("seq", "=", row.seq));
			}
		}
	});
	return changedRows;
}
function readMediaSourceVersion(database) {
	const dataVersionRow = database.prepare("PRAGMA data_version").get();
	const counts = database.prepare(`SELECT
        (SELECT COUNT(*) FROM transcript_events) AS transcript_rows,
        (SELECT COALESCE(SUM(LENGTH(event_json)), 0) FROM transcript_events) AS transcript_bytes,
        (SELECT CAST(COALESCE(SUM(created_at), 0) AS TEXT) FROM transcript_events) AS transcript_created_at,
        (SELECT COUNT(*) FROM trajectory_runtime_events) AS trajectory_rows,
        (SELECT COALESCE(SUM(LENGTH(event_json)), 0) FROM trajectory_runtime_events) AS trajectory_bytes`).get();
	const number = (value) => typeof value === "bigint" ? Number(value) : typeof value === "number" ? value : 0;
	const count = (key) => number(isRecord(counts) ? counts[key] : void 0);
	return {
		dataVersion: number(isRecord(dataVersionRow) ? dataVersionRow.data_version : void 0),
		trajectoryBytes: count("trajectory_bytes"),
		trajectoryRows: count("trajectory_rows"),
		transcriptBytes: count("transcript_bytes"),
		transcriptCreatedAt: String((isRecord(counts) ? counts.transcript_created_at : void 0) ?? "0"),
		transcriptRows: count("transcript_rows")
	};
}
function mediaSourceDriftMessage(pathname, expected, current) {
	if (expected.transcriptRows !== current.transcriptRows || expected.transcriptBytes !== current.transcriptBytes || expected.transcriptCreatedAt !== current.transcriptCreatedAt) return `${pathname} transcript source changed before migration commit`;
	if (expected.trajectoryRows !== current.trajectoryRows || expected.trajectoryBytes !== current.trajectoryBytes) return `${pathname} trajectory source changed before migration commit`;
	return `${pathname} source changed before migration transaction`;
}
function createMigrationDatabaseHandle(database, agentId, pathname) {
	return {
		agentId,
		db: database,
		path: pathname,
		walMaintenance: {
			checkpoint: () => false,
			close: () => false
		}
	};
}
function refreshAgentDatabasePlannerStatistics(database) {
	database.exec("PRAGMA analysis_limit=1000; ANALYZE main;");
}
function migrateAgentDatabase(params) {
	const database = openNodeSqliteDatabase(params.pathname);
	try {
		database.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`);
		let metadata = assertOpenClawAgentDatabaseOwner(database, {
			agentId: params.agentId,
			pathname: params.pathname
		});
		let userVersion = readSqliteUserVersion(database);
		const initialVersion = userVersion;
		if (userVersion <= PREVIOUS_MEDIA_SCHEMA_VERSION) {
			migrateOpenClawAgentDatabaseToMediaPrerequisiteSchema(database, {
				agentId: params.agentId,
				path: params.pathname
			});
			metadata = assertOpenClawAgentDatabaseOwner(database, {
				agentId: params.agentId,
				pathname: params.pathname
			});
			userVersion = readSqliteUserVersion(database);
		}
		if (metadata.schemaVersion !== userVersion) throw new Error(`${params.pathname} metadata schema version ${metadata.schemaVersion ?? "invalid"} does not match ${userVersion}`);
		if (userVersion >= 17) {
			ensureOpenClawAgentDatabaseSchema(database, {
				agentId: params.agentId,
				path: params.pathname
			});
			userVersion = readSqliteUserVersion(database);
		}
		const schemaMode = userVersion < 19 ? "legacy" : "current";
		const schemaSql = schemaMode === "legacy" ? withLegacySessionParticipantsSchema(OPENCLAW_AGENT_SCHEMA_SQL) : OPENCLAW_AGENT_SCHEMA_SQL;
		if (userVersion === PREVIOUS_MEDIA_SCHEMA_VERSION) repairCanonicalSqliteIndexes(database, params.pathname, schemaSql, { validateAfterRepair: () => assertOpenClawAgentSchemaContains(database, params.pathname, schemaSql, schemaMode) });
		assertOpenClawAgentSchemaContains(database, params.pathname, schemaSql, schemaMode);
		const mediaSchemaUpgrade = userVersion === PREVIOUS_MEDIA_SCHEMA_VERSION;
		if (!mediaSchemaUpgrade) {
			const detected = runSqliteDeferredTransactionSync(database, () => ({
				rewrittenSessions: scanTranscriptRows({
					database,
					pathname: params.pathname
				}),
				rewrittenTrajectoryRows: scanTrajectoryRows({
					database,
					pathname: params.pathname,
					rewrite: false
				})
			}), {
				databaseLabel: params.pathname,
				operationLabel: "media-persistence-detection"
			});
			if (detected.rewrittenSessions === 0 && detected.rewrittenTrajectoryRows === 0) {
				refreshAgentDatabasePlannerStatistics(database);
				return {
					...detected,
					initialVersion,
					finalVersion: userVersion
				};
			}
		}
		const sourceVersion = readMediaSourceVersion(database);
		params.beforeTransaction?.();
		const owner = createMigrationDatabaseHandle(database, params.agentId, params.pathname);
		const rewritten = runSqliteImmediateTransactionSync(database, () => {
			const currentSourceVersion = readMediaSourceVersion(database);
			if (currentSourceVersion.dataVersion !== sourceVersion.dataVersion) throw new Error(mediaSourceDriftMessage(params.pathname, sourceVersion, currentSourceVersion));
			const rewrittenSessions = scanTranscriptRows({
				database,
				pathname: params.pathname,
				writer: owner
			});
			const rewrittenTrajectoryRows = scanTrajectoryRows({
				database,
				pathname: params.pathname,
				rewrite: true
			});
			if (mediaSchemaUpgrade) {
				const db = getNodeSqliteKysely(database);
				database.exec(`PRAGMA user_version = 17;`);
				executeSqliteQuerySync(database, db.updateTable("schema_meta").set({
					app_version: VERSION,
					schema_version: 17,
					updated_at: Date.now()
				}).where("meta_key", "=", "primary"));
			}
			return {
				rewrittenSessions,
				rewrittenTrajectoryRows
			};
		}, {
			busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS,
			databaseLabel: params.pathname,
			operationLabel: "media-persistence-retirement"
		});
		ensureOpenClawAgentDatabaseSchema(database, {
			agentId: params.agentId,
			path: params.pathname
		});
		refreshAgentDatabasePlannerStatistics(database);
		return {
			...rewritten,
			initialVersion,
			finalVersion: readSqliteUserVersion(database)
		};
	} finally {
		clearNodeSqliteKyselyCacheForDatabase(database);
		database.close();
	}
}
function readArchiveSourceSnapshot(filePath) {
	const stat = fs.lstatSync(filePath);
	if (!stat.isFile() || stat.isSymbolicLink()) throw new Error(`${filePath} is not a regular archive file`);
	const bytes = fs.readFileSync(filePath);
	return {
		dev: stat.dev,
		ino: stat.ino,
		mtimeMs: stat.mtimeMs,
		sha256: createHash("sha256").update(bytes).digest("hex"),
		size: stat.size
	};
}
function archiveSourceMatches(filePath, expected) {
	try {
		const current = readArchiveSourceSnapshot(filePath);
		return current.dev === expected.dev && current.ino === expected.ino && current.mtimeMs === expected.mtimeMs && current.sha256 === expected.sha256 && current.size === expected.size;
	} catch {
		return false;
	}
}
function parseArchiveContent(content, filePath) {
	if (content === "") return [];
	return (content.endsWith("\n") ? content.slice(0, -1).split("\n") : content.split("\n")).map((line, index) => {
		if (!line) throw new Error(`${filePath} contains a blank JSONL record at line ${index + 1}`);
		return parseTranscriptEvent(line, `${filePath}:${index + 1}`);
	});
}
function serializeArchiveEvents(events, trailingNewline) {
	if (events.length === 0) return "";
	return `${events.map((event) => JSON.stringify(event)).join("\n")}${trailingNewline ? "\n" : ""}`;
}
function migrateTranscriptArchive(filePath, options = {}) {
	const source = readArchiveSourceSnapshot(filePath);
	const content = readSessionArchiveContentSync(filePath);
	let nulTailStart = content.length;
	while (nulTailStart > 0 && content.charCodeAt(nulTailStart - 1) === 0) nulTailStart -= 1;
	const hasTerminalNulSuffix = nulTailStart < content.length;
	if (hasTerminalNulSuffix && nulTailStart === 0) throw new Error(`${filePath} contains no JSONL records before its terminal NUL suffix`);
	const recoveredContent = hasTerminalNulSuffix ? content.slice(0, nulTailStart) : content;
	const events = parseArchiveContent(recoveredContent, filePath);
	let mediaChanged = false;
	const transformed = events.map((event) => {
		const result = transformTranscriptEvent(event);
		mediaChanged ||= result.changed;
		return result.event;
	});
	if (!hasTerminalNulSuffix && !mediaChanged) return false;
	assertEventIdentitiesUnchanged(events, transformed, filePath);
	const rewritten = mediaChanged ? serializeArchiveEvents(transformed, recoveredContent.endsWith("\n")) : recoveredContent;
	const compressed = filePath.endsWith(SESSION_ARCHIVE_ZSTD_SUFFIX);
	const encoded = compressed ? encodeSessionArchiveContent(rewritten) : {
		bytes: Buffer.from(rewritten, "utf8"),
		suffix: ""
	};
	if (compressed && encoded.suffix !== ".zst") throw new Error(`${filePath} could not be re-encoded with its zstd codec`);
	options.beforeReplace?.();
	replaceFileAtomicSync({
		filePath,
		content: encoded.bytes,
		preserveExistingMode: true,
		syncParentDir: true,
		syncTempFile: true,
		tempPrefix: `${path.basename(filePath)}${ARCHIVE_TEMP_MARKER}`,
		beforeRename: ({ tempPath }) => {
			if (!archiveSourceMatches(filePath, source)) throw new Error(`${filePath} changed before atomic media migration replacement`);
			const staged = decodeSessionArchiveBytes(fs.readFileSync(tempPath), compressed);
			if (staged !== rewritten) throw new Error(`${filePath} failed codec readback before replacement`);
			assertEventIdentitiesUnchanged(events, parseArchiveContent(staged, tempPath), filePath);
		}
	});
	if (readSessionArchiveContentSync(filePath) !== rewritten) throw new Error(`${filePath} failed codec readback after replacement`);
	return true;
}
/** Doctor-only migration from top-level Media* transcript fields to canonical facts. */
async function migrateLegacyMediaPersistence(params = {}) {
	const env = params.env ?? process.env;
	const changes = [];
	const warnings = [];
	try {
		await withAgentDatabaseMaintenanceLease({ env }, async () => {
			const targets = resolveAgentDatabaseMigrationTargets({
				changes,
				configuredAgentDatabaseTargets: params.configuredAgentDatabaseTargets ?? [],
				env,
				warnings
			});
			const seenPaths = /* @__PURE__ */ new Set();
			let databaseMigrationFailed = false;
			const archiveDirectories = /* @__PURE__ */ new Set();
			for (const entry of targets) {
				const pathname = entry.path;
				archiveDirectories.add(resolveSqliteTranscriptArchiveDirectory({
					agentId: entry.agentId,
					path: pathname
				}));
				if (seenPaths.has(entry.realPath)) continue;
				seenPaths.add(entry.realPath);
				try {
					const result = migrateAgentDatabase({
						agentId: entry.agentId,
						beforeTransaction: params.hooks?.beforeDatabaseTransaction ? () => params.hooks?.beforeDatabaseTransaction?.(pathname) : void 0,
						pathname
					});
					const schemaAdvanced = result.finalVersion > result.initialVersion;
					if (entry.source !== "registry" || schemaAdvanced) registerOpenClawAgentDatabase({
						agentId: entry.agentId,
						env,
						path: pathname
					});
					if (schemaAdvanced) changes.push(`Upgraded agent database schema in ${pathname}: v${result.initialVersion} -> v${result.finalVersion}.`);
					if (result.rewrittenSessions > 0 || result.rewrittenTrajectoryRows > 0) changes.push(`Migrated media persistence in ${pathname}: ${result.rewrittenSessions} transcript session(s), ${result.rewrittenTrajectoryRows} trajectory row(s), schema v19.`);
				} catch (error) {
					databaseMigrationFailed = true;
					warnings.push(`Skipped agent database migration for ${pathname}: ${String(error)}`);
				}
			}
			if (!databaseMigrationFailed && seenPaths.size > 0) {
				const repairedFailures = repairGatewayAgentMediaMigrationStartupFailures({
					databasePaths: [...seenPaths],
					env
				});
				if (repairedFailures > 0) changes.push(`Repaired ${repairedFailures} gateway startup failure ${repairedFailures === 1 ? "record" : "records"} after media migration.`);
			}
			for (const directory of archiveDirectories) {
				let archives;
				try {
					archives = listTranscriptArchives(directory);
				} catch (error) {
					warnings.push(`Could not enumerate transcript archives in ${directory}: ${String(error)}`);
					continue;
				}
				for (const archive of archives) try {
					if (migrateTranscriptArchive(archive, { beforeReplace: params.hooks?.beforeArchiveReplace ? () => params.hooks?.beforeArchiveReplace?.(archive) : void 0 })) changes.push(`Migrated archived transcript media in ${archive}.`);
				} catch (error) {
					warnings.push(`Skipped archived transcript media migration for ${archive}: ${String(error)}`);
				}
			}
		});
	} catch (error) {
		warnings.push(`Agent database maintenance deferred: ${formatErrorMessage(error)}`);
	}
	return {
		changes,
		warnings
	};
}
//#endregion
export { migrateLegacyMediaPersistence as t };