openclaw/dist/config/sessions/session-accessor.sqlite-archive.worker.js

Multi-channel AI gateway with extensible messaging integrations

import { a as getNodeSqliteKysely, o as iterateSqliteQuerySync, r as executeSqliteQuerySync } from "../../kysely-sync-COmh4HWh.js";
import { w as settleOpenClawAgentDatabaseWorkerClose } from "../../openclaw-agent-db-CWtDoRbC.js";
import { n as withOpenClawAgentDatabaseReadOnly } from "../../openclaw-agent-db-readonly-CHjf8FxN.js";
import { n as sqliteSessionStateDeleteSnapshotsEqual, t as readSessionStateDeleteSnapshot } from "../../session-accessor.sqlite-delete-snapshot-Dc3XCL8u.js";
import { S as resolveSqliteTranscriptArchivePath, v as MAX_MATERIALIZED_ARCHIVE_BATCH_BYTES, x as publishEncodedSessionTranscriptArchive, y as hashSessionArchiveBytes } from "../../session-accessor.sqlite-lifecycle-state-BTh4yZ7R.js";
import { a as reclaimSqliteSessionInTransaction, l as markSqliteReclamationSettled, u as waitForSqliteReclamationCommit } from "../../session-accessor.sqlite-reclamation-CoML3XOn.js";
import fs from "node:fs";
import path from "node:path";
import { randomUUID } from "node:crypto";
import zlib from "node:zlib";
import { Transform } from "node:stream";
import { parentPort, workerData } from "node:worker_threads";
import { pipeline as pipeline$1 } from "node:stream/promises";
//#region src/config/sessions/session-accessor.sqlite-archive.worker.ts
/** Worker entrypoint for SQLite transcript archive materialization off the gateway event loop. */
const WORKER_CLOSE_MAX_ATTEMPTS = 3;
async function settleReclamationDatabase(pathname) {
	const warnings = /* @__PURE__ */ new Set();
	let outcome = {
		errors: [],
		settled: false
	};
	for (let attempt = 0; attempt < WORKER_CLOSE_MAX_ATTEMPTS; attempt += 1) {
		outcome = settleOpenClawAgentDatabaseWorkerClose(pathname);
		outcome.errors.forEach((error) => warnings.add(error.message));
		if (outcome.settled) break;
		if (attempt + 1 < WORKER_CLOSE_MAX_ATTEMPTS) await new Promise((resolve) => {
			setTimeout(resolve, 25 * 2 ** attempt);
		});
	}
	return {
		cleanupWarnings: [...warnings],
		settled: outcome.settled
	};
}
function isSqliteTranscriptArchiveWorkerData(value) {
	return Boolean(value) && typeof value === "object" && !Array.isArray(value) && value.type === "sqlite-transcript-archive-v2";
}
function parsePublishWorkerPlans(value) {
	if (!value || typeof value !== "object" || Array.isArray(value)) return;
	const plans = value.plans;
	if (!Array.isArray(plans)) return;
	const parsed = [];
	for (const planValue of plans) {
		if (!planValue || typeof planValue !== "object" || Array.isArray(planValue)) return;
		const plan = planValue;
		if (typeof plan.agentId !== "string" || typeof plan.archiveDirectory !== "string" || typeof plan.databasePath !== "string" || typeof plan.generation !== "string" || typeof plan.sessionId !== "string") return;
		parsed.push({
			agentId: plan.agentId,
			archiveDirectory: plan.archiveDirectory,
			databasePath: plan.databasePath,
			generation: plan.generation,
			sessionId: plan.sessionId
		});
	}
	return parsed;
}
function parseSessionStateDeleteSnapshot(value) {
	if (!value || typeof value !== "object" || Array.isArray(value)) return null;
	const snapshot = value;
	if (typeof snapshot.acpParentStreamEventCount !== "number" || snapshot.generation !== null && typeof snapshot.generation !== "string" || snapshot.lastSeq !== null && typeof snapshot.lastSeq !== "number" || snapshot.sessionKey !== null && typeof snapshot.sessionKey !== "string" || snapshot.sessionUpdatedAt !== null && typeof snapshot.sessionUpdatedAt !== "number" || snapshot.trajectoryLastSeq !== null && typeof snapshot.trajectoryLastSeq !== "number" || snapshot.transcriptUpdatedAt !== null && typeof snapshot.transcriptUpdatedAt !== "number") return null;
	return {
		acpParentStreamEventCount: snapshot.acpParentStreamEventCount,
		generation: snapshot.generation,
		lastSeq: snapshot.lastSeq,
		sessionKey: snapshot.sessionKey,
		sessionUpdatedAt: snapshot.sessionUpdatedAt,
		trajectoryLastSeq: snapshot.trajectoryLastSeq,
		transcriptUpdatedAt: snapshot.transcriptUpdatedAt
	};
}
function parseWorkerPlans(value) {
	if (!value || typeof value !== "object" || Array.isArray(value)) return;
	const plans = value.plans;
	if (!Array.isArray(plans)) return;
	const parsed = [];
	for (const planValue of plans) {
		if (!planValue || typeof planValue !== "object" || Array.isArray(planValue)) return;
		const plan = planValue;
		const snapshot = parseSessionStateDeleteSnapshot(plan.snapshot);
		if (typeof plan.agentId !== "string" || typeof plan.archiveDirectory !== "string" || typeof plan.databasePath !== "string" || plan.reason !== "deleted" && plan.reason !== "reset" || typeof plan.sessionId !== "string" || !snapshot) return;
		parsed.push({
			agentId: plan.agentId,
			archiveDirectory: plan.archiveDirectory,
			databasePath: plan.databasePath,
			reason: plan.reason,
			sessionId: plan.sessionId,
			snapshot
		});
	}
	return parsed;
}
function stageTranscriptArchiveContent(database, sessionId, stagedPath) {
	const fd = fs.openSync(stagedPath, "wx", 384);
	let rowCount = 0;
	try {
		const db = getNodeSqliteKysely(database);
		for (const row of iterateSqliteQuerySync(database, db.selectFrom("transcript_events").select("event_json").where("session_id", "=", sessionId).orderBy("seq", "asc"))) {
			if (typeof row.event_json !== "string") throw new Error(`Invalid transcript event row for ${sessionId}`);
			fs.writeFileSync(fd, row.event_json);
			fs.writeFileSync(fd, "\n");
			rowCount += 1;
		}
		fs.fsyncSync(fd);
	} finally {
		fs.closeSync(fd);
	}
	return rowCount;
}
function createArchiveByteLimitTransform() {
	let encodedBytes = 0;
	return new Transform({ transform(chunk, encoding, callback) {
		let chunkBytes;
		if (typeof chunk === "string") chunkBytes = Buffer.byteLength(chunk, encoding);
		else if (chunk instanceof Uint8Array) chunkBytes = chunk.byteLength;
		else {
			callback(/* @__PURE__ */ new TypeError("Archive encoder emitted an unsupported chunk type"));
			return;
		}
		encodedBytes += chunkBytes;
		if (encodedBytes > 268435456) {
			callback(/* @__PURE__ */ new Error(`Archive exceeds ${MAX_MATERIALIZED_ARCHIVE_BATCH_BYTES} bytes during encoding`));
			return;
		}
		callback(null, chunk);
	} });
}
async function encodeStagedTranscriptArchive(params) {
	const createdAt = Date.now();
	const createZstdCompress = zlib.createZstdCompress;
	const compressed = typeof createZstdCompress === "function";
	const archivePath = `${resolveSqliteTranscriptArchivePath({
		archiveDirectory: params.archiveDirectory,
		generation: params.generation,
		identityOwner: "registry",
		reason: params.reason,
		sessionId: params.sessionId,
		nowMs: createdAt
	})}${compressed ? ".zst" : ""}`;
	const encodedPath = `${archivePath}.${randomUUID()}.stage`;
	try {
		if (compressed) await pipeline$1(fs.createReadStream(params.stagedPath), createZstdCompress.call(zlib), createArchiveByteLimitTransform(), fs.createWriteStream(encodedPath, {
			flags: "wx",
			mode: 384
		}));
		else await pipeline$1(fs.createReadStream(params.stagedPath), createArchiveByteLimitTransform(), fs.createWriteStream(encodedPath, {
			flags: "wx",
			mode: 384
		}));
		const bytes = fs.readFileSync(encodedPath);
		return {
			archiveName: path.basename(archivePath),
			bytes,
			createdAt,
			encoding: compressed ? "zstd" : "identity",
			sha256: hashSessionArchiveBytes(bytes)
		};
	} finally {
		fs.rmSync(encodedPath, { force: true });
	}
}
async function materializeTranscriptArchiveInWorker(plan) {
	fs.mkdirSync(plan.archiveDirectory, {
		recursive: true,
		mode: 448
	});
	const stagedPath = `${resolveSqliteTranscriptArchivePath({
		archiveDirectory: plan.archiveDirectory,
		generation: plan.snapshot.generation ?? void 0,
		identityOwner: "registry",
		reason: plan.reason,
		sessionId: plan.sessionId
	})}.${randomUUID()}.jsonl-stage`;
	try {
		const opened = withOpenClawAgentDatabaseReadOnly((database) => {
			let transactionOpen = false;
			try {
				database.db.exec("BEGIN");
				transactionOpen = true;
				const snapshot = readSessionStateDeleteSnapshot(database.db, plan.sessionId);
				if (!sqliteSessionStateDeleteSnapshotsEqual(snapshot, plan.snapshot)) throw new Error(`SQLite session state changed before archive materialization for ${plan.sessionId}`);
				const rowCount = stageTranscriptArchiveContent(database.db, plan.sessionId, stagedPath);
				database.db.exec("COMMIT");
				transactionOpen = false;
				return {
					rowCount,
					snapshot
				};
			} catch (error) {
				if (transactionOpen) database.db.exec("ROLLBACK");
				throw error;
			}
		}, {
			agentId: plan.agentId,
			path: plan.databasePath
		});
		if (!opened.found) throw new Error(`Cannot archive SQLite transcript ${plan.sessionId}: ${opened.reason.replaceAll("-", " ")}`);
		const generation = plan.snapshot.generation;
		if (opened.value.rowCount > 0 && !generation) throw new Error(`Cannot archive SQLite transcript without a generation for ${plan.sessionId}`);
		return {
			archive: opened.value.rowCount > 0 && generation ? await encodeStagedTranscriptArchive({
				archiveDirectory: plan.archiveDirectory,
				generation,
				reason: plan.reason,
				sessionId: plan.sessionId,
				stagedPath
			}) : null,
			sessionId: plan.sessionId
		};
	} finally {
		fs.rmSync(stagedPath, { force: true });
	}
}
function publishTranscriptArchiveInWorker(plan) {
	try {
		const opened = withOpenClawAgentDatabaseReadOnly((database) => {
			const db = getNodeSqliteKysely(database.db);
			return executeSqliteQuerySync(database.db, db.selectFrom("session_transcript_archives").select([
				"archive_blob",
				"archive_name",
				"archive_sha256"
			]).where("session_id", "=", plan.sessionId).where("generation", "=", plan.generation)).rows[0];
		}, {
			agentId: plan.agentId,
			path: plan.databasePath
		});
		if (!opened.found || !opened.value) throw new Error(`Canonical SQLite transcript archive is missing for ${plan.sessionId}`);
		if (hashSessionArchiveBytes(opened.value.archive_blob) !== opened.value.archive_sha256) throw new Error(`Canonical SQLite transcript archive is corrupt for ${plan.sessionId}`);
		return {
			archivedPath: publishEncodedSessionTranscriptArchive({
				archiveDirectory: plan.archiveDirectory,
				archiveName: opened.value.archive_name,
				bytes: opened.value.archive_blob,
				sha256: opened.value.archive_sha256
			}),
			generation: plan.generation,
			sessionId: plan.sessionId
		};
	} catch (error) {
		return {
			error: error instanceof Error ? error.message : String(error),
			generation: plan.generation,
			sessionId: plan.sessionId
		};
	}
}
async function runWorkerPort(port, plans) {
	let materializedBytes = 0;
	for (const plan of plans) {
		const result = await materializeTranscriptArchiveInWorker(plan);
		materializedBytes += result.archive?.bytes.byteLength ?? 0;
		if (materializedBytes > 268435456) throw new Error(`Archive batch exceeds ${MAX_MATERIALIZED_ARCHIVE_BATCH_BYTES} bytes; use fewer sessions`);
		port.postMessage({
			type: "done",
			results: [result]
		});
	}
	port.close();
}
function runPublishWorkerPort(port, plans) {
	const results = plans.map((plan) => publishTranscriptArchiveInWorker(plan));
	port.postMessage({
		type: "published",
		results
	});
	port.close();
}
async function runReclamationWorkerPort(port, data) {
	let result;
	const commitGate = data.commitGate;
	try {
		let transactionDatabase;
		try {
			result = reclaimSqliteSessionInTransaction(data.plan, { onCommit: commitGate ? (database) => {
				transactionDatabase = database.db;
				waitForSqliteReclamationCommit(commitGate, () => port.postMessage({ type: "commit-request" }));
			} : void 0 });
		} finally {
			if (transactionDatabase && (!transactionDatabase.isOpen || !transactionDatabase.isTransaction)) markSqliteReclamationSettled(commitGate);
		}
	} catch (error) {
		const cleanup = await settleReclamationDatabase(data.plan.databaseOptions.path);
		if (cleanup.settled) markSqliteReclamationSettled(commitGate);
		else throw new AggregateError([error, ...cleanup.cleanupWarnings.map((warning) => new Error(warning))], "SQLite session reclamation failed and Worker cleanup is incomplete; restart OpenClaw before deleting the owning agent", { cause: error });
		throw error;
	}
	const cleanup = await settleReclamationDatabase(data.plan.databaseOptions.path);
	const workerResult = {
		result,
		...cleanup.cleanupWarnings.length > 0 ? { cleanupWarnings: cleanup.cleanupWarnings } : {},
		...!cleanup.settled ? { cleanupIncomplete: true } : {}
	};
	port.postMessage({
		type: "reclaimed",
		results: [workerResult]
	});
	port.close();
}
if (isSqliteTranscriptArchiveWorkerData(workerData)) {
	if (!parentPort) throw new Error("SQLite transcript archive worker requires a parent port");
	const operation = workerData.operation;
	if (operation === "materialize") {
		const plans = parseWorkerPlans(workerData);
		if (!plans) throw new Error("SQLite transcript archive worker requires valid materialization data");
		await runWorkerPort(parentPort, plans);
	} else if (operation === "publish") {
		const plans = parsePublishWorkerPlans(workerData);
		if (!plans) throw new Error("SQLite transcript archive worker requires valid publication data");
		runPublishWorkerPort(parentPort, plans);
	} else if (operation === "reclaim") await runReclamationWorkerPort(parentPort, workerData);
	else throw new Error("SQLite transcript archive worker requires a supported operation");
}
//#endregion
export { materializeTranscriptArchiveInWorker, publishTranscriptArchiveInWorker };