openclaw/dist/session-transcript-reconcile-D3LiCrAp.js

Multi-channel AI gateway with extensible messaging integrations

import { u as toStringifiedError } from "./error-coercion-D_-xJ90S.js";
import "./path-guards-Cp-mGr3-.js";
import { n as ok, t as err } from "./result-BQGgYouL.js";
import { w as resolveStateDir } from "./paths-D2sRr1a_.js";
import { t as createSubsystemLogger } from "./subsystem-Dy2tqXOS.js";
import { l as runSqliteDeferredTransactionSync } from "./node-sqlite-BpQX3W0e.js";
import { i as executeSqliteQueryTakeFirstSync } from "./kysely-sync-COmh4HWh.js";
import { t as runtimeProcessEntrypoints } from "./runtime-process-entrypoints-CEMkwAJ_.js";
import { r as resolveRuntimeWorkerUrl } from "./runtime-worker-url-CpdriB1D.js";
import { o as isGatewayExternallySupervised } from "./gateway-supervision-D7p37rG2.js";
import { D as isIncognitoOpenClawAgentSqlitePath, k as resolveOpenClawAgentSqlitePath } from "./openclaw-agent-db-lease-Djvd6LWN.js";
import { c as getOpenClawAgentDatabaseIfOpen, g as runOpenClawAgentWriteTransaction, l as isIncognitoOpenClawAgentDatabase, n as borrowOpenClawAgentDatabase, v as withOpenClawAgentDatabaseAsync } from "./openclaw-agent-db-CWtDoRbC.js";
import { n as withOpenClawAgentDatabaseReadOnly } from "./openclaw-agent-db-readonly-CHjf8FxN.js";
import { d as resolveSqliteTranscriptReadScope, i as getSessionKysely, m as toDatabaseOptions, p as runExclusiveSqliteSessionWrite } from "./session-accessor.sqlite-scope-2KfMzb44.js";
import { d as appendPreparedSessionTranscriptProjectionChunkInTransaction, f as claimPreparedSessionTranscriptProjectionInTransaction, h as finalizePreparedSessionTranscriptProjectionInTransaction, i as deleteOrphanedTranscriptIndexRowsInTransaction, l as sessionTranscriptIndexNeedsReconcile, o as listSessionsNeedingTranscriptIndexReconcile, p as deletePreparedSessionTranscriptProjectionChunkInTransaction } from "./session-transcript-index-DmCW_GtW.js";
import { randomInt, randomUUID } from "node:crypto";
import { sql } from "kysely";
import { Worker } from "node:worker_threads";
import { setTimeout } from "node:timers/promises";
//#region src/config/sessions/session-transcript-reconcile-memory.ts
const SOURCE_FRAME_BYTES = 262144;
function readSnapshot(database, sessionId) {
	const db = getSessionKysely(database.db);
	const row = executeSqliteQueryTakeFirstSync(database.db, db.selectFrom("session_windows as window").leftJoin("transcript_rewrite_watermarks as rewrite", "rewrite.session_id", "window.session_id").select((eb) => [
		"window.transcript_updated_at",
		"rewrite.generation",
		eb.selectFrom("transcript_events").select("seq").where("session_id", "=", sessionId).orderBy("seq", "desc").limit(1).as("max_seq")
	]).where("window.session_id", "=", sessionId));
	return row && row.max_seq !== null ? {
		sessionId,
		transcriptUpdatedAt: row.transcript_updated_at,
		maxSeq: row.max_seq,
		generation: row.generation
	} : void 0;
}
/** The parent alone owns memory state; a worker receives bytes, never its sentinel path. */
function createMemoryTranscriptProjectionSource(database, options) {
	let snapshot;
	let afterSeq = -1;
	let offset = 0;
	let row;
	const assertCurrentOwner = () => {
		if (!database.db.isOpen || getOpenClawAgentDatabaseIfOpen(options) !== database) throw new Error("Incognito transcript database was disposed during reconciliation");
	};
	const snapshotMatches = () => {
		if (!snapshot) return false;
		const current = readSnapshot(database, snapshot.sessionId);
		return current?.generation === snapshot.generation && current?.maxSeq === snapshot.maxSeq && current?.transcriptUpdatedAt === snapshot.transcriptUpdatedAt;
	};
	return {
		assertCurrentOwner,
		clear() {
			snapshot = void 0;
			row = void 0;
		},
		isCurrentPlan(plan) {
			return snapshot?.sessionId === plan.sessionId && snapshot.maxSeq === plan.sourceIndexedSeq && snapshot.transcriptUpdatedAt === plan.sourceTranscriptUpdatedAt && snapshotMatches();
		},
		read(sessionId) {
			assertCurrentOwner();
			return runSqliteDeferredTransactionSync(database.db, () => {
				if (snapshot?.sessionId !== sessionId) {
					snapshot = readSnapshot(database, sessionId);
					row = void 0;
					afterSeq = -1;
					offset = 0;
				}
				if (!snapshotMatches()) {
					row = void 0;
					snapshot = void 0;
					return { type: "source-unavailable" };
				}
				if (!row) {
					row = executeSqliteQueryTakeFirstSync(database.db, getSessionKysely(database.db).selectFrom("transcript_events").select([
						"seq",
						"created_at",
						sql`CAST(event_json AS BLOB)`.as("bytes")
					]).where("session_id", "=", sessionId).where("seq", ">", afterSeq).orderBy("seq", "asc").limit(1));
					offset = 0;
				}
				if (!row) return {
					type: "source-end",
					snapshot
				};
				const bytes = Uint8Array.from(row.bytes.subarray(offset, offset + SOURCE_FRAME_BYTES));
				offset += bytes.byteLength;
				const frame = {
					type: "source-frame",
					seq: row.seq,
					createdAt: row.created_at,
					bytes,
					final: offset === row.bytes.byteLength
				};
				if (frame.final) {
					afterSeq = row.seq;
					row = void 0;
				}
				return frame;
			}, {
				databaseLabel: database.path,
				operationLabel: "sessions.transcript-index.memory-source"
			});
		}
	};
}
//#endregion
//#region src/config/sessions/session-transcript-reconcile.ts
const log = createSubsystemLogger("sessions/transcript-index");
const PROJECTION_WRITE_CHUNK_ROWS = 512;
const PROJECTION_READY_POLL_MS = 10;
const runningReconciles = /* @__PURE__ */ new Map();
function prepareReconcileParams(params) {
	return {
		...params,
		env: { ...params.env ?? process.env }
	};
}
function reconcileKey(params) {
	return resolveOpenClawAgentSqlitePath(params);
}
function captureMemorySource(params) {
	const database = getOpenClawAgentDatabaseIfOpen(params);
	return database && isIncognitoOpenClawAgentDatabase(database) ? createMemoryTranscriptProjectionSource(database, {
		...params,
		path: database.path
	}) : void 0;
}
function yieldToGateway() {
	return new Promise((resolve) => {
		setImmediate(resolve);
	});
}
function nextProjectionClaimId() {
	return -randomInt(1, 2 ** 47);
}
function continueProjectionWorker(worker, accepted) {
	worker.postMessage({
		accepted,
		type: "continue"
	}, []);
}
function observeWorkerLeaseRelease(worker) {
	let released = false;
	let releaseFailed = false;
	let failure;
	worker.on("message", (message) => {
		if (message.type === "lease-released") released = true;
		else if (message.type === "lease-release-failed") {
			releaseFailed = true;
			failure = new Error(message.error);
		}
	});
	worker.once("error", (error) => {
		failure ??= toStringifiedError(error);
	});
	return new Promise((resolve) => {
		worker.once("exit", (code) => {
			if (code !== 0 || !released) failure ??= /* @__PURE__ */ new Error(`transcript worker exited with code ${code} before lease release`);
			resolve({
				released,
				releaseFailed,
				failure
			});
		});
	});
}
async function runProjectionWrite(databaseOptions, operationLabel, operation, memorySource) {
	return await runExclusiveSqliteSessionWrite(databaseOptions, async () => {
		const write = () => {
			memorySource?.assertCurrentOwner();
			return runOpenClawAgentWriteTransaction(operation, databaseOptions, { operationLabel });
		};
		return !isIncognitoOpenClawAgentSqlitePath(databaseOptions.path, databaseOptions) && !getOpenClawAgentDatabaseIfOpen(databaseOptions) ? withOpenClawAgentDatabaseAsync(databaseOptions, write) : write();
	});
}
async function claimPreparedSessionTranscriptProjection(databaseOptions, plan, memorySource) {
	const claimId = nextProjectionClaimId();
	if (!await runProjectionWrite(databaseOptions, "sessions.transcript-index.claim", (database) => (!memorySource || memorySource.isCurrentPlan(plan)) && claimPreparedSessionTranscriptProjectionInTransaction(database.db, plan, claimId), memorySource)) return;
	let deleteResult = {
		hasMore: true,
		owned: true
	};
	while (deleteResult.hasMore && deleteResult.owned) {
		deleteResult = await runProjectionWrite(databaseOptions, "sessions.transcript-index.delete-chunk", (database) => deletePreparedSessionTranscriptProjectionChunkInTransaction(database.db, {
			maxRowsPerTable: PROJECTION_WRITE_CHUNK_ROWS,
			sessionId: plan.sessionId,
			claimId
		}), memorySource);
		await yieldToGateway();
	}
	if (!deleteResult.owned) return;
	return {
		claimId,
		plan
	};
}
function decodeFtsChunk(chunk) {
	const decoder = new TextDecoder();
	return chunk.rows.map((row) => ({
		messageId: row.messageId,
		role: row.role,
		text: decoder.decode(chunk.textBytes.subarray(row.textByteOffset, row.textByteOffset + row.textByteLength)),
		timestamp: row.timestamp
	}));
}
async function appendPreparedProjectionChunk(databaseOptions, active, rows, memorySource) {
	const owned = await runProjectionWrite(databaseOptions, "activeRows" in rows ? "sessions.transcript-index.active-chunk" : "sessions.transcript-index.fts-chunk", (database) => appendPreparedSessionTranscriptProjectionChunkInTransaction(database.db, {
		...rows,
		claimId: active.claimId,
		sessionId: active.plan.sessionId
	}), memorySource);
	await yieldToGateway();
	return owned;
}
async function finalizePreparedProjection(databaseOptions, active, memorySource) {
	return await runProjectionWrite(databaseOptions, "sessions.transcript-index.finalize", (database) => (!memorySource || memorySource.isCurrentPlan(active.plan)) && finalizePreparedSessionTranscriptProjectionInTransaction(database.db, active.plan, active.claimId), memorySource);
}
/** Prepares full trees off-thread, then commits bounded chunks through the runtime writer owner. */
async function reconcileSessionTranscriptIndexes(params) {
	return reconcilePreparedTranscriptIndexes(prepareReconcileParams(params));
}
async function reconcilePreparedTranscriptIndexes(params) {
	const databasePath = resolveOpenClawAgentSqlitePath(params);
	const databaseOptions = {
		agentId: params.agentId,
		env: params.env,
		path: databasePath
	};
	let releaseDatabase;
	const memorySource = captureMemorySource(databaseOptions);
	let memorySessionIds = [];
	try {
		await runProjectionWrite(databaseOptions, "sessions.transcript-index.preflight", (database) => {
			deleteOrphanedTranscriptIndexRowsInTransaction(database.db);
			const sessionIds = listSessionsNeedingTranscriptIndexReconcile(database.db);
			if (sessionIds.length > 0) {
				releaseDatabase = borrowOpenClawAgentDatabase(databaseOptions).release;
				if (memorySource) {
					const preferred = params.preferredSessionId;
					memorySessionIds = preferred && sessionIds.includes(preferred) ? [preferred, ...sessionIds.filter((sessionId) => sessionId !== preferred)] : sessionIds;
				}
			}
		}, memorySource);
		if (!releaseDatabase) return { reconciledSessions: 0 };
		const workerUrl = resolveRuntimeWorkerUrl(runtimeProcessEntrypoints.sessionTranscriptReconcile);
		const sourceWorkerExecArgv = workerUrl.pathname.endsWith(".ts") ? ["--import", "tsx"] : void 0;
		const input = memorySource ? {
			mode: "memory",
			sessionIds: memorySessionIds
		} : {
			mode: "disk",
			leaseId: randomUUID(),
			agentId: params.agentId,
			path: databasePath,
			stateDir: resolveStateDir(params.env),
			externallySupervised: isGatewayExternallySupervised(params.env),
			...params.preferredSessionId ? { preferredSessionId: params.preferredSessionId } : {}
		};
		const createWorker = params.createWorker ?? ((filename, options) => new Worker(filename, options));
		let worker;
		try {
			worker = createWorker(workerUrl, {
				workerData: input,
				execArgv: sourceWorkerExecArgv
			});
		} catch (error) {
			throw toStringifiedError(error);
		}
		const leaseRelease = observeWorkerLeaseRelease(worker);
		let handlingMessage;
		let terminalReceived = false;
		let outcome;
		try {
			const value = await new Promise((resolve, reject) => {
				let active;
				let reconciledSessions = 0;
				let settled = false;
				const settle = (finish) => {
					if (settled) return;
					settled = true;
					finish();
				};
				const handleMessage = async (message) => {
					if (message.type === "failed") {
						terminalReceived = true;
						settle(() => reject(new Error(message.error)));
						return;
					}
					if (message.type === "done") {
						terminalReceived = true;
						if (active) {
							settle(() => reject(/* @__PURE__ */ new Error("session transcript reconcile worker ended mid-plan")));
							return;
						}
						try {
							await runProjectionWrite(databaseOptions, "sessions.transcript-index.orphan-sweep", (database) => deleteOrphanedTranscriptIndexRowsInTransaction(database.db), memorySource);
						} catch (error) {
							settle(() => reject(toStringifiedError(error)));
							return;
						}
						settle(() => resolve({ reconciledSessions }));
						return;
					}
					try {
						if (message.type === "source-read") {
							if (!memorySource || !memorySessionIds.includes(message.sessionId)) throw new Error("session transcript worker requested an unavailable memory source");
							const frame = memorySource.read(message.sessionId);
							await yieldToGateway();
							worker.postMessage(frame, frame.type === "source-frame" ? [frame.bytes.buffer] : []);
							return;
						}
						if (message.type === "plan-start") {
							if (active) throw new Error("session transcript reconcile worker started overlapping plans");
							active = await claimPreparedSessionTranscriptProjection(databaseOptions, message.plan, memorySource);
							continueProjectionWorker(worker, active !== void 0);
							return;
						}
						if (!active || active.plan.sessionId !== message.sessionId) throw new Error("session transcript reconcile worker sent a chunk for no active plan");
						if (message.type === "plan-finish") {
							const finalized = await finalizePreparedProjection(databaseOptions, active, memorySource);
							active = void 0;
							if (finalized) reconciledSessions += 1;
							continueProjectionWorker(worker, finalized);
							return;
						}
						const owned = await appendPreparedProjectionChunk(databaseOptions, active, message.type === "active-chunk" ? { activeRows: message.rows } : { ftsRows: decodeFtsChunk(message.chunk) }, memorySource);
						if (!owned) active = void 0;
						continueProjectionWorker(worker, owned);
					} catch (error) {
						settle(() => reject(toStringifiedError(error)));
					}
				};
				worker.on("message", (message) => {
					if (settled || message.type === "lease-released" || message.type === "lease-release-failed") return;
					handlingMessage = handleMessage(message);
				});
				worker.once("error", (error) => {
					settle(() => reject(toStringifiedError(error)));
				});
				worker.once("exit", (code) => {
					if (input.mode === "memory" && terminalReceived && code === 0) return;
					settle(() => reject(/* @__PURE__ */ new Error(`session transcript reconcile worker exited with code ${code}`)));
				});
			});
			outcome = ok(value);
		} catch (error) {
			outcome = err(error);
		}
		let plannerFailure;
		try {
			const termination = input.mode !== "disk" || !terminalReceived ? worker.terminate() : void 0;
			await handlingMessage;
			if (input.mode === "disk" && terminalReceived && worker.threadId !== -1) worker.postMessage({ type: "release" }, []);
			else await (termination ?? worker.terminate());
			const plannerRelease = await leaseRelease;
			if (input.mode === "disk") {
				let cleanup = plannerRelease;
				if (!cleanup.released && !cleanup.releaseFailed) {
					const releaseWorker = createWorker(workerUrl, {
						workerData: {
							mode: "release",
							leaseId: input.leaseId,
							stateDir: input.stateDir,
							externallySupervised: input.externallySupervised
						},
						execArgv: sourceWorkerExecArgv
					});
					try {
						cleanup = await observeWorkerLeaseRelease(releaseWorker);
					} finally {
						await releaseWorker.terminate();
						releaseWorker.removeAllListeners();
					}
				}
				if (cleanup.failure) throw cleanup.failure;
				if (outcome.ok && plannerRelease.failure) plannerFailure = plannerRelease.failure;
			}
		} catch (error) {
			const failure = new Error(`Transcript lease cleanup incomplete; restart OpenClaw before deleting this agent: ${toStringifiedError(error).message}`, { cause: error });
			throw outcome.ok ? failure : new AggregateError([outcome.error, failure], failure.message, { cause: failure });
		} finally {
			worker.removeAllListeners();
		}
		if (!outcome.ok) throw outcome.error;
		if (plannerFailure) throw plannerFailure;
		return outcome.value;
	} finally {
		memorySource?.clear();
		releaseDatabase?.();
	}
}
/** Starts one deferred reconcile. No transcript rows are read on the caller's stack. */
function startSessionTranscriptIndexReconcile(input) {
	const params = prepareReconcileParams(input);
	const key = reconcileKey(params);
	const running = runningReconciles.get(key);
	if (running) {
		running.pending = true;
		running.preferredSessionId ??= params.preferredSessionId;
		return;
	}
	const state = {
		pending: false,
		...params.preferredSessionId ? { preferredSessionId: params.preferredSessionId } : {}
	};
	const memorySource = captureMemorySource(params);
	state.promise = yieldToGateway().then(async () => {
		let reconciledSessions = 0;
		while (true) {
			memorySource?.assertCurrentOwner();
			state.pending = false;
			const preferredSessionId = state.preferredSessionId;
			delete state.preferredSessionId;
			const result = await reconcilePreparedTranscriptIndexes({
				...params,
				...preferredSessionId ? { preferredSessionId } : {}
			});
			reconciledSessions += result.reconciledSessions;
			if (state.pending) continue;
			if (runningReconciles.get(key) === state) runningReconciles.delete(key);
			return { reconciledSessions };
		}
	}).catch(async (error) => {
		log.warn(`session transcript reconcile failed agent=${params.agentId} error=${error instanceof Error ? error.message : String(error)}`);
		const shouldHandoff = state.pending;
		const preferredSessionId = state.preferredSessionId;
		if (runningReconciles.get(key) === state) runningReconciles.delete(key);
		if (shouldHandoff && (!memorySource || captureMemorySource(params))) {
			startSessionTranscriptIndexReconcile({
				...params,
				...preferredSessionId ? { preferredSessionId } : {}
			});
			await waitForSessionTranscriptIndexReconcile(params);
		}
		return { reconciledSessions: 0 };
	});
	runningReconciles.set(key, state);
}
function isSessionTranscriptIndexReconcileRunning(params) {
	return runningReconciles.has(reconcileKey(params));
}
/** Test and maintenance wait hook for an already-scheduled reconcile. */
async function waitForSessionTranscriptIndexReconcile(params) {
	await runningReconciles.get(reconcileKey(params))?.promise;
}
/** Waits only until the requested session's scheduled projection rebuild settles. */
async function waitForSessionTranscriptProjection(scope, abortSignal) {
	const resolved = resolveSqliteTranscriptReadScope(scope);
	const databaseOptions = toDatabaseOptions(resolved);
	while (isSessionTranscriptIndexReconcileRunning(databaseOptions)) {
		const pending = withOpenClawAgentDatabaseReadOnly(({ db }) => sessionTranscriptIndexNeedsReconcile(db, resolved.sessionId), databaseOptions, { throwOnMissingTable: true });
		if (!pending.found || !pending.value) break;
		await setTimeout(PROJECTION_READY_POLL_MS, void 0, abortSignal ? { signal: abortSignal } : void 0);
	}
}
//#endregion
export { waitForSessionTranscriptProjection as a, waitForSessionTranscriptIndexReconcile as i, reconcileSessionTranscriptIndexes as n, startSessionTranscriptIndexReconcile as r, isSessionTranscriptIndexReconcileRunning as t };