openclaw/dist/openclaw-state-lease-Ciyr2uVu.js

Multi-channel AI gateway with extensible messaging integrations

import { n as MAX_TIMER_TIMEOUT_MS } from "./number-coercion-CLj0HTDM.js";
import { n as loggingState } from "./state-UxwJeH7V.js";
import { n as computeBackoff, s as sleepWithAbort } from "./src-BQ327IOM.js";
import { c as isSqliteLockError, p as runWithSqliteBusyTimeout } from "./node-sqlite-BpQX3W0e.js";
import { t as runtimeProcessEntrypoints } from "./runtime-process-entrypoints-CEMkwAJ_.js";
import { n as resolveRuntimeWorkerArgv, r as resolveRuntimeWorkerUrl } from "./runtime-worker-url-CpdriB1D.js";
import { i as openOpenClawStateDatabase, s as runOpenClawStateWriteTransaction } from "./openclaw-state-db-BRTnL-D8.js";
import { t as createDeferredCore } from "./deferred-D0La5CRk.js";
import "./backoff-BkMI1WEL.js";
import { a as leaseHeartbeatState, i as renewOpenClawStateLeaseInTransaction, n as readOpenClawStateLeaseExpiry, r as releaseOpenClawStateLeaseInTransaction, t as acquireOpenClawStateLeaseInTransaction } from "./openclaw-state-lease-store-jGIk9tQ0.js";
import { randomUUID } from "node:crypto";
import { Worker } from "node:worker_threads";
//#region src/state/openclaw-state-lease-heartbeat.ts
const WORKER_START_TIMEOUT_MS = 5e3;
const WORKER_RESPONSE_TIMEOUT_MS = 1e3;
function startOpenClawStateLeaseHeartbeat(params) {
	const shared = new BigInt64Array(new SharedArrayBuffer(3 * BigInt64Array.BYTES_PER_ELEMENT));
	const url = resolveRuntimeWorkerUrl(runtimeProcessEntrypoints.stateLeaseHeartbeat);
	const worker = new Worker(url, {
		workerData: {
			path: params.path,
			identity: {
				scope: params.identity.scope,
				key: params.identity.key,
				owner: params.identity.owner
			},
			leaseMs: params.leaseMs,
			heartbeatMs: params.heartbeatMs,
			shared: shared.buffer
		},
		env: {},
		execArgv: resolveRuntimeWorkerArgv(url).slice(0, -1),
		stdout: true,
		stderr: true
	});
	worker.stdout.resume();
	worker.stderr.resume();
	const ready = createDeferredCore();
	const fail = (error) => {
		if (Atomics.load(shared, leaseHeartbeatState.status) === leaseHeartbeatState.closed) return;
		Atomics.store(shared, leaseHeartbeatState.status, leaseHeartbeatState.lost);
		Atomics.notify(shared, leaseHeartbeatState.ack);
		clearTimeout(startTimer);
		ready.reject(error);
		params.onLost(error);
	};
	const settleStartup = () => {
		clearTimeout(startTimer);
		if (Atomics.compareExchange(shared, leaseHeartbeatState.status, leaseHeartbeatState.starting, leaseHeartbeatState.lost) === leaseHeartbeatState.ready) ready.resolve();
		else fail(/* @__PURE__ */ new Error("state lease heartbeat did not become ready"));
	};
	const startTimer = setTimeout(settleStartup, Math.max(1, Math.min(WORKER_START_TIMEOUT_MS, params.expiresAt - Date.now())));
	worker.once("error", fail);
	worker.once("exit", () => fail(/* @__PURE__ */ new Error("state lease heartbeat exited")));
	worker.once("message", settleStartup);
	let stopping;
	const close = () => {
		Atomics.store(shared, leaseHeartbeatState.status, leaseHeartbeatState.closed);
		Atomics.notify(shared, leaseHeartbeatState.ack);
		clearTimeout(startTimer);
		ready.reject(/* @__PURE__ */ new Error("state lease heartbeat closed"));
	};
	return {
		ready: ready.promise,
		close,
		stop() {
			close();
			return stopping ??= worker.terminate();
		},
		assertResponsive(expiresAt) {
			const deadline = performance.now() + Math.min(WORKER_RESPONSE_TIMEOUT_MS, expiresAt - Date.now());
			const request = Atomics.add(shared, leaseHeartbeatState.request, 1n) + 1n;
			worker.postMessage(null, []);
			while (Atomics.load(shared, leaseHeartbeatState.status) === leaseHeartbeatState.ready) {
				const ack = Atomics.load(shared, leaseHeartbeatState.ack);
				if (ack === request && Atomics.load(shared, leaseHeartbeatState.status) === leaseHeartbeatState.ready) return;
				const remainingMs = deadline - performance.now();
				if (remainingMs <= 0) break;
				Atomics.wait(shared, leaseHeartbeatState.ack, ack, remainingMs);
			}
			const error = /* @__PURE__ */ new Error("state lease heartbeat is not responsive");
			fail(error);
			throw error;
		}
	};
}
//#endregion
//#region src/state/openclaw-state-lease.ts
var OpenClawStateLeaseError = class extends Error {
	constructor(message, options) {
		super(message, { cause: options.cause });
		this.name = "OpenClawStateLeaseError";
		this.code = options.code;
	}
};
const ACQUIRE_BACKOFF = {
	initialMs: 25,
	maxMs: 250,
	factor: 1.5,
	jitter: .25
};
const MIN_LEASE_MS = 1e3;
const LEASE_DB_BUSY_TIMEOUT_MS = 0;
const RELEASE_RETRY_TIMEOUT_MS = 2e3;
const processExitLeaseCleanups = /* @__PURE__ */ new Set();
let processExitListenerInstalled = false;
function runProcessExitLeaseCleanups() {
	processExitListenerInstalled = false;
	const previousForceConsoleToStderr = loggingState.forceConsoleToStderr;
	loggingState.forceConsoleToStderr = true;
	try {
		for (const cleanup of processExitLeaseCleanups) try {
			cleanup();
		} catch {}
		processExitLeaseCleanups.clear();
	} finally {
		loggingState.forceConsoleToStderr = previousForceConsoleToStderr;
	}
}
function registerProcessExitLeaseCleanup(cleanup) {
	processExitLeaseCleanups.add(cleanup);
	if (!processExitListenerInstalled) {
		process.once("exit", runProcessExitLeaseCleanups);
		processExitListenerInstalled = true;
	}
	return () => {
		processExitLeaseCleanups.delete(cleanup);
		if (processExitLeaseCleanups.size === 0 && processExitListenerInstalled) {
			process.removeListener("exit", runProcessExitLeaseCleanups);
			processExitListenerInstalled = false;
		}
	};
}
function leaseError(code, message, cause) {
	return new OpenClawStateLeaseError(message, {
		code,
		...cause === void 0 ? {} : { cause }
	});
}
function invalidInput(message) {
	return leaseError("OPENCLAW_STATE_LEASE_INVALID_INPUT", message);
}
function validateDuration(value, label, minimum, maximum) {
	if (!Number.isInteger(value) || value < minimum || value > maximum) throw invalidInput(`${label} must be an integer between ${minimum} and ${maximum}`);
	return value;
}
function validateNonEmptyString(value, label) {
	if (typeof value !== "string" || !value.trim() || value.includes("\0")) throw invalidInput(`${label} must be a non-empty string without NUL bytes`);
	return value;
}
function validateOptions(options) {
	if (typeof options !== "object" || options === null || Array.isArray(options)) throw invalidInput("state lease options must be an object");
	if (options.signal !== void 0 && !(options.signal instanceof AbortSignal)) throw invalidInput("state lease signal must be an AbortSignal");
	const database = options.database;
	if (typeof database !== "object" || database === null || Array.isArray(database)) throw invalidInput("state lease database must be an object");
	if (database.scope !== "shared") throw invalidInput("state lease database scope must be shared");
	const leaseLabel = options.leaseLabel === void 0 ? "state lease" : validateNonEmptyString(options.leaseLabel, "state lease label");
	const operationLabel = options.operationLabel === void 0 ? "state.lease" : validateNonEmptyString(options.operationLabel, "state lease operationLabel");
	return {
		scope: validateNonEmptyString(options.scope, `${leaseLabel} scope`),
		key: validateNonEmptyString(options.key, `${leaseLabel} key`),
		database,
		leaseMs: validateDuration(options.leaseMs, `${leaseLabel} leaseMs`, MIN_LEASE_MS, MAX_TIMER_TIMEOUT_MS),
		waitMs: validateDuration(options.waitMs, `${leaseLabel} waitMs`, 0, MAX_TIMER_TIMEOUT_MS),
		signal: options.signal,
		heartbeat: options.heartbeat,
		leaseLabel,
		operationLabel
	};
}
function withLeaseWriteTransaction(database, operationLabel, operation, busyTimeoutMs = LEASE_DB_BUSY_TIMEOUT_MS) {
	const stateDatabase = openOpenClawStateDatabase(database.options);
	const run = () => runOpenClawStateWriteTransaction(({ db }) => operation(db), {
		...database.options,
		database: stateDatabase
	}, {
		operationLabel,
		busyTimeoutMs
	});
	return runWithSqliteBusyTimeout(stateDatabase.db, busyTimeoutMs, run);
}
function tryAcquire(params) {
	return withLeaseWriteTransaction(params.database, params.operationLabel, (db) => acquireOpenClawStateLeaseInTransaction(db, params, params.leaseMs));
}
function renew(params) {
	return withLeaseWriteTransaction(params.database, params.operationLabel, (db) => {
		const expiresAt = renewOpenClawStateLeaseInTransaction(db, params, params.leaseMs);
		if (expiresAt === void 0) throw leaseError("OPENCLAW_STATE_LEASE_LOST", `${params.leaseLabel} ${params.scope}/${params.key} was lost`);
		return expiresAt;
	});
}
function assertLeaseOwnedInDatabase(database, params) {
	const expiresAt = readOpenClawStateLeaseExpiry(database, params);
	if (expiresAt === void 0) throw leaseError("OPENCLAW_STATE_LEASE_LOST", `${params.leaseLabel} ${params.scope}/${params.key} was lost`);
	return expiresAt;
}
function verifyLeaseOwnership(params) {
	try {
		if (params.transaction) return assertLeaseOwnedInDatabase(params.transaction, params);
		if (!params.database) throw new Error("state lease ownership check requires a database");
		return assertLeaseOwnedInDatabase(openOpenClawStateDatabase(params.database.options).db, params);
	} catch (error) {
		if (error instanceof OpenClawStateLeaseError) throw error;
		throw leaseError("OPENCLAW_STATE_LEASE_STORAGE_FAILED", `failed to verify ${params.leaseLabel} ${params.scope}/${params.key}`, error);
	}
}
function release(params) {
	withLeaseWriteTransaction(params.database, params.operationLabel, (db) => releaseOpenClawStateLeaseInTransaction(db, params));
}
async function releaseBestEffort(params) {
	const deadline = performance.now() + RELEASE_RETRY_TIMEOUT_MS;
	let attempt = 0;
	while (true) try {
		release(params);
		return;
	} catch (error) {
		const now = performance.now();
		if (!isSqliteLockError(error) || now >= deadline) return;
		attempt += 1;
		await sleepWithAbort(Math.min(deadline - now, computeBackoff(ACQUIRE_BACKOFF, attempt)));
	}
}
function abortError(signal, label, leaseLabel) {
	return leaseError("OPENCLAW_STATE_LEASE_ABORTED", `${leaseLabel} ${label} was aborted`, signal.reason);
}
/** Run one trusted operation under a host-owned SQLite lease. */
async function withOpenClawStateLease(options, run) {
	const validated = validateOptions(options);
	if (validated.signal?.aborted) throw abortError(validated.signal, "acquisition", validated.leaseLabel);
	const owner = randomUUID();
	const deadline = performance.now() + validated.waitMs;
	let attempt = 0;
	let confirmedExpiresAt;
	while (confirmedExpiresAt === void 0) {
		if (validated.signal?.aborted) throw abortError(validated.signal, "acquisition", validated.leaseLabel);
		try {
			confirmedExpiresAt = tryAcquire({
				database: validated.database,
				operationLabel: validated.operationLabel,
				scope: validated.scope,
				key: validated.key,
				owner,
				leaseMs: validated.leaseMs,
				leaseLabel: validated.leaseLabel
			});
		} catch (error) {
			if (error instanceof OpenClawStateLeaseError) throw error;
			if (!isSqliteLockError(error)) throw leaseError("OPENCLAW_STATE_LEASE_STORAGE_FAILED", `failed to acquire ${validated.leaseLabel} ${validated.scope}/${validated.key}`, error);
		}
		const now = performance.now();
		if (confirmedExpiresAt !== void 0) {
			if (validated.signal?.aborted || validated.waitMs > 0 && now >= deadline) {
				await releaseBestEffort({
					database: validated.database,
					operationLabel: validated.operationLabel,
					scope: validated.scope,
					key: validated.key,
					owner,
					leaseLabel: validated.leaseLabel
				});
				if (validated.signal?.aborted) throw abortError(validated.signal, "acquisition", validated.leaseLabel);
				throw leaseError("OPENCLAW_STATE_LEASE_TIMEOUT", `timed out waiting for ${validated.leaseLabel} ${validated.scope}/${validated.key}`);
			}
			break;
		}
		if (now >= deadline) throw leaseError("OPENCLAW_STATE_LEASE_TIMEOUT", `timed out waiting for ${validated.leaseLabel} ${validated.scope}/${validated.key}`);
		attempt += 1;
		const delayMs = Math.min(deadline - now, computeBackoff(ACQUIRE_BACKOFF, attempt));
		try {
			await sleepWithAbort(delayMs, validated.signal);
		} catch (error) {
			if (validated.signal?.aborted) throw abortError(validated.signal, "acquisition", validated.leaseLabel);
			throw error;
		}
	}
	const identity = {
		scope: validated.scope,
		key: validated.key,
		owner,
		leaseLabel: validated.leaseLabel
	};
	let closed = false;
	let workerHeartbeat;
	const unregisterProcessExitCleanup = registerProcessExitLeaseCleanup(() => {
		closed = true;
		workerHeartbeat?.close();
		release({
			...identity,
			database: validated.database,
			operationLabel: validated.operationLabel
		});
	});
	const leaseLost = new AbortController();
	const operationSignal = validated.signal ? AbortSignal.any([validated.signal, leaseLost.signal]) : leaseLost.signal;
	const heartbeatMs = Math.max(250, Math.min(3e4, Math.floor(validated.leaseMs / 3)));
	let expiryTimer;
	let heartbeat;
	const abortLost = (cause) => {
		if (!leaseLost.signal.aborted) leaseLost.abort(cause instanceof OpenClawStateLeaseError ? cause : leaseError("OPENCLAW_STATE_LEASE_LOST", `${validated.leaseLabel} ${validated.scope}/${validated.key} was lost`, cause));
	};
	const scheduleExpiry = () => {
		if (expiryTimer) clearTimeout(expiryTimer);
		expiryTimer = setTimeout(() => abortLost(), Math.max(1, (confirmedExpiresAt ?? Date.now()) - Date.now()));
		expiryTimer.unref?.();
	};
	const renewAndSchedule = () => {
		confirmedExpiresAt = renew({
			...identity,
			database: validated.database,
			operationLabel: validated.operationLabel,
			leaseMs: validated.leaseMs
		});
		scheduleExpiry();
	};
	const renewOperation = () => {
		assertActive();
		if (workerHeartbeat) assertOperationOwned();
		else renewAndSchedule();
	};
	const renewFromTimer = () => {
		try {
			renewAndSchedule();
		} catch (error) {
			if (error instanceof OpenClawStateLeaseError && error.code === "OPENCLAW_STATE_LEASE_LOST") abortLost(error);
			else if (confirmedExpiresAt !== void 0 && Date.now() >= confirmedExpiresAt) abortLost(error);
		}
	};
	const assertActive = () => {
		if (leaseLost.signal.aborted) throw leaseLost.signal.reason;
		if (validated.signal?.aborted) throw abortError(validated.signal, "operation", validated.leaseLabel);
		if (closed) {
			abortLost();
			throw leaseLost.signal.reason;
		}
	};
	const assertOperationOwned = (transaction) => {
		assertActive();
		const params = {
			...identity,
			database: validated.database,
			transaction
		};
		const expiresAt = verifyLeaseOwnership(params);
		if (workerHeartbeat) {
			try {
				workerHeartbeat.assertResponsive(expiresAt);
			} catch (error) {
				abortLost(error);
				throw leaseLost.signal.reason;
			}
			assertActive();
			verifyLeaseOwnership(params);
		}
	};
	const stopWorker = () => {
		workerHeartbeat?.stop();
	};
	try {
		let result;
		try {
			if (validated.heartbeat === "worker") {
				workerHeartbeat = startOpenClawStateLeaseHeartbeat({
					path: openOpenClawStateDatabase(validated.database.options).path,
					identity,
					leaseMs: validated.leaseMs,
					heartbeatMs,
					expiresAt: confirmedExpiresAt,
					onLost: abortLost
				});
				validated.signal?.addEventListener("abort", stopWorker, { once: true });
				if (validated.signal?.aborted) stopWorker();
				await workerHeartbeat.ready;
			} else {
				scheduleExpiry();
				heartbeat = setInterval(renewFromTimer, heartbeatMs);
				heartbeat.unref?.();
			}
			assertOperationOwned();
			result = await run({
				signal: operationSignal,
				renew: renewOperation,
				assertOwned: assertOperationOwned,
				assertOwnedInTransaction: assertOperationOwned
			});
		} catch (error) {
			if (leaseLost.signal.aborted) throw leaseLost.signal.reason;
			if (validated.signal?.aborted) throw abortError(validated.signal, "operation", validated.leaseLabel);
			throw error;
		}
		assertOperationOwned();
		return result;
	} finally {
		closed = true;
		unregisterProcessExitCleanup();
		validated.signal?.removeEventListener("abort", stopWorker);
		clearInterval(heartbeat);
		if (expiryTimer) clearTimeout(expiryTimer);
		await workerHeartbeat?.stop();
		await releaseBestEffort({
			...identity,
			database: validated.database,
			operationLabel: validated.operationLabel
		});
	}
}
//#endregion
export { withOpenClawStateLease as n, OpenClawStateLeaseError as t };