openclaw
Version:
Multi-channel AI gateway with extensible messaging integrations
995 lines (994 loc) • 43.8 kB
JavaScript
import { u as toErrorObject } from "./error-coercion-CvJK4lmH.mjs";
import { t as createDeferredCore } from "./deferred-D0La5CRk.mjs";
import { h as sleep } from "./utils-DgZn2dLu.mjs";
import { r as resolveStateDir } from "./state-dir-BHJORNly.mjs";
import { o as isDefaultInstallIdentity } from "./paths-DGIwWbpu.mjs";
import { t as formatErrorMessage } from "./errors-O33D01XW.mjs";
import { t as formatCliCommand } from "./command-format-DwEdrDKL.mjs";
import { o as registerSignalExitGate } from "./signal-exit-barrier-BFniU76K.mjs";
import { a as admitOpenClawMaintenanceLiveAuthorityReads, l as getOpenClawDatabaseMaintenanceScope, n as createOpenClawDatabaseMaintenanceScope } from "./openclaw-state-db-async-lifecycle-DLKxSPCk.mjs";
import { c as resolveOpenClawStateSqlitePath } from "./openclaw-state-db.paths-DfifnSGp.mjs";
import { t as GatewayStateOwnerContentionError, u as resolveGatewayStateOwnerPath } from "./gateway-state-owner-DmHycK5o.mjs";
import { r as createSqliteReadOnlyWorkerScope } from "./sqlite-readonly-worker-TSy0m8v6.mjs";
import { n as openDoctorStateSchemaReadAdmission } from "./openclaw-state-db-doctor-schema-BlUz8kWR.mjs";
import { t as DoctorUnreadableStateDatabaseError } from "./state-repair-message-B5Bu99oR.mjs";
import { r as hasCommandProcessCleanupError } from "./exec-result-s7WOth0N.mjs";
import { l as withCommandProcessScope, n as resolveCommandProcessSignal } from "./exec-spawn-IxJJNQFo.mjs";
import { n as GatewayServiceAuthorityError } from "./service-update-authority-CoX8GMUL.mjs";
import { c as hasGatewayServiceStopUnsafeError, o as findServiceOwnershipRefusal } from "./service-inspection-error-Bsltcpi4.mjs";
import { i as readPackageVersion } from "./package-json-DuhFvY5l.mjs";
import { n as readBuiltGatewayBuildId } from "./update-git-runtime-DgMBx4xF.mjs";
import { s as closeOpenClawStateDatabaseByPathAsync } from "./openclaw-state-db-cache-BKCnj4of.mjs";
import { r as createUpdateFailureFact } from "./update-failure-facts-CetdnXr-.mjs";
import { n as recordUpdateDoctorRefusal, r as resolveUpdateDoctorGitRecovery, t as formatUpdateDoctorServiceStopRefusal } from "./doctor-update-refusal-quKzq4et.mjs";
import { c as readStateLeaseProcessOwnerStatus } from "./openclaw-state-lease-store-gySh31JU.mjs";
import { o as acquireWithWait } from "./startup-migration-checkpoint-Bkqf-Qf8.mjs";
import { a as UpdateDoctorError, d as createUpdateDoctorDatabaseWriteCapture, t as DoctorMaintenanceRefusalError } from "./update-doctor-result-b1ffujJO.mjs";
import { r as readGatewayOwnerLease } from "./gateway-owner-lease-0RgOZO_d.mjs";
import { n as GatewayLockError, o as readActiveGatewayLockIdentity, r as acquireGatewayLock } from "./gateway-lock-B6mKYs2x.mjs";
import { i as UPDATE_RUN_ID_ENV } from "./update-control-plane-sentinel-Du9790-7.mjs";
import { n as GATEWAY_SERVICE_STOP_TIMEOUT_MS, r as GATEWAY_SHUTDOWN_RESERVE_MS } from "./gateway-shutdown-budget-v-vGFkEN.mjs";
import { n as probePortUsage } from "./ports-probe-DMGtt9dg.mjs";
import { n as resolveGatewayService } from "./service-C5e3lih7.mjs";
import { t as resolveGatewayRestartDeferralTimeoutMs } from "./restart-budget-C6PCt2me.mjs";
import { p as resolveUpdatedGatewayRestartPort } from "./update-command-service-plan-nt53M07v.mjs";
import { n as sanitizeDoctorNote } from "./emit-notes-CPyL1dzc.mjs";
import { s as inspectGatewayRestart } from "./restart-health-CjTwCs9F.mjs";
import { s as TICK_INTERVAL_MS } from "./server-constants-BH_EolwD.mjs";
import { n as readLegacyGatewayLockIdentity, t as assertLegacyGatewayStoppedForMaintenance } from "./gateway-lock-legacy-j9m2CmSY.mjs";
import { c as resolveUpdateParentGatewayActivation, l as shouldManageGatewayService, n as assertDoctorServiceSelection, o as isServiceRepairExternallyManaged } from "./doctor-service-repair-policy-dEj5KU5E.mjs";
import { a as readOfflineAppOwnedDoctorInspection, i as readDoctorGatewayOwnerLease, r as classifyDoctorMaintenanceRefusal, t as assertDoctorMaintenanceInspection } from "./doctor-maintenance-inspection-wOSWNH-z.mjs";
import { t as resolveDoctorUpdateAdmission } from "./doctor-maintenance-admission-DjemgY-t.mjs";
import { n as resolveDoctorRepairMode, t as isDoctorUpdateRepairMode } from "./doctor-repair-mode-W3I4YQ3X.mjs";
import path from "node:path";
//#region src/commands/doctor-agent-lease-refusal.ts
function createDoctorAgentLeaseRefusal(env, cause) {
const message = "Doctor could not enter maintenance. An agent database is in use. Stop other OpenClaw processes using this state, then retry the update.";
return new DoctorMaintenanceRefusalError(message, {
kind: "deferred",
reason: "agent-database-in-use"
}, {
cause,
failureFacts: [createUpdateFailureFact({
check: "doctor",
code: "agent-database-lease-active",
message
}, env)]
});
}
async function preflightExternalDoctorAgentLease(env) {
const { readActiveOpenClawAgentDatabaseLeasesReadOnly } = await import("./openclaw-agent-db-lease-DPdUAF5J.mjs");
let activeAgentLease = false;
try {
activeAgentLease = readActiveOpenClawAgentDatabaseLeasesReadOnly({ env }, openDoctorStateSchemaReadAdmission).length > 0;
} catch (error) {
if (hasCommandProcessCleanupError(error)) throw error;
}
if (activeAgentLease) throw createDoctorAgentLeaseRefusal(env);
}
/** The caller holds both lifecycle coordinators; this check never opens a state writer. */
async function assertDoctorAgentLeaseAdmission(env) {
const { assertNoOpenClawAgentDatabaseLeasesReadOnly, OpenClawAgentDatabaseLeaseActiveError } = await import("./openclaw-agent-db-lease-DPdUAF5J.mjs");
try {
assertNoOpenClawAgentDatabaseLeasesReadOnly({ env }, openDoctorStateSchemaReadAdmission);
} catch (error) {
if (error instanceof OpenClawAgentDatabaseLeaseActiveError) throw createDoctorAgentLeaseRefusal(env, error);
const { preflightOpenClawDatabaseSchemas } = await import("./openclaw-database-preflight-ClnnU5e8.mjs");
const unreadable = (await preflightOpenClawDatabaseSchemas({
env,
scope: "state",
openStateSchemaReadAdmission: openDoctorStateSchemaReadAdmission
})).indeterminate.find((database) => database.kind === "state");
if (unreadable) throw new DoctorUnreadableStateDatabaseError(unreadable.path, unreadable.reason);
throw error;
}
}
//#endregion
//#region src/commands/doctor-maintenance-exit.ts
/** Let admitted repair work settle before restoring the service and exiting. */
function holdDoctorMaintenanceExit(reportInterruption) {
const controller = new AbortController();
const finished = createDeferredCore();
finished.promise.catch(() => void 0);
const unregister = registerSignalExitGate(finished.promise, (signal) => {
if (controller.signal.aborted) return;
const message = `Doctor interrupted${signal ? ` by ${signal}` : ""}; cancelling inspections and settling admitted repairs before exit.`;
controller.abort(new Error(message));
try {
reportInterruption?.(message);
} catch {}
});
let active = true;
const release = (failed = false) => {
if (!active) return;
active = false;
unregister();
if (failed) finished.reject(/* @__PURE__ */ new Error("Doctor maintenance did not complete."));
else finished.resolve();
};
return {
signal: controller.signal,
release
};
}
//#endregion
//#region src/commands/doctor-maintenance-stale-service.ts
function doctorGatewayMaintenanceError(params) {
const restart = formatCliCommand("openclaw gateway restart", params.env);
const next = `Run ${formatCliCommand("openclaw gateway status --deep", params.env)}; resolve the reported failure, then ${formatCliCommand("openclaw doctor --fix", params.env)} and ${restart}.`;
const message = `Doctor ${params.phase} failed. ${params.detail} ${next}`;
const failureFacts = [createUpdateFailureFact({
check: params.phase,
code: params.code,
message: params.detail
}, params.env), createUpdateFailureFact({
check: params.phase,
code: "stale-gateway-recovery-command",
message: next
}, params.env)];
return params.phase === "gateway-stop" ? new DoctorMaintenanceRefusalError(message, {
kind: "data-at-risk",
reason: "gateway-state-unverified"
}, {
cause: params.cause,
failureFacts
}) : new UpdateDoctorError(message, failureFacts, { cause: params.cause });
}
/** Identify predecessor code without requiring startup before offline migrations. */
async function inspectStaleDoctorGateway(params) {
const { before, root, env } = params;
const legacy = await readLegacyGatewayLockIdentity(env);
if (!before.running && !legacy) return;
const serviceEnv = before.serviceEnv ?? env;
const service = resolveGatewayService();
const [version, buildId, serviceCommand] = await Promise.all([
readPackageVersion(root),
readBuiltGatewayBuildId(root),
service.readCommand(serviceEnv).catch(() => null)
]);
params.assertCurrent?.();
if ((!version || !buildId) && !legacy) return;
const port = await resolveUpdatedGatewayRestartPort({
serviceEnv,
serviceCommand
});
const health = legacy ? void 0 : await inspectGatewayRestart({
service,
port,
env: serviceEnv,
expectedVersion: version,
expectedBuildId: buildId,
openStateSchemaReadAdmission: openDoctorStateSchemaReadAdmission,
requirePluginHealth: false
});
params.assertCurrent?.();
if (!(legacy || health?.buildIdMismatch?.actual != null || health?.versionMismatch || health?.staleConnection !== void 0)) return;
const pid = legacy?.pid ?? before.servicePid ?? health?.runtime.pid;
if (before.serviceUpdateVerdict?.kind !== "owned" || !before.serviceEnv || !before.running || legacy && (legacy.state !== "alive" || legacy.pid !== before.servicePid)) throw doctorGatewayMaintenanceError({
env,
phase: "gateway-stop",
code: "stale-gateway-service-unverified",
detail: `Gateway PID ${pid ?? "unknown"} is stale, but its managed service ownership is unverified. ${before.blockMessage ?? before.serviceMutationSkipMessage ?? ""}`
});
if (!version || !buildId) throw doctorGatewayMaintenanceError({
env,
phase: "gateway-stop",
code: "stale-gateway-identity-unavailable",
detail: "The installed candidate's build identity is unavailable."
});
return {
version,
buildId,
pid,
port
};
}
async function assertStaleDoctorGatewayStopped(params) {
const runtime = await resolveGatewayService().readRuntime(params.env);
const port = await probePortUsage(params.stale.port);
const legacy = await readLegacyGatewayLockIdentity(params.env);
params.assertCurrent?.();
if (runtime.status !== "stopped" || (runtime.pid ?? 0) > 0 || port !== "free" || legacy) throw new Error(`Stale Gateway PID ${params.stale.pid ?? "unknown"} did not stop: service=${runtime.status}, PID=${runtime.pid ?? "none"}, port ${params.stale.port}=${port}, legacy lock owner=${legacy?.pid ?? "none"}.`);
}
//#endregion
//#region src/commands/doctor-maintenance-foreground.ts
async function acquireDoctorGatewayMaintenanceOwner(databasePath, env, params) {
const updateRepair = isDoctorUpdateRepairMode(resolveDoctorRepairMode(params.options));
let foreground;
let ownerlessDeadlineMs;
return await acquireWithWait({
acquire: async () => {
try {
const owner = await acquireGatewayLock({
env,
role: "sqlite-maintenance",
allowInTests: true,
lifecycleDeadlineMs: params.deadlineMs,
assertCurrent: params.assertCurrent,
onWait: params.runtime.log,
relocatedMaintenanceOwner: params.relocatedMaintenanceOwner
});
if (!owner) throw new Error(`Doctor could not acquire maintenance ownership for ${databasePath}`);
return owner;
} catch (error) {
if (error instanceof GatewayLockError && error.cause instanceof GatewayStateOwnerContentionError) throw error.cause;
throw error;
}
},
shouldRetry: (error) => {
if (!updateRepair || !params.assertCurrent || !(error instanceof GatewayStateOwnerContentionError)) return false;
params.assertCurrent();
const current = readGatewayOwnerLease({
env,
current: true,
openStateSchemaReadAdmission: openDoctorStateSchemaReadAdmission
});
if (!foreground) {
if (!current) {
if (ownerlessDeadlineMs === void 0) {
ownerlessDeadlineMs = performance.now() + GATEWAY_SHUTDOWN_RESERVE_MS;
params.runtime.log("Waiting for Gateway state ownership cleanup to finish.");
}
return performance.now() < ownerlessDeadlineMs;
}
if (ownerlessDeadlineMs !== void 0) return false;
if (current.state !== "live" || current.mode !== "foreground") return false;
foreground = current;
params.runtime.log("Waiting for the previous foreground Gateway to release state.");
} else if (current && (current.owner !== foreground.owner || current.pid !== foreground.pid || current.startedAt !== foreground.startedAt || current.host !== foreground.host || current.mode !== "foreground")) return false;
return readStateLeaseProcessOwnerStatus(foreground) === "live";
},
deadlineMs: Math.min(params.deadlineMs ?? Infinity, performance.now() + TICK_INTERVAL_MS + resolveGatewayRestartDeferralTimeoutMs() + GATEWAY_SERVICE_STOP_TIMEOUT_MS),
pollIntervalMs: 100,
maxPollIntervalMs: 1e3,
sleep: (ms) => sleep(ownerlessDeadlineMs === void 0 ? ms : Math.min(ms, Math.max(0, ownerlessDeadlineMs - performance.now())))
});
}
//#endregion
//#region src/commands/doctor-maintenance-state.ts
/** Database custody can change paths while stopped-service custody remains with Doctor. */
function createDoctorMaintenanceState(options) {
const { params, env, settle } = options;
let resources;
let resourcesParent;
let inspections;
let owner;
let selectedEnv = env;
let captureAdmitted = false;
let liveAuthorityReadsAdmitted = false;
const capture = createUpdateDoctorDatabaseWriteCapture(params.databaseGenerations, {
env,
root: params.root ?? void 0,
signal: options.signal,
assertCurrent: () => owner.assertCurrent(options.assertCurrent),
warn: options.warn
});
const closeResources = async (agentRoot) => {
const drainRoot = agentRoot === null ? void 0 : agentRoot ?? (resourcesParent ? void 0 : resolveStateDir(selectedEnv));
const failures = [];
const closeScope = async (close, clear) => {
try {
await close();
clear();
} catch (error) {
try {
await close();
clear();
failures.push(error);
} catch (retryError) {
failures.push(new AggregateError([error, retryError], "Doctor maintenance resource cleanup failed.", { cause: retryError }));
}
}
};
const currentResources = resources;
let resourcesClosed = currentResources === void 0;
if (currentResources) await closeScope(() => currentResources.close(drainRoot === void 0 ? void 0 : async () => {
const { closeOpenClawAgentDatabasesAsync } = await import("./openclaw-agent-db-lifecycle-CxGnM_Lx.mjs");
await closeOpenClawAgentDatabasesAsync(drainRoot);
}), () => {
resourcesClosed = true;
});
if (inspections) {
const current = inspections;
await closeScope(() => current.close(), () => {
inspections = void 0;
});
}
await closeScope(async () => {
const { closeAuthProfileReadPool } = await import("./sqlite-read-pool-B1wqThEa.mjs");
closeAuthProfileReadPool({
kind: "root",
rootPath: resolveStateDir(selectedEnv)
});
}, () => {
if (resourcesClosed) {
resources = void 0;
resourcesParent = void 0;
}
});
if (failures.length === 1) throw failures[0];
if (failures.length > 1) throw new AggregateError(failures, "Doctor maintenance resource cleanup failed.");
};
const settleCapture = async () => {
if (owner && capture && captureAdmitted) {
await settle(() => capture.settle());
captureAdmitted = false;
}
};
const enterResources = async (acquired) => {
owner = acquired;
try {
acquired.assertCurrent(options.assertCurrent);
resourcesParent = getOpenClawDatabaseMaintenanceScope();
resources = createOpenClawDatabaseMaintenanceScope({
schemaMaintenance: true,
assertDatabaseAccess: acquired.assertDatabaseAccess,
assertOwnerCurrent: (access) => {
acquired.assertCurrent(() => {
options.assertCurrent?.();
options.assertReadCurrent();
}, access);
}
});
if (liveAuthorityReadsAdmitted) resources.run(() => admitOpenClawMaintenanceLiveAuthorityReads(resolveOpenClawStateSqlitePath(selectedEnv)));
inspections = createSqliteReadOnlyWorkerScope({
signal: options.signal,
deadlineOwnedByCaller: false
});
} catch (error) {
await acquired.release();
owner = void 0;
throw error;
}
if (capture && !captureAdmitted) {
await settle(() => resources.run(() => capture.admit()));
captureAdmitted = true;
}
};
const assertOwnerCurrent = (databasePath) => {
const assertCurrent = () => {
owner.assertCurrent(options.assertCurrent);
if (databasePath !== void 0) owner.assertDatabaseAccess(databasePath);
};
if (resources) resources.run(assertCurrent);
else owner.run(assertCurrent);
};
const state = {
get owner() {
return owner;
},
get resources() {
return resources;
},
get hasOpenResources() {
return Boolean(resources || inspections);
},
get receipt() {
return owner ? void 0 : capture?.receipt;
},
run(operation) {
return resources.run(() => inspections.run(operation));
},
async acquire(relocatedMaintenanceOwner) {
if (resources) return;
const assertCurrent = relocatedMaintenanceOwner ? () => relocatedMaintenanceOwner.assertCurrent(options.assertCurrent) : options.assertCurrent;
const acquired = await acquireDoctorGatewayMaintenanceOwner(path.resolve(resolveOpenClawStateSqlitePath(selectedEnv)), selectedEnv, {
...params,
assertCurrent,
deadlineMs: options.deadline(),
relocatedMaintenanceOwner
});
await enterResources(acquired);
},
async prepareRepair() {
const beforeStateMutation = params.beforeStateMutation;
if (beforeStateMutation) {
await resources.run(() => beforeStateMutation({
env: selectedEnv,
signal: options.signal
}));
resources.assertAdmission();
}
resources.run(() => admitOpenClawMaintenanceLiveAuthorityReads(resolveOpenClawStateSqlitePath(selectedEnv)));
liveAuthorityReadsAdmitted = true;
const { resolvePendingLegacyStateDirMigrationPaths, prepareLegacyStateDirMigration } = await import("./state-migrations.state-dir-DovHCDPq.mjs");
const pending = resolvePendingLegacyStateDirMigrationPaths({ env });
const sourceDir = resolveStateDir(env);
if (!pending || path.resolve(sourceDir) !== path.resolve(pending.source)) return;
owner.assertCurrent(options.assertCurrent);
const sourceDatabase = resolveOpenClawStateSqlitePath(env);
await closeResources(sourceDir);
await closeOpenClawStateDatabaseByPathAsync(sourceDatabase);
await settleCapture();
const migration = owner.run(() => {
options.assertCurrent?.();
owner.assertCurrent();
return prepareLegacyStateDirMigration({ env });
});
selectedEnv = {
...env,
OPENCLAW_STATE_DIR: migration?.stateDir ?? sourceDir
};
if (resolveGatewayStateOwnerPath(resolveOpenClawStateSqlitePath(selectedEnv)) !== owner.lockPath) await state.acquire(owner);
else await enterResources(owner);
if (migration) {
const result = migration.result;
for (const change of [...result.changes, ...result.notices ?? []]) params.runtime.log(sanitizeDoctorNote(change));
for (const warning of result.warnings) options.warn(sanitizeDoctorNote(warning));
}
},
async repairSqliteNoCow(paths) {
if (paths.length === 0) return {
changes: [],
warnings: []
};
const { repairDoctorSqliteNoCow } = await import("./doctor-sqlite-nocow-CCqGJgfz.mjs");
const stateDir = resolveStateDir(selectedEnv);
const databasePath = resolveOpenClawStateSqlitePath(selectedEnv);
assertOwnerCurrent();
await closeResources(stateDir);
try {
return await owner.run(async () => {
await closeOpenClawStateDatabaseByPathAsync(databasePath);
await assertDoctorAgentLeaseAdmission(selectedEnv);
return repairDoctorSqliteNoCow({
paths,
stateDir,
assertCurrent: () => assertOwnerCurrent(databasePath)
});
});
} finally {
await enterResources(owner);
}
},
async enableSqliteReclamation(agents) {
const { enableDoctorSqliteReclamation } = await import("./doctor-sqlite-reclamation-AF8F9_qh.mjs");
const { closeOpenClawAgentDatabaseByPathAsync } = await import("./openclaw-agent-db-lifecycle-CxGnM_Lx.mjs");
const databasePath = resolveOpenClawStateSqlitePath(selectedEnv);
options.signal.throwIfAborted();
assertOwnerCurrent();
await resources.run(async () => {
for (const agent of agents) await closeOpenClawAgentDatabaseByPathAsync(agent.path, agent.agentId);
});
await closeResources(null);
let operationError;
let result;
try {
result = await owner.run(async () => {
await closeOpenClawStateDatabaseByPathAsync(databasePath);
await assertDoctorAgentLeaseAdmission(selectedEnv);
return enableDoctorSqliteReclamation({
env: selectedEnv,
agents,
signal: options.signal,
assertCurrent: () => assertOwnerCurrent(databasePath),
log: params.runtime.log
});
});
} catch (error) {
operationError = toErrorObject(error, "SQLite reclamation failed.");
}
try {
await enterResources(owner);
} catch (error) {
if (operationError !== void 0) throw new AggregateError([operationError, error], "SQLite reclamation and maintenance scope restoration failed.", { cause: error });
throw error;
}
if (operationError !== void 0) throw operationError;
return result;
},
async cleanupRetainedRuntimes(inspectService) {
const { captureRetainedNativeWorkerSource } = await import("./worker-native-lifecycle-B7QmGwq_.mjs");
const { retireIdleOpenClawStateReadWorkers } = await import("./openclaw-state-read-worker-jDU7OqV_.mjs");
const { prepareRetainedUpdateRuntimeCleanup } = await import("./doctor-retained-runtime-B8UnnHqz.mjs");
assertOwnerCurrent();
const cleanup = await state.run(() => prepareRetainedUpdateRuntimeCleanup(selectedEnv, { inspectService }));
const nativeSource = captureRetainedNativeWorkerSource();
let readersRetired;
let brokerRetired;
try {
await closeResources();
readersRetired = await retireIdleOpenClawStateReadWorkers(nativeSource);
brokerRetired = await nativeSource.retireIdleBroker();
} catch (cause) {
throw new DoctorMaintenanceRefusalError(`Doctor inspection resource cleanup failed: ${formatErrorMessage(cause)}. Resolve this cleanup failure before restarting the Gateway or rerunning openclaw doctor --fix.`, {
kind: "data-at-risk",
reason: "active-mutation"
}, { cause });
}
try {
await owner.run(() => cleanup(true, {
assertCurrent() {
assertOwnerCurrent();
owner.assertDatabaseAccess(resolveOpenClawStateSqlitePath(selectedEnv));
},
assertResourcesSettled() {
if (!readersRetired || !brokerRetired || nativeSource.hasActiveWorkers) throw new Error(`independent native work in this process (PID: ${process.pid}); let these holders finish, then rerun openclaw doctor --fix`);
}
}));
} finally {
await enterResources(owner);
}
},
async release() {
const failures = [];
try {
await closeResources();
} catch (error) {
if (hasCommandProcessCleanupError(error) || state.hasOpenResources) throw error;
failures.push(error);
}
try {
await settleCapture();
} catch (error) {
if (hasCommandProcessCleanupError(error)) throw error;
failures.push(error);
}
try {
await owner?.release();
owner = void 0;
} catch (error) {
failures.push(error);
}
if (failures.length === 1) throw failures[0];
if (failures.length > 1) throw new AggregateError(failures, "Doctor maintenance state cleanup failed.");
}
};
return state;
}
//#endregion
//#region src/commands/doctor-maintenance.ts
async function beginDoctorMaintenance(params) {
if (!(params.options.repair === true || params.options.yes === true)) return;
const env = {
...process.env,
...params.runId ? { [UPDATE_RUN_ID_ENV]: params.runId } : {}
};
const parentActivation = isDoctorUpdateRepairMode(resolveDoctorRepairMode(params.options)) ? resolveUpdateParentGatewayActivation(env) : void 0;
let stopped;
let serviceUpdateVerdict;
let stopDeadline;
const warnings = [];
const warn = (message) => {
warnings.push(message);
params.runtime.log(message);
};
const failureFacts = [];
let repairStoresMayBeOpen = false;
let retainStoppedInstallation = false;
let inspectingActivation = false;
let staleReplacement;
let assertUpdateAdmissionReadCurrent;
let assertUpdateAdmissionCurrent;
let authorityRefused = false;
const assertAuthority = (assertion) => {
try {
assertion();
} catch (error) {
authorityRefused = true;
throw error;
}
};
const callerAssertCurrent = params.assertCurrent;
const assertCallerCurrent = callerAssertCurrent ? () => assertAuthority(callerAssertCurrent) : void 0;
let cleanupFailure;
const settle = async (operation) => {
if (cleanupFailure) throw cleanupFailure.error;
try {
return await withCommandProcessScope(operation);
} catch (error) {
if (hasCommandProcessCleanupError(error)) cleanupFailure ??= { error };
throw error;
}
};
const acquireStoppedMaintenanceResources = () => acquireWithWait({
acquire: () => state.acquire(),
shouldRetry: (error) => stopped?.stopped === true && error instanceof GatewayStateOwnerContentionError,
deadlineMs: stopDeadline ?? performance.now(),
pollIntervalMs: 250,
maxPollIntervalMs: 2e3
});
const releaseState = async () => {
if (cleanupFailure) throw cleanupFailure.error;
await state.release();
repairStoresMayBeOpen = false;
};
let deferredStateReleaseFailure;
const throwDeferredStateReleaseFailure = () => {
if (deferredStateReleaseFailure !== void 0) throw deferredStateReleaseFailure;
};
const release = async (assertCustody) => {
await settle(async () => {
try {
await releaseState();
} catch (error) {
if (hasCommandProcessCleanupError(error) || state.hasOpenResources) throw error;
deferredStateReleaseFailure ??= toErrorObject(error, "Doctor state release failed.");
}
const recovery = stopped?.windowsTaskAutoStartRecovery;
try {
assertCustody?.();
if (!retainStoppedInstallation) await settle(async () => {
await recovery?.restore(true);
});
} finally {
if (!cleanupFailure) await settle(async () => recovery?.complete(!retainStoppedInstallation));
}
});
};
const finish = async (cfg, assertCustody, writeConfig, assertRestoreAdmission = assertUpdateAdmissionCurrent) => {
await release(assertCustody);
assertCustody?.();
const before = stopped;
const root = params.root;
if (!before?.serviceEnv || !root) return;
if (!before.stopped) {
const verdict = before.serviceUpdateVerdict;
if (verdict?.kind === "owned" && verdict.requiresInstallRootRefresh) {
const { inspectGatewayServiceInstallationDrift } = await import("./service-layout-B33QDzfj.mjs");
const drift = await inspectGatewayServiceInstallationDrift({ packageRootReal: verdict.root }, root);
if (drift) {
const { formatGatewayServiceInstallationDrift } = await import("./shared-DlDa7JMe.mjs");
const message = formatGatewayServiceInstallationDrift(drift, void 0, before.serviceEnv, {
stopped: true,
port: before.servicePort
});
warn(message);
return;
}
}
if (resolveDoctorRepairMode(params.options).updateInProgress || before.offline !== true || verdict?.kind !== "owned" || verdict.requiresInstallRootRefresh) return;
}
try {
const serviceEnv = before.serviceEnv;
const [{ restoreDoctorGatewayService }, { resolveUpdatedGatewayRestartPort }, { renderRestartDiagnostics, waitForGatewayHealthyRestart }] = await Promise.all([
import("./doctor-maintenance-restoration-CR0WUI9k.mjs"),
import("./update-command-service-plan-D81Z2v3m.mjs"),
import("./restart-health-DtiD03aX.mjs")
]);
const { service, state, cfg: restoredConfig } = await restoreDoctorGatewayService({
before,
serviceEnv,
root,
env,
cfg,
writeConfig,
options: params.options,
runtime: params.runtime,
signal: exit.signal,
warnings,
settle,
assertCustody,
assertRestoreAdmission,
assertInstallationAdmission: assertUpdateAdmissionCurrent
});
if (!state) return;
const port = await resolveUpdatedGatewayRestartPort({
config: restoredConfig,
serviceEnv: state.env,
serviceCommand: state.command
});
const health = await settle(() => waitForGatewayHealthyRestart({
service,
port,
env: state.env,
requireRunningService: true,
...staleReplacement ? {
expectedVersion: staleReplacement.version,
expectedBuildId: staleReplacement.buildId,
requirePluginHealth: false
} : {}
}));
if (health.outcome === "starting") {
warn(`Warning: Doctor repair complete; Gateway is still starting — check \`${formatCliCommand("openclaw gateway status", state.env)}\` in a minute.`);
return;
}
if (health.outcome !== "ready") throw doctorGatewayMaintenanceError({
env,
phase: "gateway-restoration",
code: "doctor-gateway-rpc-verification-failed",
detail: `Doctor repaired state, but the managed Gateway did not become ready: ${renderRestartDiagnostics(health).join(" ")}.`
});
} catch (error) {
if (hasCommandProcessCleanupError(error)) {
cleanupFailure ??= { error };
throw error;
}
if (error instanceof GatewayServiceAuthorityError) {
const { formatDaemonServiceInstallCommand } = await import("./shared-DlDa7JMe.mjs");
const serviceEnv = before.serviceEnv;
const outcome = error.outcome ?? "recovery-pending";
const message = `Doctor could not finish Gateway installation or activation under its maintenance authority (${outcome}). ${outcome === "unchanged" ? "The previous service definition was left unchanged." : outcome === "restored" ? "The previous service definition was restored from its captured backup." : "Restoration was not verified; inspect the current service before replacing it."} Run \`${formatCliCommand("openclaw gateway status --deep", serviceEnv)}\`; after the active maintenance or update finishes, run \`${formatDaemonServiceInstallCommand(serviceEnv, before.servicePort)}\` from the active CLI. Reason: ${error.message}`;
failureFacts.push(createUpdateFailureFact({
check: "gateway-restoration",
code: `${error.code}-${outcome}`,
message
}, serviceEnv));
warnings.push(message);
params.runtime.error(message);
if (outcome === "recovery-pending") throw new UpdateDoctorError(message, failureFacts, { cause: error });
return;
}
if (error instanceof UpdateDoctorError) throw error;
throw doctorGatewayMaintenanceError({
env,
phase: "gateway-restoration",
code: findServiceOwnershipRefusal(error)?.reason ?? "doctor-gateway-restoration-failed",
detail: `The managed Gateway could not be restored after Doctor maintenance: ${String(error)}`,
cause: error
});
}
params.runtime.log("Gateway restarted and verified after Doctor repair.");
if (staleReplacement) {
const warning = `Warning: Replaced stale Gateway PID ${staleReplacement.pid ?? "unknown"} through its service manager; verified ${staleReplacement.version} build ${staleReplacement.buildId} after Doctor maintenance.`;
warn(warning);
}
};
const admitRepair = async () => {
inspectingActivation = false;
await assertLegacyGatewayStoppedForMaintenance(env);
await acquireStoppedMaintenanceResources();
assertUpdateAdmissionCurrent?.();
await assertDoctorAgentLeaseAdmission(env);
repairStoresMayBeOpen = true;
};
let admissionFailureHandled = false;
const failAdmission = async (error, assertStopCustody) => {
admissionFailureHandled = true;
if (hasCommandProcessCleanupError(error)) throw error;
try {
if (stopped?.stopped) {
try {
await acquireStoppedMaintenanceResources();
} catch (ownershipError) {
const warning = ownershipError instanceof GatewayStateOwnerContentionError ? `Warning: The stopped Gateway still holds state ownership after the service stop deadline. Shared-state repair is unsafe while that writer remains active. Restoring its service; run ${formatCliCommand("openclaw gateway status --deep", env)}, then retry ${formatCliCommand("openclaw doctor --fix", env)} after shutdown completes.` : `Warning: Doctor could not reacquire maintenance ownership: ${String(ownershipError)} Restoring its service without repairing shared state.`;
warn(warning);
}
const { readConfigFileSnapshot } = await import("./config/config.js");
await finish((await readConfigFileSnapshot({
skipPluginValidation: true,
observe: false
})).config, assertStopCustody, void 0, assertStopCustody ?? assertUpdateAdmissionCurrent);
throwDeferredStateReleaseFailure();
} else {
await release();
throwDeferredStateReleaseFailure();
}
} catch (restoreError) {
throw new AggregateError([error, restoreError], `${String(error)} ${String(restoreError)}`, { cause: restoreError });
}
resolveCommandProcessSignal()?.throwIfAborted();
if (authorityRefused || error instanceof DoctorUnreadableStateDatabaseError) throw error;
const refusal = error instanceof DoctorMaintenanceRefusalError ? error : new DoctorMaintenanceRefusalError(`Doctor could not enter maintenance. ${String(error)}${hasGatewayServiceStopUnsafeError(error) ? "" : ` Stop the Gateway service and other OpenClaw processes using this state, then run ${formatCliCommand("openclaw doctor --fix", env)} from an independent shell.`}`, classifyDoctorMaintenanceRefusal(error), {
cause: error,
...error instanceof UpdateDoctorError ? { failureFacts: error.failureFacts } : {}
});
const recovery = inspectingActivation ? await resolveUpdateDoctorGitRecovery({ root: params.root }) : void 0;
if (recovery) {
refusal.message += `\n${recovery.message}`;
recordUpdateDoctorRefusal(refusal.message);
}
throw refusal;
};
const exit = holdDoctorMaintenanceExit((message) => {
warnings.push(message);
params.runtime.error(message);
});
const state = createDoctorMaintenanceState({
params,
env,
signal: exit.signal,
deadline: () => stopDeadline,
assertCurrent: assertCallerCurrent,
assertReadCurrent: () => assertUpdateAdmissionReadCurrent?.(),
settle,
warn
});
try {
await settle(async () => {
const externallyManaged = isServiceRepairExternallyManaged();
if (externallyManaged) await preflightExternalDoctorAgentLease(env);
if (params.root && isDefaultInstallIdentity(env) && !externallyManaged && await shouldManageGatewayService(env)) {
const { maybeStopManagedServiceBeforeMutableUpdate } = await import("./update-command-service-maintenance-CSZZDAPb.mjs");
inspectingActivation = true;
let inspection;
try {
inspection = await maybeStopManagedServiceBeforeMutableUpdate({
updateInstallKind: "package",
root: params.root,
shouldRestart: true,
jsonMode: true,
phase: "inspect"
});
} catch (error) {
const offlineApp = await readOfflineAppOwnedDoctorInspection(error, env);
if (!offlineApp) throw error;
inspection = offlineApp;
warn(offlineApp.serviceMutationSkipMessage);
}
assertDoctorMaintenanceInspection(inspection, env);
serviceUpdateVerdict = inspection.serviceUpdateVerdict;
if (inspection.serviceUpdateVerdict?.kind !== "absent" && inspection.offline !== true) assertAuthority(() => {
const admitted = resolveDoctorUpdateAdmission(env);
assertUpdateAdmissionReadCurrent = () => assertAuthority(admitted.assertCurrent);
const checkAndRecord = () => assertAuthority(admitted.recordContinuation);
assertUpdateAdmissionCurrent = () => state.owner ? state.owner.run(checkAndRecord) : checkAndRecord();
});
if (inspection.serviceUpdateVerdict?.kind === "owned" && inspection.serviceEnv) assertDoctorServiceSelection(env, inspection.serviceEnv);
staleReplacement = await inspectStaleDoctorGateway({
root: params.root,
env,
before: inspection,
assertCurrent: assertUpdateAdmissionCurrent
});
if (parentActivation !== void 0 && !staleReplacement && inspection.serviceUpdateVerdict?.kind === "owned" && inspection.offline !== true) throw new DoctorMaintenanceRefusalError(`Doctor could not enter maintenance. Error: ${await formatUpdateDoctorServiceStopRefusal(inspection.serviceEnv ?? env)}`, {
kind: "data-at-risk",
reason: "gateway-state-unverified"
});
try {
await state.acquire();
} catch (error) {
if (hasCommandProcessCleanupError(error)) throw error;
const observationSignal = resolveCommandProcessSignal(exit.signal) ?? exit.signal;
const servingOwner = await readDoctorGatewayOwnerLease(env, observationSignal);
const legacyGatewayLock = servingOwner ? void 0 : await readActiveGatewayLockIdentity({
env: inspection.serviceEnv ?? env,
requireInspection: true
});
observationSignal.throwIfAborted();
assertCallerCurrent?.();
assertUpdateAdmissionReadCurrent?.();
if (!inspection.running || !(servingOwner?.state === "live" && servingOwner.mode === "supervised" || inspection.servicePid !== void 0 && legacyGatewayLock?.pid === inspection.servicePid)) throw error;
}
if (inspection.serviceUpdateVerdict?.kind === "owned") {
inspectingActivation = false;
if (inspection.serviceEnv) assertDoctorServiceSelection(env, inspection.serviceEnv);
if (parentActivation === void 0 || staleReplacement) {
inspection.serviceUpdateVerdict.refreshDefinition = inspection.serviceUpdateVerdict.requiresInstallRootRefresh === true;
const root = params.root;
const { withGatewayServiceOperationLock } = await import("./service-operation-lock-oHJtCFhr.mjs");
await withGatewayServiceOperationLock(inspection.serviceEnv ?? env, async (assertStopCustody) => {
let assertServiceCurrent = assertUpdateAdmissionCurrent;
try {
await settle(async () => {
try {
stopDeadline = performance.now() + GATEWAY_SERVICE_STOP_TIMEOUT_MS;
const stopService = () => maybeStopManagedServiceBeforeMutableUpdate({
updateInstallKind: "package",
root,
shouldRestart: true,
jsonMode: true,
expectedService: inspection,
retainNativeIdentity: true,
assertCurrent: () => assertServiceCurrent?.(),
warn,
onStopped: (before) => {
stopped = before;
}
});
stopped = await (state.resources ? state.resources.run(stopService) : stopService());
assertDoctorMaintenanceInspection(stopped, env);
if (stopped.serviceUpdateVerdict?.kind === "unavailable") warn(stopped.serviceUpdateVerdict.message);
if (staleReplacement && stopped.serviceEnv) await assertStaleDoctorGatewayStopped({
stale: staleReplacement,
env: stopped.serviceEnv,
assertCurrent: assertUpdateAdmissionCurrent
});
} catch (error) {
if (!staleReplacement || hasCommandProcessCleanupError(error)) throw error;
throw doctorGatewayMaintenanceError({
env,
phase: "gateway-stop",
code: "stale-gateway-stop-failed",
detail: String(error),
cause: error
});
}
if (stopped?.stopped) params.runtime.log("Stopped the managed Gateway for Doctor repair.");
await admitRepair();
assertStopCustody();
});
} catch (error) {
assertServiceCurrent = assertStopCustody;
await failAdmission(error, assertStopCustody);
}
});
}
} else if (inspection.serviceUpdateVerdict?.kind === "unavailable") warn(inspection.serviceUpdateVerdict.message);
else if (inspection.serviceUpdateVerdict?.kind !== "absent") params.runtime.log("The stopped Gateway service was left unchanged; repairing Doctor's selected state only.");
}
if (!repairStoresMayBeOpen) await admitRepair();
stopped?.windowsTaskAutoStartRecovery?.beginMutation();
retainStoppedInstallation = stopped?.serviceUpdateVerdict?.kind === "owned" && stopped.serviceUpdateVerdict.requiresInstallRootRefresh === true;
await state.prepareRepair();
});
} catch (error) {
try {
if (admissionFailureHandled) throw error;
await failAdmission(error);
} finally {
exit.release(true);
}
}
let custody = "held";
const maintenance = {
signal: exit.signal,
serviceUpdateVerdict,
warnings,
failureFacts,
get databaseWrites() {
return state.receipt;
},
run: (operation) => state.run(operation),
releaseState: () => settle(releaseState),
async repairSqliteNoCow(paths) {
if (this !== maintenance || custody !== "held") throw new Error("SQLite NOCOW repair requires its original live maintenance owner.");
const result = await settle(() => state.repairSqliteNoCow(paths));
for (const message of result.changes) params.runtime.log(message);
for (const message of result.warnings) warn(message);
},
async enableSqliteReclamation(agents) {
if (this !== maintenance || custody !== "held") throw new Error("SQLite reclamation requires its original live maintenance owner.");
const result = await settle(() => state.enableSqliteReclamation(agents));
for (const message of result.warnings) warn(message);
},
async cleanupRetainedRuntimes() {
if (this !== maintenance || custody !== "held") throw new Error("Updater runtime cleanup requires its original live maintenance owner.");
await settle(() => state.cleanupRetainedRuntimes(serviceUpdateVerdict !== void 0));
},
async release() {
if (this !== maintenance) throw new Error("Gateway restoration requires its original live maintenance owner.");
custody = "released";
try {
await release();
throwDeferredStateReleaseFailure();
} catch (error) {
exit.release(true);
throw error;
} finally {
exit.release();
}
},
async finish(initialConfig, writeConfig, failure) {
let cfg = initialConfig;
if (cleanupFailure) throw cleanupFailure.error;
const assertCustody = (expected = "restoring") => {
if (this !== maintenance || custody !== expected) throw new Error("Gateway restoration requires its original live maintenance owner.");
};
assertCustody("held");
custody = "restoring";
let failed = failure !== void 0;
try {
if (!cfg && !stopped?.stopped) {
await release(assertCustody);
throwDeferredStateReleaseFailure();
return;
}
if (!cfg) try {
const { readConfigFileSnapshot } = await import("./config/config.js");
cfg = (await readConfigFileSnapshot({
skipPluginValidation: true,
observe: false
})).config;
} catch (error) {
throw new DoctorMaintenanceRefusalError(`Doctor could not restore the Gateway because persisted repair state is not ready: ${formatErrorMessage(error)}`, {
kind: "data-at-risk",
reason: "incomplete-migration"
}, { cause: error });
}
await finish(cfg, assertCustody, writeConfig);
throwDeferredStateReleaseFailure();
} catch (restoreError) {
failed = true;
if (failure !== void 0) throw new AggregateError([failure, restoreError], `${formatErrorMessage(failure)} ${formatErrorMessage(restoreError)}`, { cause: restoreError });
throw restoreError;
} finally {
custody = "released";
exit.release(failed);
}
}
};
return maintenance;
}
//#endregion
export { beginDoctorMaintenance };