UNPKG

openclaw

Version:

Multi-channel AI gateway with extensible messaging integrations

1,013 lines (1,012 loc) 45.7 kB
import { o as asDateTimestampMs } from "./number-coercion-CLj0HTDM.js"; import "./src-vebZIeLe.js"; import { t as expectDefined } from "./expect-CyE8FADM.js"; import { r as truncateUtf16Safe } from "./utf16-slice-D_ngcYKd.js"; import { t as pruneMapToMaxSize } from "./map-size-CNcWiFKu.js"; import { O as parseAgentSessionKey, T as isSubagentSessionKey, d as normalizeOptionalAgentId } from "./session-key-BnWWjqNc.js"; import { n as formatErrorMessageWithCode } from "./errors-Db3Ymjlb.js"; import { a as getNodeSqliteKysely, i as executeSqliteQueryTakeFirstSync, r as executeSqliteQuerySync } from "./kysely-sync-COmh4HWh.js"; import { S as OPENCLAW_STATE_SCHEMA_SQL } from "./openclaw-state-db-cache-C7ljO0xP.js"; import { s as runOpenClawStateWriteTransaction } from "./openclaw-state-db-BRTnL-D8.js"; import { i as isPidDefinitelyDead, t as getFileLockProcessStartTime } from "./pid-alive-XuW58Ofz.js"; import { r as executionOwnerBindingFromAdmission } from "./execution-owner-binding-DvQsTSTS.js"; import { n as bindExecutionOwnerLifecycleMetadata, r as deleteExecutionOwnerLifecycleMetadata } from "./execution-owner-lifecycle-binding-store-DtEbBIBz.js"; import { a as loadCronRows, o as loadedCronStoreFromRows } from "./row-codec-Bv7_IRNO.js"; import { a as createCronStreamSourceIdentity, c as resolveCronStreamBatching } from "./normalize-Dd-LvsuI.js"; import { n as normalizePayloadToSystemText } from "./normalize-Dm4ZnpdH.js"; import { t as parseAbsoluteTimeMs } from "./parse-tlCSjrTr.js"; import { t as coerceFiniteScheduleNumber } from "./schedule-number-dcwHraee.js"; import { a as resolveCronDeliverySessionKey } from "./session-target-DJsUULzX.js"; import { r as resolveCronStaggerMs, t as isSystemOwnedCronPayloadKind } from "./types-DQrueG6d.js"; import { t as resolveCronJobConfigRevision } from "./config-revision-CExLxl38.js"; import { t as cronStoreKey } from "./key-BBZ40bDq.js"; import { l as isCronJobActive } from "./active-jobs-C_biqiZG.js"; import { n as computePreviousRunAtMs, t as computeNextRunAtMs } from "./schedule-Dirub5qQ.js"; import crypto from "node:crypto"; //#region src/cron/failure-notification-text.ts const GENERIC_FAILURE_DETAIL = "Check automation history for details."; const SCRIPT_FAILURE_COPY = { aborted: "was aborted", invalid_input: "received invalid input", runtime_unavailable: "runtime is unavailable", timeout: "timed out", output_limit_exceeded: "exceeded its output limit", snapshot_limit_exceeded: "exceeded its state limit", internal_error: "failed internally", tool_budget_exceeded: "exceeded its tool budget" }; /** Renders only classified reasons or closed producer-authored failure facts. */ function cronFailureDetailLines(errorReason, failureNotificationDetail) { if (errorReason) return [`Cause: ${errorReason}`]; if (!failureNotificationDetail) return [GENERIC_FAILURE_DETAIL]; if (failureNotificationDetail.kind === "command-exit") return [`Cause: command exited with code ${failureNotificationDetail.exitCode}`]; if (failureNotificationDetail.kind === "command-timeout") return [failureNotificationDetail.mode === "wall-clock" ? "Cause: command timed out" : "Cause: command stopped after producing no output"]; return [`Cause: ${failureNotificationDetail.source === "payload" ? "automation script" : "trigger script"} ${SCRIPT_FAILURE_COPY[failureNotificationDetail.code]}`]; } //#endregion //#region src/cron/store/run-receipt-store.ts const CRON_RUN_RECEIPT_SCHEMA_START = "CREATE TABLE IF NOT EXISTS cron_run_receipts ("; const CRON_RUN_RECEIPT_SCHEMA_END = "ON cron_run_receipts(store_key, job_id, started_at_ms DESC, receipt_id DESC);"; const CRON_RUN_RECEIPT_TERMINAL_RETENTION = 64; const CRON_RUN_RECEIPT_DELETE_BATCH_SIZE = 500; const CRON_RUN_RECEIPT_FINISH_RETRY_MS = 1e3; /** Recovery horizon for abandoned markers and unverifiable foreign receipts. */ const CRON_STUCK_RUN_MS = 72e5; const initializedDatabases = /* @__PURE__ */ new WeakSet(); const locallyOwnedReceipts = /* @__PURE__ */ new Set(); const pendingReceiptSettlements = /* @__PURE__ */ new Map(); const pendingReceiptFinishRetries = /* @__PURE__ */ new Map(); var CronRunReceiptConflictError = class extends Error { constructor(receipt) { super(`cron job ${receipt.jobId} is already running in process ${receipt.ownerPid}`); this.receipt = receipt; this.name = "CronRunReceiptConflictError"; this.candidate = receiptHandle(receipt); } }; var CronRunReceiptRevisionError = class extends Error { constructor(receiptId, message = "cron run configuration changed", reason = "revision-changed") { super(message); this.receiptId = receiptId; this.reason = reason; this.name = "CronRunReceiptRevisionError"; } }; function ensureCronRunReceiptSchema(database) { const start = OPENCLAW_STATE_SCHEMA_SQL.indexOf(CRON_RUN_RECEIPT_SCHEMA_START); const endMarker = OPENCLAW_STATE_SCHEMA_SQL.indexOf(CRON_RUN_RECEIPT_SCHEMA_END, start); if (start < 0 || endMarker < start) throw new Error("OpenClaw cron run receipt schema marker is missing."); database.exec(OPENCLAW_STATE_SCHEMA_SQL.slice(start, endMarker + 77)); } function query(database) { return getNodeSqliteKysely(database); } function activeRow(db, key, jobId) { const find = () => { const active = query(db).selectFrom("cron_run_receipts").where("store_key", "=", key).where("status", "=", "running"); return jobId === void 0 ? executeSqliteQuerySync(db, active.select("job_id")).rows : executeSqliteQueryTakeFirstSync(db, active.selectAll().where("job_id", "=", jobId)); }; try { return find(); } catch (error) { if (!(error instanceof Error) || error.message !== "no such table: cron_run_receipts") throw error; ensureCronRunReceiptSchema(db); return find(); } } function withReceiptWrite(operationLabel, options, operation) { let initializedDatabase; const result = runOpenClawStateWriteTransaction(({ db }) => { if (!initializedDatabases.has(db)) { ensureCronRunReceiptSchema(db); initializedDatabase = db; } return operation(db); }, options, { operationLabel }); if (initializedDatabase) initializedDatabases.add(initializedDatabase); return result; } /** Binds the exact admitted execution to its authoritative receipt without changing lifecycle. */ function bindCronRunReceiptExecution(params) { const binding = executionOwnerBindingFromAdmission(params.admitted); if (!binding) return "disabled"; return withReceiptWrite("cron.run-receipt.execution-binding", params.options ?? {}, (database) => { try { assertCronRunReceiptOwnedInDatabase({ database, handle: params.handle }); } catch (error) { if (!(error instanceof CronRunReceiptRevisionError)) throw error; return "missing"; } return bindExecutionOwnerLifecycleMetadata({ db: database, ownerKind: "cron", ownerId: params.handle.receiptId, binding }); }); } function isReceiptStatus(value) { return value === "running" || value === "ok" || value === "error" || value === "skipped" || value === "interrupted" || value === "superseded"; } function receiptFromRow(row) { if (!isReceiptStatus(row.status)) throw new Error(`invalid cron run receipt status ${row.status}`); return { receiptId: row.receipt_id, storeKey: row.store_key, jobId: row.job_id, configRevision: row.config_revision, agentId: row.agent_id, ...row.request_run_id ? { requestRunId: row.request_run_id } : {}, status: row.status, ownerPid: row.owner_pid, ownerStartTime: row.owner_start_time, startedAtMs: row.started_at_ms, finishedAtMs: row.finished_at_ms, ...row.error_text ? { error: row.error_text } : {} }; } function currentJob(database, storeKey, jobId) { const rows = loadCronRows(database, storeKey, /* @__PURE__ */ new Set([jobId])); return loadedCronStoreFromRows(rows).store.jobs[0]; } function sameOwner(left, right) { return left.receipt_id === right.receiptId && left.owner_pid === right.ownerPid && left.owner_start_time === right.ownerStartTime && left.started_at_ms === right.startedAtMs; } function observeOwner(row) { return { receiptId: row.receipt_id, ownerPid: row.owner_pid, ownerStartTime: row.owner_start_time, startedAtMs: row.started_at_ms }; } function ownerStale(owner, nowMs = Date.now()) { if (owner.ownerPid === process.pid) return !locallyOwnedReceipts.has(owner.receiptId); if (isPidDefinitelyDead(owner.ownerPid)) return true; const observedStartTime = getFileLockProcessStartTime(owner.ownerPid); if (owner.ownerStartTime !== null && observedStartTime !== null) return owner.ownerStartTime !== observedStartTime; return nowMs - owner.startedAtMs > CRON_STUCK_RUN_MS; } function validateCurrentJob(params) { const job = currentJob(params.database, params.handle.storeKey, params.handle.jobId); if (!job) throw new CronRunReceiptRevisionError(params.handle.receiptId, "cron job was removed"); if (params.resolveAgentId(job) !== params.handle.agentId) throw new CronRunReceiptRevisionError(params.handle.receiptId); return job; } function receiptHandle(receipt) { return { receiptId: receipt.receiptId, storeKey: receipt.storeKey, jobId: receipt.jobId, configRevision: receipt.configRevision, agentId: receipt.agentId, ownerPid: receipt.ownerPid, ownerStartTime: receipt.ownerStartTime, startedAtMs: receipt.startedAtMs }; } function pruneTerminalReceipts(database, storeKey, jobId) { const terminalIds = executeSqliteQuerySync(database, query(database).selectFrom("cron_run_receipts").select("receipt_id").where("store_key", "=", storeKey).where("job_id", "=", jobId).where("status", "!=", "running").orderBy("finished_at_ms", "desc").orderBy("started_at_ms", "desc").orderBy("receipt_id", "desc")).rows.slice(CRON_RUN_RECEIPT_TERMINAL_RETENTION); for (let index = 0; index < terminalIds.length; index += CRON_RUN_RECEIPT_DELETE_BATCH_SIZE) { const receiptIds = terminalIds.slice(index, index + CRON_RUN_RECEIPT_DELETE_BATCH_SIZE).map((row) => row.receipt_id); deleteExecutionOwnerLifecycleMetadata({ db: database, ownerKind: "cron", ownerIds: receiptIds }); executeSqliteQuerySync(database, query(database).deleteFrom("cron_run_receipts").where("store_key", "=", storeKey).where("job_id", "=", jobId).where("status", "!=", "running").where("receipt_id", "in", receiptIds)); } } /** Prepares process liveness facts before the caller enters its commit transaction. */ function prepareCronRunReceiptAdjudication(params) { const storeKey = cronStoreKey(params.storePath); const observed = withReceiptWrite("cron.run-receipt.inspect", params.env ? { env: params.env } : {}, (database) => activeRow(database, storeKey, params.jobId)); return { storeKey, ...observed ? { observed: observeOwner(observed) } : {}, observedStale: observed ? ownerStale(observeOwner(observed), params.nowMs) : false }; } function prepareCronRunReceiptClaim(params) { const ownerStartTime = getFileLockProcessStartTime(process.pid); if (ownerStartTime === null) throw new Error("cron run cannot acquire a durable fence without process start identity"); const adjudication = prepareCronRunReceiptAdjudication({ storePath: params.storePath, jobId: params.job.id, nowMs: params.startedAtMs, env: params.env }); const storeKey = cronStoreKey(params.storePath); return { handle: { receiptId: crypto.randomUUID(), storeKey, jobId: params.job.id, configRevision: resolveCronJobConfigRevision(params.job), agentId: params.agentId, ownerPid: process.pid, ownerStartTime, startedAtMs: params.startedAtMs }, ...adjudication, ...params.requestRunId ? { requestRunId: params.requestRunId } : {} }; } /** Rechecks the owner and phase start so activation invalidates an age-based stale decision. */ function adjudicateActiveCronRunReceiptInDatabase(params) { const current = activeRow(params.database, params.prepared.storeKey, params.jobId); if (!current) return; if (params.prepared.observed && params.prepared.observedStale && sameOwner(current, params.prepared.observed)) { executeSqliteQuerySync(params.database, query(params.database).updateTable("cron_run_receipts").set({ status: "interrupted", finished_at_ms: params.finishedAtMs, error_text: "cron: job interrupted because owner is unavailable" }).where("receipt_id", "=", current.receipt_id).where("status", "=", "running")); return; } throw new CronRunReceiptConflictError(receiptFromRow(current)); } /** Claims the receipt inside the caller's synchronous cron-state transaction. */ function claimCronRunReceiptInDatabase(params) { const { handle } = params.prepared; if (handle.ownerStartTime === null) throw new Error("cron run cannot acquire a durable fence without process start identity"); adjudicateActiveCronRunReceiptInDatabase({ database: params.database, jobId: handle.jobId, prepared: params.prepared, finishedAtMs: handle.startedAtMs }); pruneTerminalReceipts(params.database, handle.storeKey, handle.jobId); validateCurrentJob({ database: params.database, handle, resolveAgentId: params.resolveAgentId }); executeSqliteQuerySync(params.database, query(params.database).insertInto("cron_run_receipts").values({ receipt_id: handle.receiptId, store_key: handle.storeKey, job_id: handle.jobId, config_revision: handle.configRevision, agent_id: handle.agentId, request_run_id: params.prepared.requestRunId ?? null, status: "running", owner_pid: handle.ownerPid, owner_start_time: handle.ownerStartTime, started_at_ms: handle.startedAtMs, finished_at_ms: null, error_text: null })); const claimed = receiptHandle(receiptFromRow(activeRow(params.database, handle.storeKey, handle.jobId))); locallyOwnedReceipts.add(claimed.receiptId); return claimed; } function findActiveCronRunReceiptInDatabase(params) { const row = activeRow(params.database, cronStoreKey(params.storePath), params.jobId); return row ? receiptHandle(receiptFromRow(row)) : void 0; } function listActiveCronRunReceiptJobIdsInDatabase(database, storePath) { return new Set(activeRow(database, cronStoreKey(storePath)).map((row) => row.job_id)); } function inspectActiveCronRunReceipt(params) { return withReceiptWrite("cron.run-receipt.recovery-inspect", params.env ? { env: params.env } : {}, (database) => findActiveCronRunReceiptInDatabase({ database, storePath: params.storePath, jobId: params.jobId })); } function isCronRunReceiptOwnerStale(candidate, nowMs = Date.now()) { return ownerStale(candidate, nowMs); } /** Synchronous transaction guard used immediately before a run side effect or state write. */ function assertCronRunReceiptOwnedInDatabase(params) { const current = activeRow(params.database, params.handle.storeKey, params.handle.jobId); if (!current || current.receipt_id !== params.handle.receiptId || current.owner_pid !== params.handle.ownerPid || current.owner_start_time !== params.handle.ownerStartTime) throw new CronRunReceiptRevisionError(params.handle.receiptId, "cron run fence is no longer current"); } /** Synchronous transaction guard used immediately before a run side effect or state write. */ function assertCronRunReceiptCurrentInDatabase(params) { assertCronRunReceiptOwnedInDatabase(params); validateCurrentJob({ database: params.database, handle: params.handle, resolveAgentId: params.resolveAgentId }); } /** Advances a queued lease to its execution start inside the marker transaction. */ function activateCronRunReceiptInDatabase(params) { assertCronRunReceiptCurrentInDatabase(params); executeSqliteQuerySync(params.database, query(params.database).updateTable("cron_run_receipts").set({ started_at_ms: params.startedAtMs }).where("receipt_id", "=", params.handle.receiptId).where("status", "=", "running")); return { ...params.handle, startedAtMs: params.startedAtMs }; } function assertCronRunReceiptCurrent(params) { if (params.isAgentAvailable && !params.isAgentAvailable(params.handle.agentId)) throw new CronRunReceiptRevisionError(params.handle.receiptId, `cron job agent is unavailable: ${params.handle.agentId}`, "owner-unavailable"); withReceiptWrite("cron.run-receipt.assert-current", params.env ? { env: params.env } : {}, (database) => params.allowMissingJob ? assertCronRunReceiptOwnedInDatabase({ database, handle: params.handle }) : assertCronRunReceiptCurrentInDatabase({ database, ...params })); } /** Keeps the durable lease live when timeout/cancel returns before the runner. */ function trackCronRunReceiptSettlement(params) { const receiptId = params.handle.receiptId; const pending = { releaseRequested: false, onFinishError: params.onFinishError }; pendingReceiptSettlements.set(receiptId, pending); const settle = () => { if (pendingReceiptSettlements.get(receiptId) !== pending) return; pendingReceiptSettlements.delete(receiptId); if (pending.finish) try { finishCronRunReceipt(pending.finish); } catch (error) { pending.onFinishError(error); } else if (pending.releaseRequested) locallyOwnedReceipts.delete(receiptId); }; params.settlement.then(settle, settle); } function isCronRunReceiptSettlementPending(handle) { return pendingReceiptSettlements.has(handle.receiptId); } function clearCronRunReceiptFinishRetry(receiptId) { const pending = pendingReceiptFinishRetries.get(receiptId); if (pending?.timer) clearTimeout(pending.timer); pendingReceiptFinishRetries.delete(receiptId); } function queueCronRunReceiptFinishRetry(finish) { const receiptId = finish.handle.receiptId; let pending = pendingReceiptFinishRetries.get(receiptId); if (!pending) { pending = { finish, timer: null }; pendingReceiptFinishRetries.set(receiptId, pending); } if (pending.timer) return; pending.timer = setTimeout(() => { pending.timer = null; try { finishCronRunReceipt(pending.finish); } catch {} }, CRON_RUN_RECEIPT_FINISH_RETRY_MS); pending.timer.unref?.(); } function finishCronRunReceipt(params) { const pending = pendingReceiptSettlements.get(params.handle.receiptId); if (pending) { pending.finish ??= params; return; } try { const result = withReceiptWrite("cron.run-receipt.finish", params.env ? { env: params.env } : {}, (database) => finishCronRunReceiptInDatabase({ database, ...params })); clearCronRunReceiptFinishRetry(params.handle.receiptId); locallyOwnedReceipts.delete(params.handle.receiptId); return result; } catch (error) { queueCronRunReceiptFinishRetry(params); throw error; } } /** Releases only this process's liveness proof after terminal persistence fails. */ function releaseLocalCronRunReceiptOwnership(handle) { const pending = pendingReceiptSettlements.get(handle.receiptId); if (pending) { pending.releaseRequested = true; return; } if (pendingReceiptFinishRetries.has(handle.receiptId)) return; locallyOwnedReceipts.delete(handle.receiptId); } /** Completes the exact active receipt inside its caller's cron-state transaction. */ function finishCronRunReceiptInDatabase(params) { executeSqliteQuerySync(params.database, query(params.database).updateTable("cron_run_receipts").set({ status: params.status, finished_at_ms: params.finishedAtMs, error_text: params.error ?? null }).where("receipt_id", "=", params.handle.receiptId).where("status", "=", "running").where("owner_pid", "=", params.handle.ownerPid)); pruneTerminalReceipts(params.database, params.handle.storeKey, params.handle.jobId); const row = executeSqliteQueryTakeFirstSync(params.database, query(params.database).selectFrom("cron_run_receipts").selectAll().where("receipt_id", "=", params.handle.receiptId)); return row ? receiptFromRow(row) : void 0; } //#endregion //#region src/cron/service/wake.ts function enqueueCronSystemEvent(state, text, opts) { return state.deps.enqueueSystemEvent(text, opts); } function requestCronHeartbeat(state, opts, retry) { if (retry) { state.deps.requestHeartbeat({ source: "cron", ...opts }, retry); return; } state.deps.requestHeartbeat({ source: "cron", ...opts }); } /** Keeps safety notices with their creator and limits failure routes to explicit origins. */ function enqueueCronNotification(state, job, text, kind) { const sessionKey = kind === "failure-alert" ? resolveCronDeliverySessionKey(job) : job.sessionKey; const agentId = normalizeOptionalAgentId(job.agentId) ?? normalizeOptionalAgentId(parseAgentSessionKey(sessionKey)?.agentId) ?? normalizeOptionalAgentId(state.deps.resolveDefaultAgentId?.()) ?? normalizeOptionalAgentId(state.deps.defaultAgentId); const deliveryContext = sessionKey || kind === "auto-disabled" && agentId ? state.deps.resolveOriginDeliveryContext?.({ agentId, sessionKey }) : void 0; enqueueCronSystemEvent(state, text, { agentId, sessionKey, contextKey: `cron:${job.id}:${kind}`, ...deliveryContext ? { deliveryContext } : {} }); if (kind === "auto-disabled" || job.wakeMode === "now" || sessionKey) requestCronHeartbeat(state, { source: "notifications-event", intent: "immediate", reason: "wake", agentId, sessionKey }); } /** Enqueues a manual cron wake event and optionally pokes the targeted heartbeat loop. */ function wake(state, opts) { const text = opts.text.trim(); if (!text) return { ok: false }; const sessionKey = opts.sessionKey?.trim() || void 0; const agentId = opts.agentId?.trim() || void 0; if (sessionKey && isSubagentSessionKey(sessionKey)) return { ok: false, reason: "unwakeable-session-key" }; const originDeliveryContext = sessionKey || agentId ? state.deps.resolveOriginDeliveryContext?.({ sessionKey, agentId }) : void 0; enqueueCronSystemEvent(state, text, sessionKey || agentId ? { ...sessionKey ? { sessionKey } : {}, ...agentId ? { agentId } : {}, ...originDeliveryContext ? { deliveryContext: originDeliveryContext } : {} } : void 0); if (opts.mode === "now" || sessionKey) requestCronHeartbeat(state, { source: "manual", intent: "immediate", reason: "wake", ...sessionKey ? { sessionKey } : {}, ...agentId ? { agentId } : {} }); return { ok: true }; } //#endregion //#region src/cron/service/auto-disable.ts /** Shared state and owner-notification policy for cron auto-disable transitions. */ /** * Run failures get more room than schedule errors (10 vs. 3) because provider * and network errors are often transient, and restart-interrupted runs count too. */ const MAX_CONSECUTIVE_RUN_FAILURES = 10; function autoDisableReasonLabel(reason) { return reason === "consecutive-failures" ? "run failures" : "schedule errors"; } /** Records one canonical auto-disable fact and queues its owning-agent notification. */ function autoDisableCronJob(params) { const { state, job } = params; if (isSystemOwnedCronPayloadKind(job.payload.kind)) return false; if (!job.enabled || job.state.autoDisabled) return false; job.enabled = false; job.state.nextRunAtMs = void 0; job.state.autoDisabled = { reason: params.reason, atMs: params.atMs, consecutiveErrors: params.consecutiveErrors }; const name = truncateUtf16Safe((job.name || job.id).replace(/\s+/g, " ").trim(), 120); const errorReason = params.reason === "consecutive-failures" ? job.state.lastErrorReason : void 0; const text = [ `⚠️ Automation "${name}" was auto-disabled after ${params.consecutiveErrors} consecutive ${autoDisableReasonLabel(params.reason)}.`, ...cronFailureDetailLines(errorReason), `Fix the underlying cause, then run \`openclaw automations enable ${job.id}\` to re-enable it.` ].join("\n"); const notify = () => enqueueCronNotification(state, job, text, "auto-disabled"); if (params.deferredNotifications) params.deferredNotifications.push(notify); else notify(); return true; } /** Auto-disables only time-based recurring jobs once their run-error streak reaches the limit. */ function maybeAutoDisableCronJobAfterRunFailure(params) { const consecutiveErrors = params.job.state.consecutiveErrors ?? 0; if (params.job.schedule.kind !== "cron" && params.job.schedule.kind !== "every" || consecutiveErrors < MAX_CONSECUTIVE_RUN_FAILURES) return false; return autoDisableCronJob({ ...params, reason: "consecutive-failures", consecutiveErrors }); } //#endregion //#region src/cron/service/jobs-scheduling.ts /** Scheduling state and next-run computation for cron jobs. */ const staggerOffsetCache = /* @__PURE__ */ new Map(); function ownsCronRunMarker(state, jobId, markerAtMs, requireForce = false) { const reservation = state.queuedRunReservationsByJobId.get(jobId); return reservation?.markerAtMs === markerAtMs && (!requireForce || reservation.preserveWhenDisabled); } function normalizeStreamScheduleBounds(schedule) { if (schedule.kind !== "stream") return schedule; const resolved = resolveCronStreamBatching(schedule); return { ...schedule, ...schedule.batchMs !== void 0 ? { batchMs: resolved.batchMs } : {}, ...schedule.maxBatchBytes !== void 0 ? { maxBatchBytes: resolved.maxBatchBytes } : {} }; } /** Default retry delays applied after consecutive cron execution errors. */ const DEFAULT_ERROR_BACKOFF_SCHEDULE_MS = [ 3e4, 6e4, 3e5, 9e5, 36e5 ]; function isFiniteTimestamp(value) { return asDateTimestampMs(value) !== void 0; } /** Returns whether a stored next-run timestamp is finite and schedulable. */ function hasScheduledNextRunAtMs(value) { return isFiniteTimestamp(value) && value > 0; } /** Resolves the newest persisted cron run status while older state is still readable. */ function resolveJobLastRunStatus(job) { return job.state.lastRunStatus ?? job.state.lastStatus; } /** Resolves the retry backoff delay for a one-based consecutive error count. */ function errorBackoffMs(consecutiveErrors, scheduleMs = DEFAULT_ERROR_BACKOFF_SCHEDULE_MS) { const idx = Math.min(consecutiveErrors - 1, scheduleMs.length - 1); return expectDefined(scheduleMs[Math.max(0, idx)], "schedule ms entry at math.max(0, idx)") ?? DEFAULT_ERROR_BACKOFF_SCHEDULE_MS[0]; } /** Returns the earliest retry timestamp after a failed cron run and its runtime duration. */ function resolveJobErrorBackoffUntilMs(job, scheduleMs = DEFAULT_ERROR_BACKOFF_SCHEDULE_MS) { if (resolveJobLastRunStatus(job) !== "error" || !isFiniteTimestamp(job.state.lastRunAtMs)) return; const consecutiveErrorsRaw = job.state.consecutiveErrors; const consecutiveErrors = typeof consecutiveErrorsRaw === "number" && Number.isFinite(consecutiveErrorsRaw) ? Math.max(1, Math.floor(consecutiveErrorsRaw)) : 1; const lastDurationMs = typeof job.state.lastDurationMs === "number" && Number.isFinite(job.state.lastDurationMs) ? Math.max(0, Math.floor(job.state.lastDurationMs)) : 0; const lastEndedAtMs = job.state.lastRunAtMs + lastDurationMs; return asDateTimestampMs(lastEndedAtMs + errorBackoffMs(consecutiveErrors, scheduleMs)); } function resolveStableCronOffsetMs(jobId, staggerMs) { if (staggerMs <= 1) return 0; const cacheKey = `${staggerMs}:${jobId}`; const cached = staggerOffsetCache.get(cacheKey); if (cached !== void 0) return cached; const offset = crypto.createHash("sha256").update(jobId).digest().readUInt32BE(0) % staggerMs; pruneMapToMaxSize(staggerOffsetCache, 4095); staggerOffsetCache.set(cacheKey, offset); return offset; } function computeStaggeredCronNextRunAtMs(job, nowMs) { if (job.schedule.kind !== "cron") return computeNextRunAtMs(job.schedule, nowMs); const staggerMs = resolveCronStaggerMs(job.schedule); const offsetMs = resolveStableCronOffsetMs(job.id, staggerMs); if (offsetMs <= 0) return computeNextRunAtMs(job.schedule, nowMs); let cursorMs = Math.max(0, nowMs - offsetMs); for (let attempt = 0; attempt < 4; attempt += 1) { const baseNext = computeNextRunAtMs(job.schedule, cursorMs); if (baseNext === void 0) return; const shifted = baseNext + offsetMs; if (isFiniteTimestamp(shifted) && shifted > nowMs) return shifted; cursorMs = Math.max(cursorMs + 1, baseNext + 1e3); } } function computeStaggeredCronPreviousRunAtMs(job, nowMs) { if (job.schedule.kind !== "cron") return; const staggerMs = resolveCronStaggerMs(job.schedule); const offsetMs = resolveStableCronOffsetMs(job.id, staggerMs); if (offsetMs <= 0) return computePreviousRunAtMs(job.schedule, nowMs); let cursorMs = Math.max(0, nowMs - offsetMs); for (let attempt = 0; attempt < 4; attempt += 1) { const basePrevious = computePreviousRunAtMs(job.schedule, cursorMs); if (basePrevious === void 0) return; const shifted = basePrevious + offsetMs; if (isFiniteTimestamp(shifted) && shifted <= nowMs) return shifted; cursorMs = Math.max(0, basePrevious - 1e3); } } function computeStaggeredCronPreviousRunAtOrBeforeMs(job, nowMs) { const previous = computeStaggeredCronPreviousRunAtMs(job, nowMs); const probeMs = nowMs + 1e3; if (!isFiniteTimestamp(probeMs)) return previous; const boundary = computeStaggeredCronPreviousRunAtMs(job, probeMs); if (isFiniteTimestamp(boundary) && boundary <= nowMs && (!isFiniteTimestamp(previous) || boundary > previous)) return boundary; return previous; } function isStaggeredCronRunAtMs(job, runAtMs) { if (job.schedule.kind !== "cron" || !isFiniteTimestamp(runAtMs)) return false; return computeStaggeredCronPreviousRunAtOrBeforeMs(job, runAtMs) === runAtMs; } function isPendingErrorBackoffSlot(params) { const { job, nextRunAtMs, nowMs } = params; const backoffUntilMs = resolveJobErrorBackoffUntilMs(job, DEFAULT_ERROR_BACKOFF_SCHEDULE_MS); return backoffUntilMs !== void 0 && nowMs < backoffUntilMs && nextRunAtMs <= backoffUntilMs; } function isStaleFutureCronSlot(job, nowMs) { const nextRun = job.state.nextRunAtMs; if (job.schedule.kind !== "cron" || !hasScheduledNextRunAtMs(nextRun) || nowMs >= nextRun || typeof job.state.queuedAtMs === "number" || typeof job.state.runningAtMs === "number") return false; if (isPendingErrorBackoffSlot({ job, nextRunAtMs: nextRun, nowMs })) return false; let naturalNext; try { naturalNext = computeStaggeredCronNextRunAtMs(job, nowMs); } catch { return false; } if (!isFiniteTimestamp(naturalNext)) return false; let isScheduledSlot; try { isScheduledSlot = isStaggeredCronRunAtMs(job, nextRun); } catch { return false; } if (isScheduledSlot) return false; if (nextRun < naturalNext) return job.payload.kind !== "agentTurn"; if (nextRun === naturalNext) return false; let followingNaturalNext; try { followingNaturalNext = computeStaggeredCronNextRunAtMs(job, naturalNext); } catch { return false; } if (!isFiniteTimestamp(followingNaturalNext)) return false; const naturalIntervalMs = followingNaturalNext - naturalNext; return naturalIntervalMs > 0 && nextRun >= followingNaturalNext + naturalIntervalMs; } function resolveEveryAnchorMs(params) { const coerced = coerceFiniteScheduleNumber(params.schedule.anchorMs); if (coerced !== void 0) return Math.max(0, Math.floor(coerced)); if (isFiniteTimestamp(params.fallbackAnchorMs)) return Math.max(0, Math.floor(params.fallbackAnchorMs)); return 0; } function hasInvalidExplicitEveryAnchor(schedule) { if (schedule.anchorMs === void 0) return false; const coerced = coerceFiniteScheduleNumber(schedule.anchorMs); return coerced === void 0 || coerced < 0; } /** Finds an in-memory cron job or throws the public unknown-id error. */ function findJobOrThrow(state, id) { const job = state.store?.jobs.find((j) => j.id === id); if (!job) throw new Error(`unknown cron job id: ${id}`); return job; } /** Returns the effective enabled flag, defaulting missing values to enabled. */ function isJobEnabled(job) { return job.enabled ?? true; } /** Computes the next run timestamp for enabled jobs across every/at/cron schedules. */ function computeJobNextRunAtMs(job, nowMs) { if (!isJobEnabled(job)) return; if (job.schedule.kind === "every") { if (hasInvalidExplicitEveryAnchor(job.schedule)) return; const everyMsRaw = coerceFiniteScheduleNumber(job.schedule.everyMs); if (everyMsRaw === void 0) return; const everyMs = Math.floor(everyMsRaw); if (everyMs < 1) return; const fallbackAnchorMs = isFiniteTimestamp(job.createdAtMs) ? job.createdAtMs : nowMs; const anchorMs = resolveEveryAnchorMs({ schedule: job.schedule, fallbackAnchorMs }); const lastRunAtMs = job.state.lastRunAtMs; if (isFiniteTimestamp(lastRunAtMs) && lastRunAtMs >= anchorMs) { const nextFromLastRun = Math.floor(lastRunAtMs) + everyMs; if (!isFiniteTimestamp(nextFromLastRun)) return; if (nextFromLastRun > nowMs) return nextFromLastRun; } const next = computeNextRunAtMs({ ...job.schedule, everyMs, anchorMs }, nowMs); return isFiniteTimestamp(next) ? next : void 0; } if (job.schedule.kind === "at") { const atMs = parseAbsoluteTimeMs(job.schedule.at); if (resolveJobLastRunStatus(job) === "ok" && job.state.lastRunAtMs) { if (atMs !== null && Number.isFinite(atMs) && atMs > job.state.lastRunAtMs) return atMs; return; } return atMs !== null && Number.isFinite(atMs) ? atMs : void 0; } const next = computeStaggeredCronNextRunAtMs(job, nowMs); if (next === void 0 && job.schedule.kind === "cron") return computeStaggeredCronNextRunAtMs(job, Math.floor(nowMs / 1e3) * 1e3 + 1e3); return isFiniteTimestamp(next) ? next : void 0; } /** Computes the latest effective cron timestamp at or before the supplied time. */ function computeJobPreviousRunAtOrBeforeMs(job, nowMs) { if (!isJobEnabled(job) || job.schedule.kind !== "cron") return; return asDateTimestampMs(computeStaggeredCronPreviousRunAtOrBeforeMs(job, nowMs)); } /** Maximum consecutive schedule errors before auto-disabling a job. */ const MAX_SCHEDULE_ERRORS = 3; /** Records a schedule-computation failure and auto-disables after repeated errors. */ function recordScheduleComputeError(params) { const { state, job, err } = params; const errorCount = (job.state.scheduleErrorCount ?? 0) + 1; const errText = formatErrorMessageWithCode(err); job.state.scheduleErrorCount = errorCount; job.state.nextRunAtMs = void 0; job.state.lastError = `schedule error: ${errText}`; if (errorCount >= MAX_SCHEDULE_ERRORS) { autoDisableCronJob({ state, job, reason: "schedule-errors", atMs: state.deps.nowMs(), consecutiveErrors: errorCount, deferredNotifications: params.deferredNotifications }); state.deps.log.error({ jobId: job.id, name: job.name, errorCount, err: errText }, "cron: auto-disabled job after repeated schedule errors"); } else state.deps.log.warn({ jobId: job.id, name: job.name, errorCount, err: errText }, "cron: failed to compute next run for job (skipping)"); return true; } function normalizeJobTickState(params) { const { state, job, nowMs } = params; let changed = false; if (!job.state) { job.state = {}; changed = true; } if (job.schedule.kind === "stream" && !job.state.streamSourceIdentity?.trim()) { job.state.streamSourceIdentity = createCronStreamSourceIdentity(); changed = true; } if (job.schedule.kind === "every" && !hasInvalidExplicitEveryAnchor(job.schedule)) { const normalizedAnchorMs = resolveEveryAnchorMs({ schedule: job.schedule, fallbackAnchorMs: isFiniteTimestamp(job.createdAtMs) ? job.createdAtMs : nowMs }); if (job.schedule.anchorMs !== normalizedAnchorMs) { job.schedule = { ...job.schedule, anchorMs: normalizedAnchorMs }; job.state.pacedNextRunAtMs = void 0; job.state.forcePreservedNextRunAtMs = void 0; changed = true; } } if (!isJobEnabled(job)) { for (const key of [ "startupCatchupAtMs", "pacedNextRunAtMs", "forcePreservedNextRunAtMs", "nextRunAtMs" ]) if (job.state[key] !== void 0) { job.state[key] = void 0; changed = true; } if (job.state.queuedAtMs !== void 0 && !ownsCronRunMarker(state, job.id, job.state.queuedAtMs, true)) { job.state.queuedAtMs = void 0; changed = true; } if (job.state.runningAtMs !== void 0 && !ownsCronRunMarker(state, job.id, job.state.runningAtMs, true) && !isCronJobActive(job.id)) { job.state.runningAtMs = void 0; changed = true; } return { changed, skip: true }; } if (!hasScheduledNextRunAtMs(job.state.nextRunAtMs) && job.state.nextRunAtMs !== void 0) { job.state.nextRunAtMs = void 0; changed = true; } const forcePreservedNextRunAtMs = job.state.forcePreservedNextRunAtMs; if (forcePreservedNextRunAtMs !== void 0 && (!isFiniteTimestamp(forcePreservedNextRunAtMs) || forcePreservedNextRunAtMs !== job.state.nextRunAtMs)) { job.state.forcePreservedNextRunAtMs = void 0; changed = true; } const queuedAt = job.state.queuedAtMs; if (typeof queuedAt === "number" && Math.abs(nowMs - queuedAt) > 72e5 && !ownsCronRunMarker(state, job.id, queuedAt)) { state.deps.log.warn({ jobId: job.id, queuedAtMs: queuedAt }, "cron: clearing stuck queued marker"); job.state.queuedAtMs = void 0; changed = true; } const runningAt = job.state.runningAtMs; if (typeof runningAt === "number" && Math.abs(nowMs - runningAt) > 72e5 && !ownsCronRunMarker(state, job.id, runningAt)) { state.deps.log.warn({ jobId: job.id, runningAtMs: runningAt }, "cron: clearing stuck running marker"); job.state.runningAtMs = void 0; changed = true; const nextRun = job.state.nextRunAtMs; const lastRun = job.state.lastRunAtMs; const alreadyExecutedSlot = hasScheduledNextRunAtMs(nextRun) && isFiniteTimestamp(lastRun) && lastRun >= nextRun; return { changed, skip: !alreadyExecutedSlot }; } return { changed, skip: false }; } function walkSchedulableJobs(state, fn, nowMs = state.deps.nowMs()) { if (!state.store) return false; let changed = false; for (const job of state.store.jobs) { const tick = normalizeJobTickState({ state, job, nowMs }); if (tick.changed) changed = true; if (tick.skip) continue; if (fn({ job, nowMs })) changed = true; } return changed; } function recomputeJobNextRunAtMs(params) { let changed = false; try { let newNext = computeJobNextRunAtMs(params.job, params.nowMs); if (params.job.schedule.kind !== "at" && resolveJobLastRunStatus(params.job) === "error" && isFiniteTimestamp(params.job.state.lastRunAtMs)) { const backoffFloor = resolveJobErrorBackoffUntilMs(params.job, DEFAULT_ERROR_BACKOFF_SCHEDULE_MS); if (newNext !== void 0) newNext = backoffFloor !== void 0 ? Math.max(newNext, backoffFloor) : newNext; } if (params.job.state.nextRunAtMs !== newNext) { params.job.state.nextRunAtMs = newNext; changed = true; } if (params.job.state.scheduleErrorCount) { params.job.state.scheduleErrorCount = void 0; changed = true; } } catch (err) { if (params.skipScheduleErrorHandling) return false; if (recordScheduleComputeError({ state: params.state, job: params.job, err, deferredNotifications: params.deferredNotifications })) changed = true; } return changed; } /** Recomputes missing, due, or repairable next-run timestamps for all schedulable jobs. */ function recomputeNextRuns(state) { return walkSchedulableJobs(state, ({ job, nowMs: now }) => { const nextRun = job.state.nextRunAtMs; const hasForcePreservedNextRun = isFiniteTimestamp(job.state.forcePreservedNextRunAtMs) && hasScheduledNextRunAtMs(nextRun) && job.state.forcePreservedNextRunAtMs === nextRun; const isDueOrMissing = !hasScheduledNextRunAtMs(nextRun) || now >= nextRun; return !hasForcePreservedNextRun && (isDueOrMissing || isStaleFutureCronSlot(job, now)) && recomputeJobNextRunAtMs({ state, job, nowMs: now }); }); } function isExpiredCronScheduleRepairCandidate(job, nowMs) { const nextRunAtMs = job.state.nextRunAtMs; const backoffUntilMs = resolveJobErrorBackoffUntilMs(job, DEFAULT_ERROR_BACKOFF_SCHEDULE_MS); return hasScheduledNextRunAtMs(nextRunAtMs) && nowMs >= nextRunAtMs && !hasActiveCronRun(job) && job.state.startupCatchupAtMs !== nextRunAtMs && job.state.forcePreservedNextRunAtMs !== nextRunAtMs && (isFiniteTimestamp(job.state.lastRunAtMs) && job.state.lastRunAtMs >= nextRunAtMs || backoffUntilMs !== void 0 && nowMs < backoffUntilMs && nextRunAtMs < backoffUntilMs); } function needsCronTimerMaintenance(job, nowMs) { return isExpiredCronScheduleRepairCandidate(job, nowMs) || isStaleFutureCronSlot(job, nowMs) || job.schedule.kind !== "stream" && job.schedule.kind !== "on-exit" && isJobEnabled(job) && !hasScheduledNextRunAtMs(job.state.nextRunAtMs) && !hasActiveCronRun(job); } function recomputeSingleJobForMaintenance(state, job, opts) { const now = opts?.nowMs ?? state.deps.nowMs(); const tick = normalizeJobTickState({ state, job, nowMs: now }); let changed = tick.changed; if (tick.skip) return changed; const recomputeExpired = opts?.recomputeExpired ?? false; const repairFutureCronNextRunAtMs = opts?.repairFutureCronNextRunAtMs ?? true; const recomputeJob = () => recomputeJobNextRunAtMs({ state, job, nowMs: now, deferredNotifications: opts?.deferredNotifications, skipScheduleErrorHandling: opts?.skipScheduleErrorHandling }); const startupCatchupAtMs = job.state.startupCatchupAtMs; const pacedNextRunAtMs = job.state.pacedNextRunAtMs; const nextRunAtMs = job.state.nextRunAtMs; const hasForcePreservedNextRun = isFiniteTimestamp(job.state.forcePreservedNextRunAtMs) && hasScheduledNextRunAtMs(nextRunAtMs) && job.state.forcePreservedNextRunAtMs === nextRunAtMs; const hasPendingStartupCatchup = isFiniteTimestamp(startupCatchupAtMs) && hasScheduledNextRunAtMs(nextRunAtMs) && startupCatchupAtMs === nextRunAtMs; if (startupCatchupAtMs !== void 0 && !hasPendingStartupCatchup) { job.state.startupCatchupAtMs = void 0; changed = true; } const hasPendingPacedNextRun = isFiniteTimestamp(pacedNextRunAtMs) && hasScheduledNextRunAtMs(nextRunAtMs) && pacedNextRunAtMs === nextRunAtMs && (now < pacedNextRunAtMs || opts?.preserveExpiredPacedNextRunJobId === job.id); if (pacedNextRunAtMs !== void 0 && !hasPendingPacedNextRun) { job.state.pacedNextRunAtMs = void 0; changed = true; } if (!hasScheduledNextRunAtMs(job.state.nextRunAtMs)) changed = recomputeJob() || changed; else if (repairFutureCronNextRunAtMs && !hasPendingStartupCatchup && !hasPendingPacedNextRun && !hasForcePreservedNextRun && isStaleFutureCronSlot(job, now)) changed = recomputeJob() || changed; else if (recomputeExpired && isExpiredCronScheduleRepairCandidate(job, now)) changed = recomputeJob() || changed; return changed; } function recomputeNextRunsForMaintenance(state, opts) { if (!state.store) return false; let changed = false; for (const job of state.store.jobs) changed = recomputeSingleJobForMaintenance(state, job, opts) || changed; return changed; } /** Summarizes the resident job schedule in one pass for timer and status projections. */ function summarizeCronJobSchedule(state) { const jobs = state.store?.jobs ?? []; let nextWake; let jobCount = 0; let enabledCount = 0; for (const job of jobs) { jobCount += 1; const nextRun = job.state.nextRunAtMs; const hasNextRun = hasScheduledNextRunAtMs(nextRun); const rawEnabled = job.enabled; if (rawEnabled) enabledCount += 1; if ((rawEnabled ?? true) && hasNextRun) nextWake = nextWake === void 0 ? nextRun : Math.min(nextWake, nextRun); } return { jobCount, enabledCount, nextWakeAtMs: nextWake }; } /** Returns the next enabled wake timestamp from the in-memory cron store. */ function nextWakeAtMs(state) { return summarizeCronJobSchedule(state).nextWakeAtMs; } /** Applies one canonical server-authored authority envelope to a tool-bearing job. */ function hasActiveCronRun(job) { return typeof job.state.queuedAtMs === "number" || typeof job.state.runningAtMs === "number" || isCronJobActive(job.id); } /** Returns whether a cron job should execute at `nowMs`, honoring force mode and active runs. */ function isJobDue(job, nowMs, opts) { if (!job.state) job.state = {}; if (hasActiveCronRun(job)) return false; if (opts.forced) return true; return isJobEnabled(job) && hasScheduledNextRunAtMs(job.state.nextRunAtMs) && nowMs >= job.state.nextRunAtMs; } /** Returns main-session queue text for system-event jobs, or undefined when empty/unsupported. */ function resolveJobPayloadTextForMain(job) { if (job.payload.kind !== "systemEvent") return; const text = normalizePayloadToSystemText(job.payload); return text.trim() ? text : void 0; } //#endregion export { CronRunReceiptConflictError as A, finishCronRunReceipt as B, summarizeCronJobSchedule as C, enqueueCronSystemEvent as D, enqueueCronNotification as E, assertCronRunReceiptCurrentInDatabase as F, listActiveCronRunReceiptJobIdsInDatabase as G, inspectActiveCronRunReceipt as H, assertCronRunReceiptOwnedInDatabase as I, releaseLocalCronRunReceiptOwnership as J, prepareCronRunReceiptAdjudication as K, bindCronRunReceiptExecution as L, activateCronRunReceiptInDatabase as M, adjudicateActiveCronRunReceiptInDatabase as N, requestCronHeartbeat as O, assertCronRunReceiptCurrent as P, claimCronRunReceiptInDatabase as R, resolveJobPayloadTextForMain as S, maybeAutoDisableCronJobAfterRunFailure as T, isCronRunReceiptOwnerStale as U, finishCronRunReceiptInDatabase as V, isCronRunReceiptSettlementPending as W, cronFailureDetailLines as X, trackCronRunReceiptSettlement as Y, recomputeSingleJobForMaintenance as _, findJobOrThrow as a, resolveJobErrorBackoffUntilMs as b, isJobDue as c, needsCronTimerMaintenance as d, nextWakeAtMs as f, recomputeNextRunsForMaintenance as g, recomputeNextRuns as h, errorBackoffMs as i, CronRunReceiptRevisionError as j, wake as k, isJobEnabled as l, recomputeJobNextRunAtMs as m, computeJobNextRunAtMs as n, hasActiveCronRun as o, normalizeStreamScheduleBounds as p, prepareCronRunReceiptClaim as q, computeJobPreviousRunAtOrBeforeMs as r, hasScheduledNextRunAtMs as s, DEFAULT_ERROR_BACKOFF_SCHEDULE_MS as t, isStaleFutureCronSlot as u, recordScheduleComputeError as v, autoDisableCronJob as w, resolveJobLastRunStatus as x, resolveEveryAnchorMs as y, findActiveCronRunReceiptInDatabase as z };