openclaw
Version:
Multi-channel AI gateway with extensible messaging integrations
551 lines (550 loc) • 19.2 kB
JavaScript
import { d as toErrorObject } from "./error-coercion-W9BtigRm.mjs";
import { t as createDeferredCore } from "./deferred-D0La5CRk.mjs";
import { t as sleep } from "./sleep-dMoNLoKC.mjs";
import { t as formatErrorMessage } from "./errors-BvOkmQM5.mjs";
import { l as hasSqliteWorkerOutcomeUnknown } from "./sqlite-worker-contract-C51cJHzg.mjs";
import { P as waitForGatewayRestartFenceSettlement, _ as onGatewaySuspendAdmissionChange, d as getGatewaySuspendAdmissionPhase, l as getGatewayRestartDrainSignal, p as isGatewayRestartDraining } from "./gateway-work-admission-DAV5kkgV.mjs";
import { n as createChannelIngressDrain } from "./ingress-drain-BOq8s3i5.mjs";
import { t as ChannelIngressUnavailableError } from "./ingress-unavailable-VNzzJV3Q.mjs";
//#region src/channels/message/ingress-monitor-tasks.ts
/** Join the monitor-owned task collection, including work added while awaiting it. */
async function waitForPending(read, reject = false) {
for (;;) {
const pending = [...read()];
if (pending.length === 0) return;
await (reject ? Promise.all(pending) : Promise.allSettled(pending));
}
}
/** Serialize admission and claim work without deferring the first task. */
function createAdmissionClaimLock() {
let admissionClaimLocked = false;
const admissionClaimWaiters = [];
return (task) => {
const run = () => {
admissionClaimLocked = true;
let result;
try {
result = Promise.resolve(task());
} catch (error) {
result = Promise.reject(toErrorObject(error, "Channel ingress admission task failed"));
}
return result.finally(() => {
const next = admissionClaimWaiters.shift();
if (next) next();
else admissionClaimLocked = false;
});
};
if (!admissionClaimLocked) return run();
return new Promise((resolve, reject) => {
admissionClaimWaiters.push(() => {
run().then(resolve, reject);
});
});
};
}
//#endregion
//#region src/channels/message/ingress-monitor.ts
/** Shared durable channel-ingress admission, pump, retention, and shutdown lifecycle. */
const DEFAULT_APPEND_RETRY_DELAYS_MS = [
0,
100,
300
];
/** Replay-guard retention defaults; changing a value requires a per-channel keyspace audit. */
const CHANNEL_INGRESS_RETENTION_DEFAULTS = Object.freeze({
pruneIntervalMs: 36e5,
completedTtlMs: 2592e6,
completedMaxEntries: 2e4,
failedTtlMs: 2592e6,
failedMaxEntries: 2e4
});
/**
* Creates the shared monitor around a durable queue and ingress drain.
* Channel code keeps transport inspection, payload shape, and delivery policy.
*/
function createChannelIngressMonitor(options) {
const now = options.now ?? Date.now;
const waitForDeliveryIdleBeforeRepump = options.waitForDeliveryIdleBeforeRepump ?? false;
const { pruneIntervalMs, ...pruneOptions } = options.retention === "standard" ? CHANNEL_INGRESS_RETENTION_DEFAULTS : {
...CHANNEL_INGRESS_RETENTION_DEFAULTS,
...options.retention
};
const shutdown = new AbortController();
const drainAbortSignal = options.abortSignal ? AbortSignal.any([shutdown.signal, options.abortSignal]) : shutdown.signal;
const activeDeliveries = /* @__PURE__ */ new Set();
const activeInspections = /* @__PURE__ */ new Set();
const deferredStartCapacityLimit = options.drain?.deferredLaneOccupancy === "release" ? options.drain.startLimit ?? 0 : 0;
let deferredStartCapacity = 0;
const deferredClaims = /* @__PURE__ */ new Set();
const suppliedQueue = options.queue;
const queueFactory = typeof suppliedQueue === "function" ? suppliedQueue : () => suppliedQueue;
let queue = typeof suppliedQueue === "function" ? void 0 : suppliedQueue;
let drain;
let running = false;
let stopped = false;
let requested = false;
let pumping;
let drainIdleWake;
let drainIdleWakeRequested = false;
let restartFenceWake;
let releaseRestartFenceWake = () => {};
let suspensionDrainPending = false;
let unsubscribeSuspension;
let pollTimer;
let lastPrunedAt = 0;
let admissionTail = Promise.resolve();
const withAdmissionClaimLock = createAdmissionClaimLock();
let stopTask;
let lastReportedActive = false;
const clearSuspensionSubscription = () => {
suspensionDrainPending = false;
unsubscribeSuspension?.();
unsubscribeSuspension = void 0;
};
const reportError = (error) => {
try {
options.onError?.(error);
} catch {}
};
const publishActivity = () => {
const active = activeInspections.size + activeDeliveries.size > 0 || running && (requested || pumping !== void 0);
if (active === lastReportedActive) return;
lastReportedActive = active;
try {
options.onActivityChange?.(active);
} catch (error) {
reportError(error);
}
};
const createStoppedError = () => options.createStoppedError?.() ?? /* @__PURE__ */ new Error("Channel ingress monitor is stopped.");
const getQueue = () => queue ??= queueFactory();
const ensureQueueAvailable = () => {
try {
getQueue();
} catch (error) {
throw new ChannelIngressUnavailableError(`Channel ingress queue is unavailable: ${formatErrorMessage(error)}`, { cause: error });
}
};
const isAborted = () => drainAbortSignal.aborted;
const inspect = options.inspectAsync ?? options.inspect;
const waitForActiveDeliveries = () => waitForPending(() => activeDeliveries);
const waitForPumpIdle = () => waitForPending(() => pumping ? [pumping] : [], true);
const waitForDeferredClaims = () => waitForPending(() => deferredClaims);
const getDrain = () => {
drain ??= createChannelIngressDrain({
...options.drain,
queue: getQueue(),
abortSignal: drainAbortSignal,
now,
retryPolicy: options.drain?.retryPolicy ?? {
maxAttempts: 8,
deadLetterMinAgeMs: 864e5
},
formatError: options.drain?.formatError ?? formatErrorMessage,
dispatchClaimedEvent: async (claim, lifecycle) => {
const inspection = (async () => {
if (!running || isAborted() || lifecycle.abortSignal.aborted) {
await lifecycle.onCancelled?.();
return;
}
let decoded;
if (options.payload.storage === "raw-event") {
const stored = claim.payload;
if (!stored || typeof stored.rawEvent !== "string") throw options.payload.createClaimError("invalid-version", claim);
decoded = {
version: stored.version,
body: stored.rawEvent
};
} else decoded = options.payload.decode(claim.payload, { claim });
if (decoded.version !== options.payload.version) throw options.payload.createClaimError("invalid-version", claim);
const raw = options.payload.deserialize(decoded.body, { claim });
const claimedLaneKey = claim.laneKey ?? options.drain?.deriveLaneKey?.(claim);
const facts = await inspect(raw, {
phase: "claim",
claimedId: claim.id,
claimedLaneKey
});
if (!running || isAborted() || lifecycle.abortSignal.aborted) {
await lifecycle.onCancelled?.();
return;
}
if (!facts || facts.eventId !== claim.id || facts.laneKey !== claimedLaneKey) throw options.payload.createClaimError("identity-mismatch", claim);
return { raw };
})();
activeInspections.add(inspection);
publishActivity();
let prepared;
try {
prepared = await inspection;
} finally {
activeInspections.delete(inspection);
if (!prepared) publishActivity();
}
if (!prepared) return { kind: "deferred" };
const { raw } = prepared;
let handedOff = false;
let deferredHandoff = false;
let releasedStartCapacity = false;
let deliverySettled = false;
const releaseStartCapacity = () => {
if (releasedStartCapacity || deliverySettled || deferredStartCapacity >= deferredStartCapacityLimit) return;
releasedStartCapacity = true;
deferredStartCapacity += 1;
requestDrain();
};
const deferredClaim = options.deferredClaims ? createDeferredCore() : void 0;
let deferredClaimSettled = false;
const settleDeferredClaim = () => {
if (!deferredClaim || deferredClaimSettled) return;
deferredClaimSettled = true;
lifecycle.abortSignal.removeEventListener("abort", settleDeferredClaim);
deferredClaims.delete(deferredClaim.promise);
deferredClaim.resolve();
};
if (options.deferredClaims === "settle-on-abort") {
lifecycle.abortSignal.addEventListener("abort", settleDeferredClaim, { once: true });
if (lifecycle.abortSignal.aborted) settleDeferredClaim();
}
const trackDeferredClaim = () => {
if (deferredClaim && !deferredClaimSettled) deferredClaims.add(deferredClaim.promise);
};
const settleLifecycle = async (settle, deferred = true) => {
handedOff = true;
if (deferred) deferredHandoff = true;
trackDeferredClaim();
try {
await settle();
requestDrain();
} finally {
settleDeferredClaim();
}
};
const wrappedLifecycle = {
...lifecycle,
admission: "exclusive",
onAdopted: () => settleLifecycle(() => lifecycle.onAdopted(), false),
onDeferred: () => {
handedOff = true;
deferredHandoff = true;
trackDeferredClaim();
lifecycle.onDeferred();
releaseStartCapacity();
},
onAdoptionFinalizing: () => {
handedOff = true;
deferredHandoff = true;
trackDeferredClaim();
lifecycle.onAdoptionFinalizing();
},
onFailed: (error) => settleLifecycle(() => lifecycle.onFailed?.(error)),
onCancelled: () => settleLifecycle(() => lifecycle.onCancelled?.()),
onAbandoned: () => settleLifecycle(() => lifecycle.onAbandoned())
};
const delivery = Promise.resolve().then(() => {
if (!running || isAborted() || lifecycle.abortSignal.aborted) return wrappedLifecycle.onCancelled?.();
return options.deliver(raw, wrappedLifecycle, claim);
}).finally(() => {
deliverySettled = true;
activeDeliveries.delete(delivery);
if (releasedStartCapacity) {
releasedStartCapacity = false;
deferredStartCapacity -= 1;
}
publishActivity();
});
activeDeliveries.add(delivery);
publishActivity();
let result;
try {
result = await delivery;
} catch (error) {
if (deferredHandoff && deferredClaim && !deferredClaimSettled) {
await wrappedLifecycle.onFailed?.(error);
return { kind: "deferred" };
}
if (isAborted() || lifecycle.abortSignal.aborted) return {
kind: "failed-retryable",
error
};
throw error;
}
if (result?.kind === "failed-retryable") {
if (deferredHandoff && deferredClaim && !deferredClaimSettled) {
await wrappedLifecycle.onFailed?.(result.error);
return { kind: "deferred" };
}
return result;
}
if (result?.kind === "completed" && !deferredHandoff) return result;
if (result?.kind === "deferred" && !deferredHandoff) wrappedLifecycle.onDeferred();
if (deferredHandoff) return { kind: "deferred" };
if (isAborted() || lifecycle.abortSignal.aborted) return {
kind: "failed-retryable",
error: createStoppedError()
};
if (!handedOff) await wrappedLifecycle.onAdopted();
return { kind: "completed" };
}
}, true);
return drain;
};
const pruneIfDue = async (owner) => {
if (owner === "admission" !== pruneIntervalMs <= 0) return;
const currentTime = now();
if (owner === "pump" && currentTime - lastPrunedAt < pruneIntervalMs) return;
await getQueue().prune({
...pruneOptions,
now: currentTime
});
lastPrunedAt = currentTime;
};
const scheduleDrainIdleWake = (activeDrain) => {
if (drainIdleWake) {
drainIdleWakeRequested = true;
return;
}
drainIdleWakeRequested = false;
const wake = activeDrain.waitForIdle();
drainIdleWake = wake;
wake.then(() => {
if (drainIdleWake !== wake) return;
const shouldRearm = drainIdleWakeRequested && running && !isAborted();
drainIdleWake = void 0;
drainIdleWakeRequested = false;
if (shouldRearm) scheduleDrainIdleWake(activeDrain);
requestDrain();
}, (error) => {
if (drainIdleWake === wake) {
drainIdleWake = void 0;
drainIdleWakeRequested = false;
}
reportError(error);
});
};
const runPump = async () => {
try {
for (;;) {
requested = false;
await pruneIfDue("pump");
if (!running || isAborted()) break;
const activeDrain = getDrain();
const { started } = await withAdmissionClaimLock(() => activeDrain.drainOnce({ shouldStop: () => !running || isAborted() || getGatewaySuspendAdmissionPhase() !== "accepting" || options.drain?.startLimit !== void 0 && activeDeliveries.size + activeInspections.size - deferredStartCapacity >= options.drain.startLimit }));
if (waitForDeliveryIdleBeforeRepump) {
await waitForActiveDeliveries();
await activeDrain.waitForIdle();
} else if (started > 0) scheduleDrainIdleWake(activeDrain);
if (!running || isAborted() || !requested && (!waitForDeliveryIdleBeforeRepump || started === 0)) break;
}
} catch (error) {
reportError(error);
} finally {
pumping = void 0;
if (!running || isAborted()) requested = false;
else if (requested) requestDrain();
publishActivity();
}
};
const scheduleRestartFenceWake = () => {
if (restartFenceWake) return;
const localWake = createDeferredCore();
releaseRestartFenceWake = localWake.resolve;
restartFenceWake = Promise.race([waitForGatewayRestartFenceSettlement(), localWake.promise]).then(() => {
restartFenceWake = void 0;
releaseRestartFenceWake = () => {};
requestDrain();
});
};
drainAbortSignal.addEventListener("abort", () => {
releaseRestartFenceWake();
clearSuspensionSubscription();
}, { once: true });
const requestDrain = () => {
if (!running || isAborted() || getGatewayRestartDrainSignal().aborted) {
requested = false;
releaseRestartFenceWake();
publishActivity();
return;
}
if (isGatewayRestartDraining()) {
requested = true;
scheduleRestartFenceWake();
publishActivity();
return;
}
if (getGatewaySuspendAdmissionPhase() !== "accepting") {
suspensionDrainPending = true;
requested = false;
publishActivity();
return;
}
suspensionDrainPending = false;
requested = true;
if (pumping) {
publishActivity();
return;
}
pumping = options.runPumpTask ? options.runPumpTask(runPump) : runPump();
publishActivity();
};
const clearPollTimer = () => {
clearInterval(pollTimer);
pollTimer = void 0;
};
const pause = async () => {
running = false;
requested = false;
releaseRestartFenceWake();
clearPollTimer();
publishActivity();
await waitForPumpIdle();
};
const admitOnce = async (params) => {
let lastError;
for (const delayMs of options.appendRetryDelaysMs ?? DEFAULT_APPEND_RETRY_DELAYS_MS) {
if (delayMs > 0) await sleep(delayMs);
try {
return await getQueue().enqueue(params.facts.eventId, params.payload, {
receivedAt: params.receivedAt,
laneKey: params.facts.laneKey
});
} catch (error) {
if (hasSqliteWorkerOutcomeUnknown(error)) throw error;
lastError = error;
}
}
if (lastError instanceof Error) throw lastError;
throw new Error(lastError === void 0 ? "Channel ingress append failed without an error." : formatErrorMessage(lastError), { cause: lastError });
};
const assertAdmissionOpen = () => {
if (stopped && options.admissionMode !== "durable-after-stop" || options.admissionMode === "while-running" && !running || options.abortSignal?.aborted && options.admissionMode !== "durable-after-stop") throw createStoppedError();
};
const admitRaw = async (raw, admitOptions) => {
try {
const facts = admitOptions.facts ?? await inspect(raw, { phase: "admission" });
if (!facts) return { kind: "ignored" };
admitOptions.pruneTask ??= pruneIfDue("admission");
await admitOptions.pruneTask;
const { receivedAt } = admitOptions;
const body = options.payload.serialize(raw, {
facts,
receivedAt
});
const payload = options.payload.storage === "raw-event" ? {
version: options.payload.version,
rawEvent: body
} : options.payload.encode({
version: options.payload.version,
body
});
const queueResult = await admitOnce({
facts,
payload,
receivedAt
});
admitOptions.onDurablyAdmitted();
await options.onDurableAdmission?.(raw, {
facts,
receivedAt,
isNew: queueResult.kind === "accepted"
});
return {
kind: "durable",
queueResult
};
} catch (error) {
await options.onAdmissionFailure?.(raw, error);
throw error;
}
};
const scheduleAdmission = async (work, admitOptions) => {
assertAdmissionOpen();
const receivedAt = admitOptions?.receivedAt ?? now();
let durablyAdmitted = false;
const admission = admissionTail.then(() => withAdmissionClaimLock(() => work({
receivedAt,
onDurablyAdmitted: () => {
durablyAdmitted = true;
}
})));
admissionTail = admission.then(() => void 0, () => void 0);
try {
return await admission;
} finally {
if (durablyAdmitted) requestDrain();
}
};
return {
admit: (raw, admitOptions) => scheduleAdmission((sharedOptions) => admitRaw(raw, {
...sharedOptions,
...admitOptions?.facts ? { facts: admitOptions.facts } : {}
}), admitOptions),
admitBatch: (rawEvents, admitOptions) => scheduleAdmission(async (sharedOptions) => {
const results = [];
for (const raw of rawEvents) results.push(await admitRaw(raw, sharedOptions));
return results;
}, admitOptions),
start: () => {
if (running || stopped || isAborted()) return;
ensureQueueAvailable();
running = true;
unsubscribeSuspension ??= onGatewaySuspendAdmissionChange((phase) => {
if (!running) return;
if (phase !== "accepting") {
if (requested || pumping) {
suspensionDrainPending = true;
requested = false;
publishActivity();
}
return;
}
if (suspensionDrainPending) requestDrain();
});
pollTimer = setInterval(requestDrain, options.pollIntervalMs);
pollTimer.unref?.();
requestDrain();
},
ensureQueueAvailable,
requestDrain,
pause,
stop: () => {
stopTask ??= (async () => {
stopped = true;
running = false;
requested = false;
clearSuspensionSubscription();
releaseRestartFenceWake();
clearPollTimer();
publishActivity();
await admissionTail;
shutdown.abort(createStoppedError());
await waitForPumpIdle();
await waitForPending(() => activeInspections);
if (options.waitForDeliveryIdleOnStop !== false) {
await waitForActiveDeliveries();
await drain?.waitForIdle();
}
if (options.deferredClaims && options.deferredClaims !== "manual") await waitForDeferredClaims();
await drain?.dispose({ waitForSettlements: true });
})();
return stopTask;
},
waitForIdle: async () => {
for (;;) {
await admissionTail;
await waitForPumpIdle();
await waitForPending(() => activeInspections);
await waitForActiveDeliveries();
await drain?.waitForIdle();
await drainIdleWake;
await restartFenceWake;
if (!drainIdleWake && !pumping && activeInspections.size === 0 && activeDeliveries.size === 0 && !requested) return;
}
},
waitForDeferredClaims,
waitForPumpIdle,
isRunning: () => running,
isStopped: () => stopped
};
}
//#endregion
export { createChannelIngressMonitor as n, CHANNEL_INGRESS_RETENTION_DEFAULTS as t };